refactor: consolidated configs

2026-01-21 17:21:09 +01:00 · 2024-11-18 16:48:09 +08:00
parent 459d94ad96
commit a3b286558b
28 changed files with 329 additions and 529 deletions
--- a/.github/workflows/docker-crawlab.yml
+++ b/.github/workflows/docker-crawlab.yml
@@ -282,7 +282,7 @@ jobs:
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
-      crawlab:
+      master:
        image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }}
        env:
          CRAWLAB_NODE_MASTER: Y
@@ -295,6 +295,10 @@ jobs:
          --health-interval 30s
          --health-timeout 10s
          --health-retries 5
+      worker:
+        image: ghcr.io/${{ github.repository_owner }}/crawlab:${{ needs.setup.outputs.version }}
+        env:
+          CRAWLAB_NODE_MASTER: N
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
--- a/backend/conf/config.yml
+++ b/backend/conf/config.yml
@@ -1,27 +1,2 @@
-# Crawlab Configuration File
 edition: global.edition.community
 version: v0.7.0
-
-mongo:
-  host: localhost
-  port: 27017
-  db: crawlab_test
-  username: ""
-  password: ""
-  authSource: "admin"
-
-server:
-  host: 0.0.0.0
-  port: 8000
-
-grpc:
-  address: localhost:9666
-  server:
-    address: 0.0.0.0:9666
-  authKey: Crawlab2021!
-
-api:
-  endpoint: http://localhost:8000
-
-log:
-  path: /var/log/crawlab
--- a/core/apps/api.go
+++ b/core/apps/api.go
@@ -5,11 +5,9 @@ import (
 	"errors"
 	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/controllers"
-	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/middlewares"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"net"
 	"net/http"
 	"sync"
@@ -18,22 +16,14 @@ import (

 func init() {
 	// set gin mode
-	if viper.GetString("gin.mode") == "" {
-		gin.SetMode(gin.ReleaseMode)
-	} else {
-		gin.SetMode(viper.GetString("gin.mode"))
-	}
+	gin.SetMode(utils.GetGinMode())
 }

 type Api struct {
-	// dependencies
-	interfaces.WithConfigPath
-
 	// internals
-	app   *gin.Engine
-	ln    net.Listener
-	srv   *http.Server
-	ready bool
+	app *gin.Engine
+	ln  net.Listener
+	srv *http.Server
 }

 func (app *Api) Init() {
@@ -46,8 +36,8 @@ func (app *Api) Init() {

 func (app *Api) Start() {
 	// address
-	host := viper.GetString("server.host")
-	port := viper.GetString("server.port")
+	host := utils.GetServerHost()
+	port := utils.GetServerPort()
 	address := net.JoinHostPort(host, port)

 	// http server
@@ -62,7 +52,6 @@ func (app *Api) Start() {
 	if err != nil {
 		panic(err)
 	}
-	app.ready = true

 	// serve
 	if err := http.Serve(app.ln, app.app); err != nil {
@@ -95,17 +84,13 @@ func (app *Api) GetHttpServer() *http.Server {
 	return app.srv
 }

-func (app *Api) Ready() (ok bool) {
-	return app.ready
-}
-
 func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
 	return initModule(name, func() error {
 		return fn(app.app)
 	})
 }

-func NewApi() *Api {
+func newApi() *Api {
 	api := &Api{
 		app: gin.New(),
 	}
@@ -118,7 +103,7 @@ var apiOnce sync.Once

 func GetApi() *Api {
 	apiOnce.Do(func() {
-		api = NewApi()
+		api = newApi()
 	})
 	return api
 }
--- a/core/apps/docker.go
+++ b/core/apps/docker.go
@@ -1,189 +0,0 @@
-package apps
-
-import (
-	"bufio"
-	"fmt"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	"github.com/crawlab-team/crawlab/core/sys_exec"
-	"github.com/crawlab-team/crawlab/core/utils"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/imroc/req"
-	"github.com/spf13/viper"
-	"os"
-	"os/exec"
-	"strings"
-	"time"
-)
-
-type Docker struct {
-	// parent
-	parent ServerApp
-
-	// dependencies
-	interfaces.WithConfigPath
-
-	// seaweedfs log
-	fsLogFilePath string
-	fsLogFile     *os.File
-	fsReady       bool
-}
-
-func (app *Docker) Init() {
-	var err error
-	app.fsLogFile, err = os.OpenFile(app.fsLogFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0777))
-	if err != nil {
-		trace.PrintError(err)
-	}
-
-	// replace paths
-	if err := app.replacePaths(); err != nil {
-		panic(err)
-	}
-
-	// start nginx
-	go app.startNginx()
-
-	// start seaweedfs
-	go app.startSeaweedFs()
-}
-
-func (app *Docker) Start() {
-	// import demo
-	//if utils.IsDemo() && utils.InitializedDemo() {
-	//	go app.importDemo()
-	//}
-}
-
-func (app *Docker) Wait() {
-	DefaultWait()
-}
-
-func (app *Docker) Stop() {
-}
-
-func (app *Docker) GetParent() (parent ServerApp) {
-	return app.parent
-}
-
-func (app *Docker) SetParent(parent ServerApp) {
-	app.parent = parent
-}
-
-func (app *Docker) Ready() (ok bool) {
-	return app.fsReady &&
-		app.parent.GetApi().Ready()
-}
-
-func (app *Docker) replacePaths() (err error) {
-	// read
-	indexHtmlPath := "/app/dist/index.html"
-	indexHtmlBytes, err := os.ReadFile(indexHtmlPath)
-	if err != nil {
-		return trace.TraceError(err)
-	}
-	indexHtml := string(indexHtmlBytes)
-
-	// replace paths
-	baseUrl := viper.GetString("base.url")
-	if baseUrl != "" {
-		indexHtml = app._replacePath(indexHtml, "js", baseUrl)
-		indexHtml = app._replacePath(indexHtml, "css", baseUrl)
-		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
-		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
-		indexHtml = app._replacePath(indexHtml, "window.VUE_APP_API_BASE_URL = '", baseUrl)
-	}
-
-	// replace path of baidu tongji
-	initBaiduTongji := viper.GetString("string")
-	if initBaiduTongji != "" {
-		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_BAIDU_TONGJI = ''", fmt.Sprintf("window.VUE_APP_INIT_BAIDU_TONGJI = '%s'", initBaiduTongji))
-	}
-
-	// replace path of umeng
-	initUmeng := viper.GetString("string")
-	if initUmeng != "" {
-		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_UMENG = ''", fmt.Sprintf("window.VUE_APP_INIT_UMENG = '%s'", initUmeng))
-	}
-
-	// write
-	if err := os.WriteFile(indexHtmlPath, []byte(indexHtml), os.FileMode(0766)); err != nil {
-		return trace.TraceError(err)
-	}
-
-	return nil
-}
-
-func (app *Docker) _replacePath(text, path, baseUrl string) (res string) {
-	text = strings.ReplaceAll(text, path, fmt.Sprintf("%s/%s", baseUrl, path))
-	return text
-}
-
-func (app *Docker) startNginx() {
-	cmd := exec.Command("service", "nginx", "start")
-	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
-		for scanner.Scan() {
-			line := fmt.Sprintf("[nginx] %s\n", scanner.Text())
-			_, _ = os.Stdout.WriteString(line)
-		}
-	})
-	if err := cmd.Run(); err != nil {
-		trace.PrintError(err)
-	}
-}
-
-func (app *Docker) startSeaweedFs() {
-	seaweedFsDataPath := "/data/seaweedfs"
-	if !utils.Exists(seaweedFsDataPath) {
-		_ = os.MkdirAll(seaweedFsDataPath, os.FileMode(0777))
-	}
-	cmd := exec.Command("weed", "server",
-		"-dir", "/data",
-		"-master.dir", seaweedFsDataPath,
-		"-volume.dir.idx", seaweedFsDataPath,
-		"-ip", "localhost",
-		"-volume.port", "9999",
-		"-volume.minFreeSpace", "1GiB",
-		"-filer",
-	)
-	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
-		for scanner.Scan() {
-			line := fmt.Sprintf("[seaweedfs] %s\n", scanner.Text())
-			_, _ = app.fsLogFile.WriteString(line)
-		}
-	})
-	go func() {
-		if err := cmd.Run(); err != nil {
-			trace.PrintError(err)
-		}
-	}()
-	for {
-		_, err := req.Get("http://localhost:8888")
-		if err != nil {
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		app.fsReady = true
-		return
-	}
-}
-
-func NewDocker(svr ServerApp) *Docker {
-	dck := &Docker{
-		parent:        svr,
-		fsLogFilePath: "/var/log/weed.log",
-	}
-
-	dck.Init()
-
-	return dck
-}
-
-var dck *Docker
-
-func GetDocker(svr ServerApp) *Docker {
-	if dck != nil {
-		return dck
-	}
-	dck = NewDocker(svr)
-	return dck
-}
--- a/core/apps/interfaces.go
+++ b/core/apps/interfaces.go
@@ -17,23 +17,10 @@ type ApiApp interface {
 	App
 	GetGinEngine() (engine *gin.Engine)
 	GetHttpServer() (svr *http.Server)
-	Ready() (ok bool)
-}
-
-type NodeApp interface {
-	App
-	interfaces.WithConfigPath
 }

 type ServerApp interface {
-	NodeApp
+	App
 	GetApi() (api ApiApp)
 	GetNodeService() (masterSvc interfaces.NodeService)
 }
-
-type DockerApp interface {
-	App
-	GetParent() (parent NodeApp)
-	SetParent(parent NodeApp)
-	Ready() (ok bool)
-}
--- a/core/apps/server.go
+++ b/core/apps/server.go
@@ -3,7 +3,6 @@ package apps
 import (
 	"fmt"
 	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/core/config"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/node/service"
 	"github.com/crawlab-team/crawlab/core/utils"
@@ -14,16 +13,9 @@ import (
 )

 type Server struct {
-	// settings
-	grpcAddress interfaces.Address
-
-	// dependencies
-	interfaces.WithConfigPath
-
 	// modules
 	nodeSvc interfaces.NodeService
 	api     *Api
-	dck     *Docker

 	// internals
 	quit chan int
@@ -39,11 +31,6 @@ func (app *Server) Init() {

 func (app *Server) Start() {
 	if utils.IsMaster() {
-		// start docker app
-		if utils.IsDocker() {
-			go app.dck.Start()
-		}
-
 		// start api
 		go app.api.Start()
 	}
@@ -71,9 +58,6 @@ func (app *Server) GetNodeService() interfaces.NodeService {

 func (app *Server) logNodeInfo() {
 	log.Infof("current node type: %s", utils.GetNodeType())
-	if utils.IsDocker() {
-		log.Infof("running in docker container")
-	}
 }

 func (app *Server) initPprof() {
@@ -84,22 +68,16 @@ func (app *Server) initPprof() {
 	}
 }

-func newServer() NodeApp {
+func newServer() App {
 	// server
 	svr := &Server{
-		WithConfigPath: config.NewConfigPathService(),
-		quit:           make(chan int, 1),
+		quit: make(chan int, 1),
 	}

 	// master modules
 	if utils.IsMaster() {
 		// api
 		svr.api = GetApi()
-
-		// docker
-		if utils.IsDocker() {
-			svr.dck = GetDocker(svr)
-		}
 	}

 	// node service
@@ -112,10 +90,10 @@ func newServer() NodeApp {
 	return svr
 }

-var server NodeApp
+var server App
 var serverOnce sync.Once

-func GetServer() NodeApp {
+func GetServer() App {
 	serverOnce.Do(func() {
 		server = newServer()
 	})
--- a/core/config/config.go
+++ b/core/config/config.go
@@ -1,6 +1,7 @@
 package config

 import (
+	"errors"
 	"strings"

 	"github.com/apex/log"
@@ -67,11 +68,9 @@ func (c *Config) Init() (err error) {

 	// read in config
 	if err := viper.ReadInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+		var configFileNotFoundError viper.ConfigFileNotFoundError
+		if errors.As(err, &configFileNotFoundError) {
 			log.Warn("No config file found. Using default values.")
-		} else {
-			log.Errorf("Error reading config file: %s", err)
-			return err
 		}
 	}

@@ -79,25 +78,12 @@ func (c *Config) Init() (err error) {
 }

 func (c *Config) setDefaults() {
-	viper.SetDefault("edition", "global.edition.community")
-
 	viper.SetDefault("mongo.host", "localhost")
 	viper.SetDefault("mongo.port", 27017)
 	viper.SetDefault("mongo.db", "crawlab_test")
 	viper.SetDefault("mongo.username", "")
 	viper.SetDefault("mongo.password", "")
 	viper.SetDefault("mongo.authSource", "admin")
-
-	viper.SetDefault("server.host", "0.0.0.0")
-	viper.SetDefault("server.port", 8000)
-
-	viper.SetDefault("grpc.address", "localhost:9666")
-	viper.SetDefault("grpc.server.address", "0.0.0.0:9666")
-	viper.SetDefault("grpc.authKey", "Crawlab2021!")
-
-	viper.SetDefault("api.endpoint", "http://localhost:8000")
-
-	viper.SetDefault("log.path", "/var/log/crawlab")
 }

 func (c *Config) initLogLevel() {
--- a/core/config/config_test.go
+++ b/core/config/config_test.go
@@ -33,7 +33,8 @@ func TestInitConfig(t *testing.T) {
 	assert.Equal(t, "crawlab_test", viper.GetString("mongo.db"), "Unexpected default value for mongo.db")
 	assert.Equal(t, "0.0.0.0", viper.GetString("server.host"), "Unexpected default value for server.host")
 	assert.Equal(t, 8000, viper.GetInt("server.port"), "Unexpected default value for server.port")
-	assert.Equal(t, "localhost:9666", viper.GetString("grpc.address"), "Unexpected default value for grpc.address")
+	assert.Equal(t, "localhost", viper.GetString("grpc.host"), "Unexpected default value for grpc.host")
+	assert.Equal(t, 9666, viper.GetInt("grpc.port"), "Unexpected default value for grpc.port")
 	assert.Equal(t, "Crawlab2021!", viper.GetString("grpc.authKey"), "Unexpected default value for grpc.authKey")
 	assert.Equal(t, "http://localhost:8000", viper.GetString("api.endpoint"), "Unexpected default value for api.endpoint")
 	assert.Equal(t, "/var/log/crawlab", viper.GetString("log.path"), "Unexpected default value for log.path")
--- a/core/controllers/base_file.go
+++ b/core/controllers/base_file.go
@@ -6,8 +6,8 @@ import (
 	log2 "github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/fs"
 	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"io"
 	"os"
 	"path/filepath"
@@ -296,7 +296,7 @@ func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) {
 }

 func getBaseFileFsSvc(rootPath string) (svc interfaces.FsService, err error) {
-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	fsSvc := fs.NewFsService(filepath.Join(workspacePath, rootPath))

 	return fsSvc, nil
--- a/core/controllers/router.go
+++ b/core/controllers/router.go
@@ -8,19 +8,26 @@ import (
 	"github.com/gin-gonic/gin"
 )

+// RouterGroups defines the different authentication levels for API routes
 type RouterGroups struct {
-	AuthGroup      *gin.RouterGroup
-	AnonymousGroup *gin.RouterGroup
+	AuthGroup      *gin.RouterGroup // Routes requiring full authentication
+	SyncAuthGroup  *gin.RouterGroup // Routes for sync operations with special auth
+	AnonymousGroup *gin.RouterGroup // Public routes that don't require auth
 }

+// NewRouterGroups initializes the router groups with their respective middleware
 func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) {
 	return &RouterGroups{
 		AuthGroup:      app.Group("/", middlewares.AuthorizationMiddleware()),
+		SyncAuthGroup:  app.Group("/", middlewares.SyncAuthorizationMiddleware()),
 		AnonymousGroup: app.Group("/"),
 	}
 }

+// RegisterController registers a generic controller with standard CRUD endpoints
+// and any additional custom actions
 func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseController[T]) {
+	// Track registered paths to avoid duplicates
 	actionPaths := make(map[string]bool)
 	for _, action := range ctr.actions {
 		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
@@ -37,12 +44,15 @@ func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *Bas
 	registerBuiltinHandler(group, http.MethodDelete, basePath+"", ctr.DeleteList, actionPaths)
 }

+// RegisterActions registers a list of custom action handlers to a route group
 func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) {
 	for _, action := range actions {
 		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
 	}
 }

+// registerBuiltinHandler registers a standard handler if it hasn't been overridden
+// by a custom action
 func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handlerFunc gin.HandlerFunc, existingActionPaths map[string]bool) {
 	key := method + " - " + path
 	_, ok := existingActionPaths[key]
@@ -52,10 +62,14 @@ func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handler
 	group.Handle(method, path, handlerFunc)
 }

+// InitRoutes configures all API routes for the application
 func InitRoutes(app *gin.Engine) (err error) {
-	// routes groups
+	// Initialize route groups with different auth levels
 	groups := NewRouterGroups(app)

+	// Register resource controllers with their respective endpoints
+	// Each RegisterController call sets up standard CRUD operations
+	// Additional custom actions can be specified in the controller initialization
 	RegisterController(groups.AuthGroup, "/data/collections", NewController[models.DataCollection]())
 	RegisterController(groups.AuthGroup, "/environments", NewController[models.Environment]())
 	RegisterController(groups.AuthGroup, "/nodes", NewController[models.Node]())
@@ -282,6 +296,7 @@ func InitRoutes(app *gin.Engine) (err error) {
 		},
 	}...))

+	// Register standalone action routes that don't fit the standard CRUD pattern
 	RegisterActions(groups.AuthGroup, "/export", []Action{
 		{
 			Method:      http.MethodPost,
@@ -351,6 +366,21 @@ func InitRoutes(app *gin.Engine) (err error) {
 		},
 	})

+	// Register sync routes that require special authentication
+	RegisterActions(groups.SyncAuthGroup, "/sync", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/scan",
+			HandlerFunc: GetSyncScan,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id/download",
+			HandlerFunc: GetSyncDownload,
+		},
+	})
+
+	// Register public routes that don't require authentication
 	RegisterActions(groups.AnonymousGroup, "/system-info", []Action{
 		{
 			Path:        "",
@@ -370,18 +400,6 @@ func InitRoutes(app *gin.Engine) (err error) {
 			HandlerFunc: PostLogout,
 		},
 	})
-	RegisterActions(groups.AnonymousGroup, "/sync", []Action{
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/scan",
-			HandlerFunc: GetSyncScan,
-		},
-		{
-			Method:      http.MethodGet,
-			Path:        "/:id/download",
-			HandlerFunc: GetSyncDownload,
-		},
-	})

 	return nil
 }
--- a/core/controllers/spider.go
+++ b/core/controllers/spider.go
@@ -18,7 +18,6 @@ import (
 	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/crawlab-team/crawlab/trace"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	mongo2 "go.mongodb.org/mongo-driver/mongo"
@@ -389,7 +388,7 @@ func DeleteSpiderById(c *gin.Context) {
 		for _, id := range taskIds {
 			go func(id string) {
 				// delete task logs
-				logPath := filepath.Join(viper.GetString("log.path"), id)
+				logPath := filepath.Join(utils.GetTaskLogPath(), id)
 				if err := os.RemoveAll(logPath); err != nil {
 					log.Warnf("failed to remove task log directory: %s", logPath)
 				}
@@ -494,7 +493,7 @@ func DeleteSpiderList(c *gin.Context) {
 		for _, id := range taskIds {
 			go func(id string) {
 				// delete task logs
-				logPath := filepath.Join(viper.GetString("log.path"), id)
+				logPath := filepath.Join(utils.GetTaskLogPath(), id)
 				if err := os.RemoveAll(logPath); err != nil {
 					log.Warnf("failed to remove task log directory: %s", logPath)
 				}
@@ -705,7 +704,7 @@ func GetSpiderResults(c *gin.Context) {
 }

 func getSpiderFsSvc(s *models.Spider) (svc interfaces.FsService, err error) {
-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	fsSvc := fs.NewFsService(filepath.Join(workspacePath, s.Id.Hex()))

 	return fsSvc, nil
--- a/core/controllers/sync.go
+++ b/core/controllers/sync.go
@@ -3,7 +3,6 @@ package controllers
 import (
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"net/http"
 	"path/filepath"
 )
@@ -12,7 +11,7 @@ func GetSyncScan(c *gin.Context) {
 	id := c.Param("id")
 	path := c.Query("path")

-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	dirPath := filepath.Join(workspacePath, id, path)
 	files, err := utils.ScanDirectory(dirPath)
 	if err != nil {
@@ -25,7 +24,7 @@ func GetSyncScan(c *gin.Context) {
 func GetSyncDownload(c *gin.Context) {
 	id := c.Param("id")
 	path := c.Query("path")
-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	filePath := filepath.Join(workspacePath, id, path)
 	c.File(filePath)
 }
--- a/core/controllers/task.go
+++ b/core/controllers/task.go
@@ -13,7 +13,6 @@ import (
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/crawlab-team/crawlab/db/mongo"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	mongo2 "go.mongodb.org/mongo-driver/mongo"
@@ -208,7 +207,7 @@ func DeleteTaskById(c *gin.Context) {
 	}

 	// delete task logs
-	logPath := filepath.Join(viper.GetString("log.path"), id.Hex())
+	logPath := filepath.Join(utils.GetTaskLogPath(), id.Hex())
 	if err := os.RemoveAll(logPath); err != nil {
 		log2.Warnf("failed to remove task log directory: %s", logPath)
 	}
@@ -257,7 +256,7 @@ func DeleteList(c *gin.Context) {
 	for _, id := range payload.Ids {
 		go func(id string) {
 			// delete task logs
-			logPath := filepath.Join(viper.GetString("log.path"), id)
+			logPath := filepath.Join(utils.GetTaskLogPath(), id)
 			if err := os.RemoveAll(logPath); err != nil {
 				log2.Warnf("failed to remove task log directory: %s", logPath)
 			}
--- a/core/entity/address.go
+++ b/core/entity/address.go
@@ -1,56 +0,0 @@
-package entity
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-)
-
-type Address struct {
-	Host string
-	Port string
-}
-
-func (a *Address) String() (res string) {
-	return fmt.Sprintf("%s:%s", a.Host, a.Port)
-}
-
-func (a *Address) IsEmpty() (res bool) {
-	return a.Host == "" || a.Port == ""
-}
-
-func (a *Address) Value() (res interface{}) {
-	return a
-}
-
-type AddressOptions struct {
-	Host string
-	Port string
-}
-
-func NewAddress(opts *AddressOptions) (res *Address) {
-	if opts == nil {
-		opts = &AddressOptions{}
-	}
-	//if opts.Host == "" {
-	//	opts.Host = "localhost"
-	//}
-	if opts.Port == "" {
-		opts.Port = "9666"
-	}
-	return &Address{
-		Host: opts.Host,
-		Port: opts.Port,
-	}
-}
-
-func NewAddressFromString(address string) (res *Address, err error) {
-	parts := strings.Split(address, ":")
-	if len(parts) == 1 {
-		return NewAddress(&AddressOptions{Host: parts[0]}), nil
-	} else if len(parts) == 2 {
-		return NewAddress(&AddressOptions{Host: parts[0], Port: parts[1]}), nil
-	} else {
-		return nil, errors.New(fmt.Sprintf("parsing address error: %v", err))
-	}
-}
--- a/core/fs/default.go
+++ b/core/fs/default.go
@@ -2,6 +2,7 @@ package fs

 import (
 	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/viper"
 	"path/filepath"
@@ -15,7 +16,7 @@ func init() {
 	}
 	DefaultWorkspacePath = filepath.Join(rootDir, "crawlab_workspace")

-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	if workspacePath == "" {
 		viper.Set("workspace", DefaultWorkspacePath)
 	}
--- a/core/grpc/client/client.go
+++ b/core/grpc/client/client.go
@@ -8,14 +8,10 @@ import (

 	"github.com/apex/log"
 	"github.com/cenkalti/backoff/v4"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/entity"
 	"github.com/crawlab-team/crawlab/core/grpc/middlewares"
 	"github.com/crawlab-team/crawlab/core/interfaces"
-	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
 	"github.com/crawlab-team/crawlab/core/utils"
 	grpc2 "github.com/crawlab-team/crawlab/grpc"
-	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/connectivity"
 	"google.golang.org/grpc/credentials/insecure"
@@ -26,7 +22,7 @@ type GrpcClient struct {
 	nodeCfgSvc interfaces.NodeConfigService

 	// settings
-	address interfaces.Address
+	address string
 	timeout time.Duration

 	// internals
@@ -70,14 +66,11 @@ func (c *GrpcClient) Stop() (err error) {
 		return nil
 	}

-	// grpc server address
-	address := c.address.String()
-
 	// close connection
 	if err := c.conn.Close(); err != nil {
 		return err
 	}
-	log.Infof("grpc client disconnected from %s", address)
+	log.Infof("grpc client disconnected from %s", c.address)

 	return nil
 }
@@ -124,26 +117,23 @@ func (c *GrpcClient) getRequestData(d interface{}) (data []byte) {

 func (c *GrpcClient) connect() (err error) {
 	op := func() error {
-		// grpc server address
-		address := c.address.String()
-
 		// connection options
 		opts := []grpc.DialOption{
 			grpc.WithTransportCredentials(insecure.NewCredentials()),
-			grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor(c.nodeCfgSvc)),
-			grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor(c.nodeCfgSvc)),
+			grpc.WithChainUnaryInterceptor(middlewares.GetAuthTokenUnaryChainInterceptor()),
+			grpc.WithChainStreamInterceptor(middlewares.GetAuthTokenStreamChainInterceptor()),
 		}

 		// create new client connection
-		c.conn, err = grpc.NewClient(address, opts...)
+		c.conn, err = grpc.NewClient(c.address, opts...)
 		if err != nil {
-			log.Errorf("[GrpcClient] grpc client failed to connect to %s: %v", address, err)
+			log.Errorf("[GrpcClient] grpc client failed to connect to %s: %v", c.address, err)
 			return err
 		}

 		// connect
 		c.conn.Connect()
-		log.Infof("[GrpcClient] grpc client connected to %s", address)
+		log.Infof("[GrpcClient] grpc client connected to %s", c.address)

 		return nil
 	}
@@ -151,26 +141,11 @@ func (c *GrpcClient) connect() (err error) {
 }

 func newGrpcClient() (c *GrpcClient) {
-	client := &GrpcClient{
-		address: entity.NewAddress(&entity.AddressOptions{
-			Host: constants.DefaultGrpcClientRemoteHost,
-			Port: constants.DefaultGrpcClientRemotePort,
-		}),
+	return &GrpcClient{
+		address: utils.GetGrpcAddress(),
 		timeout: 10 * time.Second,
 		stop:    make(chan struct{}),
 	}
-	client.nodeCfgSvc = nodeconfig.GetNodeConfigService()
-
-	if viper.GetString("grpc.address") != "" {
-		address, err := entity.NewAddressFromString(viper.GetString("grpc.address"))
-		if err != nil {
-			log.Errorf("failed to parse grpc address: %s", viper.GetString("grpc.address"))
-			panic(err)
-		}
-		client.address = address
-	}
-
-	return client
 }

 var _client *GrpcClient
--- a/core/grpc/middlewares/auth_token.go
+++ b/core/grpc/middlewares/auth_token.go
@@ -4,13 +4,13 @@ import (
 	"context"
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/errors"
-	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/grpc-ecosystem/go-grpc-middleware/auth"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
 )

-func GetAuthTokenFunc(nodeCfgSvc interfaces.NodeConfigService) grpc_auth.AuthFunc {
+func GetAuthTokenFunc() grpc_auth.AuthFunc {
 	return func(ctx context.Context) (ctx2 context.Context, err error) {
 		// authentication (token verification)
 		md, ok := metadata.FromIncomingContext(ctx)
@@ -29,7 +29,7 @@ func GetAuthTokenFunc(nodeCfgSvc interfaces.NodeConfigService) grpc_auth.AuthFun
 		authKey := res[0]

 		// validate
-		svrAuthKey := nodeCfgSvc.GetAuthKey()
+		svrAuthKey := utils.GetAuthKey()
 		if authKey != svrAuthKey {
 			return ctx, errors.ErrorGrpcUnauthorized
 		}
@@ -38,26 +38,20 @@ func GetAuthTokenFunc(nodeCfgSvc interfaces.NodeConfigService) grpc_auth.AuthFun
 	}
 }

-func GetAuthTokenUnaryChainInterceptor(nodeCfgSvc interfaces.NodeConfigService) grpc.UnaryClientInterceptor {
+func GetAuthTokenUnaryChainInterceptor() grpc.UnaryClientInterceptor {
 	// set auth key
-	md := metadata.Pairs(constants.GrpcHeaderAuthorization, nodeCfgSvc.GetAuthKey())
-	//header := metadata.MD{}
-	//header[constants.GrpcHeaderAuthorization] = []string{nodeCfgSvc.GetAuthKey()}
+	md := metadata.Pairs(constants.GrpcHeaderAuthorization, utils.GetAuthKey())
 	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
 		ctx = metadata.NewOutgoingContext(context.Background(), md)
-		//opts = append(opts, grpc.Header(&header))
 		return invoker(ctx, method, req, reply, cc, opts...)
 	}
 }

-func GetAuthTokenStreamChainInterceptor(nodeCfgSvc interfaces.NodeConfigService) grpc.StreamClientInterceptor {
+func GetAuthTokenStreamChainInterceptor() grpc.StreamClientInterceptor {
 	// set auth key
-	md := metadata.Pairs(constants.GrpcHeaderAuthorization, nodeCfgSvc.GetAuthKey())
-	//header := metadata.MD{}
-	//header[constants.GrpcHeaderAuthorization] = []string{nodeCfgSvc.GetAuthKey()}
+	md := metadata.Pairs(constants.GrpcHeaderAuthorization, utils.GetAuthKey())
 	return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
 		ctx = metadata.NewOutgoingContext(context.Background(), md)
-		//opts = append(opts, grpc.Header(&header))
 		s, err := streamer(ctx, desc, cc, method, opts...)
 		if err != nil {
 			return nil, err
--- a/core/grpc/server/server.go
+++ b/core/grpc/server/server.go
@@ -3,17 +3,13 @@ package server
 import (
 	"fmt"
 	"github.com/apex/log"
-	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/entity"
 	"github.com/crawlab-team/crawlab/core/grpc/middlewares"
-	"github.com/crawlab-team/crawlab/core/interfaces"
-	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
+	"github.com/crawlab-team/crawlab/core/utils"
 	grpc2 "github.com/crawlab-team/crawlab/grpc"
 	grpcmiddleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpcauth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
 	grpcrecovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	errors2 "github.com/pkg/errors"
-	"github.com/spf13/viper"
 	"google.golang.org/grpc"
 	"net"
 	"sync"
@@ -21,17 +17,13 @@ import (

 type GrpcServer struct {
 	// settings
-	cfgPath string
-	address interfaces.Address
+	address string

 	// internals
 	svr     *grpc.Server
 	l       net.Listener
 	stopped bool

-	// dependencies
-	nodeCfgSvc interfaces.NodeConfigService
-
 	// servers
 	NodeSvr             *NodeServiceServer
 	TaskSvr             *TaskServiceServer
@@ -40,29 +32,18 @@ type GrpcServer struct {
 	MetricSvr           *MetricServiceServer
 }

-func (svr *GrpcServer) GetConfigPath() (path string) {
-	return svr.cfgPath
-}
-
-func (svr *GrpcServer) SetConfigPath(path string) {
-	svr.cfgPath = path
-}
-
 func (svr *GrpcServer) Init() {
 	svr.register()
 }

 func (svr *GrpcServer) Start() (err error) {
-	// grpc server binding address
-	address := svr.address.String()
-
 	// listener
-	svr.l, err = net.Listen("tcp", address)
+	svr.l, err = net.Listen("tcp", svr.address)
 	if err != nil {
 		log.Errorf("[GrpcServer] failed to listen: %v", err)
 		return err
 	}
-	log.Infof("[GrpcServer] grpc server listens to %s", address)
+	log.Infof("[GrpcServer] grpc server listens to %s", svr.address)

 	// start grpc server
 	go func() {
@@ -116,25 +97,10 @@ func (svr *GrpcServer) recoveryHandlerFunc(p interface{}) (err error) {
 func newGrpcServer() *GrpcServer {
 	// server
 	svr := &GrpcServer{
-		address: entity.NewAddress(&entity.AddressOptions{
-			Host: constants.DefaultGrpcServerHost,
-			Port: constants.DefaultGrpcServerPort,
-		}),
+		address: utils.GetGrpcServerAddress(),
 	}

-	if viper.GetString("grpc.server.address") != "" {
-		address, err := entity.NewAddressFromString(viper.GetString("grpc.server.address"))
-		if err != nil {
-			log.Fatalf("[GrpcServer] failed to parse grpc server address: %v", err)
-			panic(err)
-		}
-		svr.address = address
-	}
-
-	// node config service
-	svr.nodeCfgSvc = nodeconfig.GetNodeConfigService()
-
-	// servers
+	// services servers
 	svr.NodeSvr = GetNodeServiceServer()
 	svr.ModelBaseServiceSvr = GetModelBaseServiceServer()
 	svr.TaskSvr = GetTaskServiceServer()
@@ -150,11 +116,11 @@ func newGrpcServer() *GrpcServer {
 	svr.svr = grpc.NewServer(
 		grpcmiddleware.WithUnaryServerChain(
 			grpcrecovery.UnaryServerInterceptor(recoveryOpts...),
-			grpcauth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)),
+			grpcauth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc()),
 		),
 		grpcmiddleware.WithStreamServerChain(
 			grpcrecovery.StreamServerInterceptor(recoveryOpts...),
-			grpcauth.StreamServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)),
+			grpcauth.StreamServerInterceptor(middlewares.GetAuthTokenFunc()),
 		),
 	)

--- a/core/middlewares/auth.go
+++ b/core/middlewares/auth.go
@@ -8,7 +8,6 @@ import (
 	"github.com/crawlab-team/crawlab/core/user"
 	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 )

@@ -16,7 +15,7 @@ func AuthorizationMiddleware() gin.HandlerFunc {
 	userSvc, _ := user.GetUserService()
 	return func(c *gin.Context) {
 		// disable auth for test
-		if viper.GetBool("auth.disabled") {
+		if utils.IsAuthDisabled() {
 			u, err := service.NewModelService[models.User]().GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil)
 			if err != nil {
 				utils.HandleErrorInternalServerError(c, err)
@@ -45,3 +44,21 @@ func AuthorizationMiddleware() gin.HandlerFunc {
 		c.Next()
 	}
 }
+
+func SyncAuthorizationMiddleware() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if utils.IsAuthDisabled() {
+			c.Next()
+			return
+		}
+
+		authKey := c.GetHeader("Authorization")
+
+		if authKey != utils.GetAuthKey() {
+			utils.HandleErrorUnauthorized(c, errors.New("invalid auth key"))
+			return
+		}
+
+		c.Next()
+	}
+}
--- a/core/middlewares/cors.go
+++ b/core/middlewares/cors.go
@@ -1,16 +1,20 @@
 package middlewares

-import "github.com/gin-gonic/gin"
+import (
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)

 func CORSMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
-		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
-		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
-		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "DELETE, POST, OPTIONS, GET, PUT")
+		c.Writer.Header().Set("Access-Control-Allow-Origin", utils.GetAllowOrigin())
+		c.Writer.Header().Set("Access-Control-Allow-Credentials", utils.GetAllowCredentials())
+		c.Writer.Header().Set("Access-Control-Allow-Headers", utils.GetAllowHeaders())
+		c.Writer.Header().Set("Access-Control-Allow-Methods", utils.GetAllowMethods())

 		if c.Request.Method == "OPTIONS" {
-			c.AbortWithStatus(204)
+			c.AbortWithStatus(http.StatusNoContent)
 			return
 		}

--- a/core/middlewares/middlewares.go
+++ b/core/middlewares/middlewares.go
--- a/core/task/handler/runner.go
+++ b/core/task/handler/runner.go
@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/crawlab-team/crawlab/core/models/models"
 	"io"
 	"net/http"
 	"os"
@@ -16,6 +15,8 @@ import (
 	"sync"
 	"time"

+	"github.com/crawlab-team/crawlab/core/models/models"
+
 	"github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/constants"
 	"github.com/crawlab-team/crawlab/core/entity"
@@ -220,7 +221,7 @@ func (r *Runner) configureCmd() (err error) {
 	// get cmd instance
 	r.cmd, err = sys_exec.BuildCmd(cmdStr)
 	if err != nil {
-		log.Errorf("Error building command: %v", err)
+		log.Errorf("error building command: %v", err)
 		trace.PrintError(err)
 		return err
 	}
@@ -297,8 +298,8 @@ func (r *Runner) configureEnv() {

 	// Default envs
 	r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex())
-	if viper.GetString("grpc.address") != "" {
-		r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address"))
+	if utils.GetGrpcAddress() != "" {
+		r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+utils.GetGrpcAddress())
 	}
 	if viper.GetString("grpc.authKey") != "" {
 		r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey"))
@@ -317,35 +318,52 @@ func (r *Runner) configureEnv() {
 	}
 }

-func (r *Runner) syncFiles() (err error) {
+func (r *Runner) createHttpRequest(method, path string) (*http.Response, error) {
+	// Construct master URL
 	var id string
-	var workingDir string
 	if r.s.GitId.IsZero() {
 		id = r.s.Id.Hex()
-		workingDir = ""
 	} else {
 		id = r.s.GitId.Hex()
+	}
+	url := fmt.Sprintf("%s/sync/%s%s", utils.GetApiEndpoint(), id, path)
+
+	// Create and execute request
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error creating request: %v", err)
+	}
+
+	for k, v := range r.getHttpRequestHeaders() {
+		req.Header.Set(k, v)
+	}
+
+	return http.DefaultClient.Do(req)
+}
+
+func (r *Runner) syncFiles() (err error) {
+	workingDir := ""
+	if !r.s.GitId.IsZero() {
 		workingDir = r.s.GitRootPath
 	}
-	masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), id)

 	// get file list from master
-	resp, err := http.Get(masterURL + "/scan?path=" + workingDir)
+	resp, err := r.createHttpRequest("GET", "/scan?path="+workingDir)
 	if err != nil {
-		log.Errorf("Error getting file list from master: %v", err)
-		return trace.TraceError(err)
+		log.Errorf("error getting file list from master: %v", err)
+		return err
 	}
 	defer resp.Body.Close()
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		log.Errorf("Error reading response body: %v", err)
-		return trace.TraceError(err)
+		log.Errorf("error reading response body: %v", err)
+		return err
 	}
 	var masterFiles map[string]entity.FsFileInfo
 	err = json.Unmarshal(body, &masterFiles)
 	if err != nil {
-		log.Errorf("Error unmarshaling JSON: %v", err)
-		return trace.TraceError(err)
+		log.Errorf("error unmarshaling JSON: %v", err)
+		return err
 	}

 	// create a map for master files
@@ -357,25 +375,25 @@ func (r *Runner) syncFiles() (err error) {
 	// create working directory if not exists
 	if _, err := os.Stat(r.cwd); os.IsNotExist(err) {
 		if err := os.MkdirAll(r.cwd, os.ModePerm); err != nil {
-			log.Errorf("Error creating worker directory: %v", err)
-			return trace.TraceError(err)
+			log.Errorf("error creating worker directory: %v", err)
+			return err
 		}
 	}

 	// get file list from worker
 	workerFiles, err := utils.ScanDirectory(r.cwd)
 	if err != nil {
-		log.Errorf("Error scanning worker directory: %v", err)
+		log.Errorf("error scanning worker directory: %v", err)
 		return trace.TraceError(err)
 	}

 	// delete files that are deleted on master node
 	for path, workerFile := range workerFiles {
 		if _, exists := masterFilesMap[path]; !exists {
-			log.Infof("Deleting file: %s", path)
+			log.Infof("deleting file: %s", path)
 			err := os.Remove(workerFile.FullPath)
 			if err != nil {
-				log.Errorf("Error deleting file: %v", err)
+				log.Errorf("error deleting file: %v", err)
 			}
 		}
 	}
@@ -398,17 +416,17 @@ func (r *Runner) syncFiles() (err error) {
 				defer wg.Done()

 				if masterFile.IsDir {
-					log.Infof("Directory needs to be synchronized: %s", path)
+					log.Infof("directory needs to be synchronized: %s", path)
 					_err := os.MkdirAll(filepath.Join(r.cwd, path), masterFile.Mode)
 					if _err != nil {
-						log.Errorf("Error creating directory: %v", _err)
+						log.Errorf("error creating directory: %v", _err)
 						err = errors.Join(err, _err)
 					}
 				} else {
-					log.Infof("File needs to be synchronized: %s", path)
-					_err := r.downloadFile(masterURL+"/download?path="+filepath.Join(workingDir, path), filepath.Join(r.cwd, path), masterFile)
+					log.Infof("file needs to be synchronized: %s", path)
+					_err := r.downloadFile(path, filepath.Join(r.cwd, path), masterFile)
 					if _err != nil {
-						log.Errorf("Error downloading file: %v", _err)
+						log.Errorf("error downloading file: %v", _err)
 						err = errors.Join(err, _err)
 					}
 				}
@@ -426,15 +444,14 @@ func (r *Runner) syncFiles() (err error) {
 	return err
 }

-func (r *Runner) downloadFile(url string, filePath string, fileInfo *entity.FsFileInfo) error {
-	// get file response
-	resp, err := http.Get(url)
+func (r *Runner) downloadFile(path string, filePath string, fileInfo *entity.FsFileInfo) error {
+	resp, err := r.createHttpRequest("GET", "/download?path="+path)
 	if err != nil {
-		log.Errorf("Error getting file response: %v", err)
+		log.Errorf("error getting file response: %v", err)
 		return err
 	}
 	if resp.StatusCode != http.StatusOK {
-		log.Errorf("Error downloading file: %s", resp.Status)
+		log.Errorf("error downloading file: %s", resp.Status)
 		return errors.New(resp.Status)
 	}
 	defer resp.Body.Close()
@@ -444,14 +461,14 @@ func (r *Runner) downloadFile(url string, filePath string, fileInfo *entity.FsFi
 	utils.Exists(dirPath)
 	err = os.MkdirAll(dirPath, os.ModePerm)
 	if err != nil {
-		log.Errorf("Error creating directory: %v", err)
+		log.Errorf("error creating directory: %v", err)
 		return err
 	}

 	// create local file
 	out, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode)
 	if err != nil {
-		log.Errorf("Error creating file: %v", err)
+		log.Errorf("error creating file: %v", err)
 		return err
 	}
 	defer out.Close()
@@ -459,12 +476,18 @@ func (r *Runner) downloadFile(url string, filePath string, fileInfo *entity.FsFi
 	// copy file content to local file
 	_, err = io.Copy(out, resp.Body)
 	if err != nil {
-		log.Errorf("Error copying file: %v", err)
+		log.Errorf("error copying file: %v", err)
 		return err
 	}
 	return nil
 }

+func (r *Runner) getHttpRequestHeaders() (headers map[string]string) {
+	headers = make(map[string]string)
+	headers["Authorization"] = utils.GetAuthKey()
+	return headers
+}
+
 // wait for process to finish and send task signal (constants.TaskSignal)
 // to task runner's channel (Runner.ch) according to exit code
 func (r *Runner) wait() {
@@ -541,7 +564,7 @@ func (r *Runner) initConnection() (err error) {
 func (r *Runner) writeLogLines(lines []string) {
 	linesBytes, err := json.Marshal(lines)
 	if err != nil {
-		log.Errorf("Error marshaling log lines: %v", err)
+		log.Errorf("error marshaling log lines: %v", err)
 		return
 	}
 	msg := &grpc.TaskServiceConnectRequest{
@@ -550,7 +573,7 @@ func (r *Runner) writeLogLines(lines []string) {
 		Data:   linesBytes,
 	}
 	if err := r.conn.Send(msg); err != nil {
-		log.Errorf("Error sending log lines: %v", err)
+		log.Errorf("error sending log lines: %v", err)
 		return
 	}
 }
@@ -598,7 +621,7 @@ func (r *Runner) sendNotification() {
 	}
 	_, err := r.c.TaskClient.SendNotification(context.Background(), req)
 	if err != nil {
-		log.Errorf("Error sending notification: %v", err)
+		log.Errorf("error sending notification: %v", err)
 		trace.PrintError(err)
 		return
 	}
@@ -659,7 +682,7 @@ func (r *Runner) _updateSpiderStat(status string) {
 }

 func (r *Runner) configureCwd() {
-	workspacePath := viper.GetString("workspace")
+	workspacePath := utils.GetWorkspace()
 	if r.s.GitId.IsZero() {
 		// not git
 		r.cwd = filepath.Join(workspacePath, r.s.Id.Hex())
@@ -698,7 +721,7 @@ func newTaskRunner(id primitive.ObjectID, svc *Service) (r2 *Runner, err error)
 	}

 	// task fs service
-	r.fsSvc = fs.NewFsService(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex()))
+	r.fsSvc = fs.NewFsService(filepath.Join(utils.GetWorkspace(), r.s.Id.Hex()))

 	// grpc client
 	r.c = client2.GetGrpcClient()
--- a/core/task/log/file_driver.go
+++ b/core/task/log/file_driver.go
@@ -128,7 +128,7 @@ func (d *FileLogDriver) Flush() (err error) {
 }

 func (d *FileLogDriver) getLogPath() (logPath string) {
-	return viper.GetString("log.path")
+	return utils.GetTaskLogPath()
 }

 func (d *FileLogDriver) getBasePath(id string) (filePath string) {
--- a/core/utils/config.go
+++ b/core/utils/config.go
@@ -0,0 +1,143 @@
+package utils
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+)
+
+const (
+	DefaultServerHost          = "0.0.0.0"
+	DefaultServerPort          = "8000"
+	DefaultTaskLogPath         = "/var/log/crawlab/tasks"
+	DefaultGrpcHost            = "localhost"
+	DefaultGrpcPort            = "9666"
+	DefaultAuthKey             = "Crawlab2024!"
+	DefaultApiEndpoint         = "http://localhost:8000"
+	DefaultApiAllowOrigin      = "*"
+	DefaultApiAllowCredentials = "true"
+	DefaultApiAllowMethods     = "DELETE, POST, OPTIONS, GET, PUT"
+	DefaultApiAllowHeaders     = "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With"
+)
+
+func IsDev() bool {
+	return viper.GetBool("dev")
+}
+
+func IsAuthDisabled() bool {
+	return viper.GetBool("auth.disabled")
+}
+
+func GetAllowOrigin() string {
+	if res := viper.GetString("api.allow.origin"); res != "" {
+		return res
+	}
+	return DefaultApiAllowOrigin
+}
+
+func GetAllowCredentials() string {
+	if res := viper.GetString("api.allow.credentials"); res != "" {
+		return res
+	}
+	return DefaultApiAllowCredentials
+}
+
+func GetAllowMethods() string {
+	if res := viper.GetString("api.allow.methods"); res != "" {
+		return res
+	}
+	return DefaultApiAllowMethods
+}
+
+func GetAllowHeaders() string {
+	if res := viper.GetString("api.allow.headers"); res != "" {
+		return res
+	}
+	return DefaultApiAllowHeaders
+}
+
+func GetGinMode() string {
+	if res := viper.GetString("gin.mode"); res != "" {
+		return res
+	}
+	if IsDev() {
+		return gin.DebugMode
+	} else {
+		return gin.ReleaseMode
+	}
+}
+
+func IsPro() bool {
+	return viper.GetString("edition") == "global.edition.pro"
+}
+
+func GetWorkspace() string {
+	return viper.GetString("workspace")
+}
+
+func GetTaskLogPath() string {
+	if res := viper.GetString("log.path"); res != "" {
+		return res
+	}
+	return DefaultTaskLogPath
+}
+
+func GetServerHost() string {
+	if res := viper.GetString("server.host"); res != "" {
+		return res
+	}
+	return DefaultServerHost
+}
+
+func GetServerPort() string {
+	if res := viper.GetString("server.port"); res != "" {
+		return res
+	}
+	return DefaultServerPort
+}
+
+func GetMasterHost() string {
+	return viper.GetString("master.host")
+}
+
+func GetGrpcAddress() string {
+	host := viper.GetString("grpc.host")
+	port := viper.GetString("grpc.port")
+	if host == "" {
+		masterHost := GetMasterHost()
+		if masterHost != "" {
+			host = masterHost
+		} else {
+			host = DefaultGrpcHost
+		}
+	}
+	if port == "" {
+		port = DefaultGrpcPort
+	}
+	return host + ":" + port
+}
+
+func GetGrpcServerAddress() string {
+	return viper.GetString("grpc.server.address")
+}
+
+func GetAuthKey() string {
+	if res := viper.GetString("auth.key"); res != "" {
+		return res
+	}
+	return DefaultAuthKey
+}
+
+func GetApiEndpoint() string {
+	if res := viper.GetString("api.endpoint"); res != "" {
+		return res
+	}
+	masterHost := GetMasterHost()
+	if masterHost != "" {
+		apiHttps := viper.GetBool("api.https")
+		if apiHttps {
+			return "https://" + masterHost + "/api"
+		}
+		return "http://" + masterHost + "/api"
+	}
+	return DefaultApiEndpoint
+}
--- a/core/utils/spider.go
+++ b/core/utils/spider.go
@@ -3,7 +3,6 @@ package utils
 import (
 	"errors"
 	"github.com/crawlab-team/crawlab/core/models/models"
-	"github.com/spf13/viper"
 	"path/filepath"
 )

@@ -23,7 +22,7 @@ func GetSpiderRootPath(s *models.Spider) (rootPath string, err error) {

 func GetSpiderFullRootPath(s *models.Spider) (rootPath string, err error) {
 	// workspace path
-	workspacePath := viper.GetString("workspace")
+	workspacePath := GetWorkspace()

 	// get spider root path
 	rootPath, err = GetSpiderRootPath(s)
--- a/core/utils/system.go
+++ b/core/utils/system.go
@@ -1,7 +0,0 @@
-package utils
-
-import "github.com/spf13/viper"
-
-func IsPro() bool {
-	return viper.GetString("edition") == "global.edition.pro"
-}
--- a/core/utils/task.go
+++ b/core/utils/task.go
@@ -1,6 +1,8 @@
 package utils

-import "github.com/crawlab-team/crawlab/core/constants"
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+)

 func IsCancellable(status string) bool {
 	switch status {
--- a/docker/base-image/Dockerfile
+++ b/docker/base-image/Dockerfile
@@ -44,9 +44,6 @@ ENV TZ Asia/Shanghai
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8

-# docker
-ENV CRAWLAB_DOCKER Y
-
 # goproxy
 ENV GOPROXY goproxy.io,direct