diff --git a/core/config/config.go b/core/config/config.go index 1eace82a..4587ceae 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -1,11 +1,11 @@ package config import ( + "strings" + "github.com/apex/log" - "github.com/crawlab-team/crawlab/trace" "github.com/fsnotify/fsnotify" "github.com/spf13/viper" - "strings" ) func init() { @@ -65,7 +65,6 @@ func (c *Config) Init() (err error) { // read in config if err := viper.ReadInConfig(); err != nil { log.Errorf("Error reading config file, %s", err) - trace.PrintError(err) return err } diff --git a/core/config/config_test.go b/core/config/config_test.go index d902687b..f8a3c433 100644 --- a/core/config/config_test.go +++ b/core/config/config_test.go @@ -1,11 +1,65 @@ package config import ( - "github.com/stretchr/testify/require" + "os" + "path/filepath" "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestInitConfig(t *testing.T) { - err := InitConfig() - require.Nil(t, err) +func init() { + // Set test environment variables + viper.Set("CRAWLAB_TEST_STRING", "test_string_value") + viper.Set("CRAWLAB_TEST_INT", 42) + viper.Set("CRAWLAB_TEST_BOOL", true) + viper.Set("CRAWLAB_TEST_NESTED_KEY", "nested_value") +} + +func TestInitConfig(t *testing.T) { + // Create a temporary directory for the test + tempDir, err := os.MkdirTemp("", "crawlab-config-test") + require.NoError(t, err, "Failed to create temp directory") + defer os.RemoveAll(tempDir) + + // Create a temporary config file + configContent := []byte(` +log: + level: info +test: + string: default_string_value + int: 0 + bool: false + nested: + key: default_nested_value +`) + configPath := filepath.Join(tempDir, "config.yaml") + err = os.WriteFile(configPath, configContent, 0644) + require.NoError(t, err, "Failed to write config file") + + // Set up the test environment + oldConfigPath := viper.ConfigFileUsed() + defer viper.SetConfigFile(oldConfigPath) + viper.SetConfigFile(configPath) + + // Create a new Config instance + c := Config{Name: configPath} + + // Initialize the config + err = c.Init() + require.NoError(t, err, "Failed to initialize config") + + // Test config values + assert.Equal(t, "default_string_value", viper.GetString("test.string"), "Unexpected value for test.string") + assert.Equal(t, 0, viper.GetInt("test.int"), "Unexpected value for test.int") + assert.False(t, viper.GetBool("test.bool"), "Unexpected value for test.bool") + assert.Equal(t, "default_nested_value", viper.GetString("test.nested.key"), "Unexpected value for test.nested.key") + assert.Empty(t, viper.GetString("non.existent.key"), "Non-existent key should return empty string") + + // Test environment variable override + os.Setenv("CRAWLAB_TEST_STRING", "env_string_value") + defer os.Unsetenv("CRAWLAB_TEST_STRING") + assert.Equal(t, "env_string_value", viper.GetString("test.string"), "Environment variable should override config value") } diff --git a/core/constants/action.go b/core/constants/action.go deleted file mode 100644 index 389a11bc..00000000 --- a/core/constants/action.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -const ( - ActionTypeVisit = "visit" - ActionTypeInstallDep = "install_dep" - ActionTypeInstallLang = "install_lang" - ActionTypeViewDisclaimer = "view_disclaimer" -) diff --git a/core/constants/anchor.go b/core/constants/anchor.go deleted file mode 100644 index f462135f..00000000 --- a/core/constants/anchor.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -const ( - AnchorStartStage = "START_STAGE" - AnchorStartUrl = "START_URL" - AnchorItems = "ITEMS" - AnchorParsers = "PARSERS" -) diff --git a/core/constants/auth.go b/core/constants/auth.go deleted file mode 100644 index 136391a0..00000000 --- a/core/constants/auth.go +++ /dev/null @@ -1,7 +0,0 @@ -package constants - -const ( - OwnerTypeAll = "all" - OwnerTypeMe = "me" - OwnerTypePublic = "public" -) diff --git a/core/constants/cache.go b/core/constants/cache.go deleted file mode 100644 index c61186f0..00000000 --- a/core/constants/cache.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -const ( - CacheColName = "cache" - CacheColKey = "k" - CacheColValue = "v" - CacheColTime = "t" -) diff --git a/core/constants/channels.go b/core/constants/channels.go deleted file mode 100644 index c38a5ac9..00000000 --- a/core/constants/channels.go +++ /dev/null @@ -1,9 +0,0 @@ -package constants - -const ( - ChannelAllNode = "nodes:public" - - ChannelWorkerNode = "nodes:" - - ChannelMasterNode = "nodes:master" -) diff --git a/core/constants/config_spider.go b/core/constants/config_spider.go deleted file mode 100644 index c29624dc..00000000 --- a/core/constants/config_spider.go +++ /dev/null @@ -1,6 +0,0 @@ -package constants - -const ( - EngineScrapy = "scrapy" - EngineColly = "colly" -) diff --git a/core/constants/data_collection.go b/core/constants/data_collection.go deleted file mode 100644 index b8505e93..00000000 --- a/core/constants/data_collection.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ( - DataCollectionKey = "_col" -) diff --git a/core/constants/data_field.go b/core/constants/data_field.go deleted file mode 100644 index 604ad1d5..00000000 --- a/core/constants/data_field.go +++ /dev/null @@ -1,12 +0,0 @@ -package constants - -const ( - DataFieldTypeGeneral = "general" - DataFieldTypeNumeric = "numeric" - DataFieldTypeDate = "date" - DataFieldTypeCurrency = "currency" - DataFieldTypeUrl = "url" - DataFieldTypeImage = "image" - DataFieldTypeAudio = "audio" - DataFieldTypeVideo = "video" -) diff --git a/core/constants/database.go b/core/constants/database.go deleted file mode 100644 index 31abb22e..00000000 --- a/core/constants/database.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ( - ColJob = "jobs" -) diff --git a/core/constants/delegate.go b/core/constants/delegate.go deleted file mode 100644 index 1255fc8b..00000000 --- a/core/constants/delegate.go +++ /dev/null @@ -1 +0,0 @@ -package constants diff --git a/core/constants/ds.go b/core/constants/ds.go deleted file mode 100644 index 268c5761..00000000 --- a/core/constants/ds.go +++ /dev/null @@ -1,31 +0,0 @@ -package constants - -const ( - DataSourceTypeMongo = "mongo" - DataSourceTypeMysql = "mysql" - DataSourceTypePostgresql = "postgresql" - DataSourceTypeMssql = "mssql" - DataSourceTypeSqlite = "sqlite" - DataSourceTypeCockroachdb = "cockroachdb" - DataSourceTypeElasticSearch = "elasticsearch" - DataSourceTypeKafka = "kafka" -) - -const ( - DefaultHost = "localhost" -) - -const ( - DefaultMongoPort = 27017 - DefaultMysqlPort = 3306 - DefaultPostgresqlPort = 5432 - DefaultMssqlPort = 1433 - DefaultCockroachdbPort = 26257 - DefaultElasticsearchPort = 9200 - DefaultKafkaPort = 9092 -) - -const ( - DataSourceStatusOnline = "on" - DataSourceStatusOffline = "off" -) diff --git a/core/constants/errors.go b/core/constants/errors.go index 11f46251..94305090 100644 --- a/core/constants/errors.go +++ b/core/constants/errors.go @@ -5,26 +5,10 @@ import ( ) var ( - //ErrorMongoError = e.NewSystemOPError(1001, "system error:[mongo]%s", http.StatusInternalServerError) - //ErrorUserNotFound = e.NewBusinessError(10001, "user not found.", http.StatusUnauthorized) - //ErrorUsernameOrPasswordInvalid = e.NewBusinessError(11001, "username or password invalid", http.StatusUnauthorized) - ErrAlreadyExists = errors.New("already exists") - ErrNotExists = errors.New("not exists") - ErrForbidden = errors.New("forbidden") - ErrInvalidOperation = errors.New("invalid operation") - ErrInvalidOptions = errors.New("invalid options") - ErrNoTasksAvailable = errors.New("no tasks available") - ErrInvalidType = errors.New("invalid type") - ErrInvalidSignal = errors.New("invalid signal") - ErrEmptyValue = errors.New("empty value") - ErrTaskError = errors.New("task error") - ErrTaskLost = errors.New("task lost") - ErrTaskCancelled = errors.New("task cancelled") - ErrUnableToCancel = errors.New("unable to cancel") - ErrUnableToDispose = errors.New("unable to dispose") - ErrAlreadyDisposed = errors.New("already disposed") - ErrStopped = errors.New("stopped") - ErrMissingCol = errors.New("missing col") - ErrInvalidValue = errors.New("invalid value") - ErrInvalidCronSpec = errors.New("invalid cron spec") + ErrNotExists = errors.New("not exists") + ErrInvalidOptions = errors.New("invalid options") + ErrInvalidSignal = errors.New("invalid signal") + ErrTaskError = errors.New("task error") + ErrTaskLost = errors.New("task lost") + ErrTaskCancelled = errors.New("task cancelled") ) diff --git a/core/constants/event.go b/core/constants/event.go deleted file mode 100644 index d474c165..00000000 --- a/core/constants/event.go +++ /dev/null @@ -1,6 +0,0 @@ -package constants - -const ( - GrpcEventServiceTypeRegister = "register" - GrpcEventServiceTypeSend = "send" -) diff --git a/core/constants/file.go b/core/constants/file.go deleted file mode 100644 index 499ea993..00000000 --- a/core/constants/file.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const EmptyFileData = " " - -const FsKeepFileName = ".gitkeep" diff --git a/core/constants/filer.go b/core/constants/filer.go deleted file mode 100644 index 0fdcc6f3..00000000 --- a/core/constants/filer.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ( - DefaultFilerAuthKey = "Crawlab2021!" -) diff --git a/core/constants/filter.go b/core/constants/filter.go index a8403568..9ed1d8db 100644 --- a/core/constants/filter.go +++ b/core/constants/filter.go @@ -6,12 +6,6 @@ const ( FilterQueryFieldFilter = "filter" ) -const ( - FilterObjectTypeString = "string" - FilterObjectTypeNumber = "number" - FilterObjectTypeBoolean = "boolean" -) - const ( FilterOpNotSet = "ns" FilterOpContains = "c" diff --git a/core/constants/git.go b/core/constants/git.go index 7fa5f7c2..1255fc8b 100644 --- a/core/constants/git.go +++ b/core/constants/git.go @@ -1,16 +1 @@ package constants - -const ( - GitAuthTypeHttp = "http" - GitAuthTypeSsh = "ssh" -) - -const ( - GitRemoteNameUpstream = "upstream" - GitRemoteNameOrigin = "origin" -) - -const ( - GitBranchMaster = "master" - GitBranchMain = "main" -) diff --git a/core/constants/grpc.go b/core/constants/grpc.go index 83915209..4577f7c2 100644 --- a/core/constants/grpc.go +++ b/core/constants/grpc.go @@ -11,7 +11,3 @@ const ( const ( GrpcHeaderAuthorization = "authorization" ) - -const ( - GrpcSubscribeTypeNode = "node" -) diff --git a/core/constants/http.go b/core/constants/http.go index 43d7a84b..ab3bf8ee 100644 --- a/core/constants/http.go +++ b/core/constants/http.go @@ -5,7 +5,3 @@ const ( HttpResponseMessageSuccess = "success" HttpResponseMessageError = "error" ) - -const ( - HttpContentTypeApplicationJson = "application/json" -) diff --git a/core/constants/log.go b/core/constants/log.go deleted file mode 100644 index 5f0b4a66..00000000 --- a/core/constants/log.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ( - ErrorRegexPattern = "(?:[ :,.]|^)((?:error|exception|traceback)s?)(?:[ :,.]|$)" -) diff --git a/core/constants/message.go b/core/constants/message.go deleted file mode 100644 index 72e5fab2..00000000 --- a/core/constants/message.go +++ /dev/null @@ -1,9 +0,0 @@ -package constants - -const ( - MsgTypeGetLog = "get-log" - MsgTypeGetSystemInfo = "get-sys-info" - MsgTypeCancelTask = "cancel-task" - MsgTypeRemoveLog = "remove-log" - MsgTypeRemoveSpider = "remove-spider" -) diff --git a/core/constants/register.go b/core/constants/register.go deleted file mode 100644 index b48f8a11..00000000 --- a/core/constants/register.go +++ /dev/null @@ -1,8 +0,0 @@ -package constants - -const ( - RegisterTypeMac = "mac" - RegisterTypeIp = "ip" - RegisterTypeHostname = "hostname" - RegisterTypeCustomName = "customName" -) diff --git a/core/constants/results.go b/core/constants/results.go deleted file mode 100644 index 7053111f..00000000 --- a/core/constants/results.go +++ /dev/null @@ -1,10 +0,0 @@ -package constants - -const ( - HashKey = "_h" -) - -const ( - DedupTypeIgnore = "ignore" - DedupTypeOverwrite = "overwrite" -) diff --git a/core/constants/rpc.go b/core/constants/rpc.go deleted file mode 100644 index 0fd7ad9f..00000000 --- a/core/constants/rpc.go +++ /dev/null @@ -1,12 +0,0 @@ -package constants - -const ( - RpcInstallLang = "install_lang" - RpcInstallDep = "install_dep" - RpcUninstallDep = "uninstall_dep" - RpcGetInstalledDepList = "get_installed_dep_list" - RpcGetLang = "get_lang" - RpcCancelTask = "cancel_task" - RpcGetSystemInfoService = "get_system_info" - RpcRemoveSpider = "remove_spider" -) diff --git a/core/constants/schedule.go b/core/constants/schedule.go deleted file mode 100644 index 520626a9..00000000 --- a/core/constants/schedule.go +++ /dev/null @@ -1,10 +0,0 @@ -package constants - -const ( - ScheduleStatusStop = "stopped" - ScheduleStatusRunning = "running" - ScheduleStatusError = "error" - - ScheduleStatusErrorNotFoundNode = "Not Found Node" - ScheduleStatusErrorNotFoundSpider = "Not Found Spider" -) diff --git a/core/constants/scrapy.go b/core/constants/scrapy.go deleted file mode 100644 index bc82508f..00000000 --- a/core/constants/scrapy.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ScrapyProtectedStageNames = "" - -const ScrapyProtectedFieldNames = "_id,task_id,ts" diff --git a/core/constants/signal.go b/core/constants/signal.go deleted file mode 100644 index dd4cd0b4..00000000 --- a/core/constants/signal.go +++ /dev/null @@ -1,5 +0,0 @@ -package constants - -const ( - SignalQuit = iota -) diff --git a/core/constants/system.go b/core/constants/system.go deleted file mode 100644 index 14c45698..00000000 --- a/core/constants/system.go +++ /dev/null @@ -1,25 +0,0 @@ -package constants - -const ( - Windows = "windows" - Linux = "linux" - Darwin = "darwin" -) - -const ( - Python = "python" - Nodejs = "node" - Java = "java" -) - -const ( - InstallStatusNotInstalled = "not-installed" - InstallStatusInstalling = "installing" - InstallStatusInstallingOther = "installing-other" - InstallStatusInstalled = "installed" -) - -const ( - LangTypeLang = "lang" - LangTypeWebDriver = "webdriver" -) diff --git a/core/constants/variable.go b/core/constants/variable.go deleted file mode 100644 index 713fbe2d..00000000 --- a/core/constants/variable.go +++ /dev/null @@ -1,9 +0,0 @@ -package constants - -const ( - String = "string" - Number = "number" - Boolean = "boolean" - Array = "array" - Object = "object" -) diff --git a/core/container/container.go b/core/container/container.go deleted file mode 100644 index 3dfffe52..00000000 --- a/core/container/container.go +++ /dev/null @@ -1,11 +0,0 @@ -package container - -import ( - "go.uber.org/dig" -) - -var c = dig.New() - -func GetContainer() *dig.Container { - return c -} diff --git a/core/utils/file.go b/core/utils/file.go index 572730ec..9e09d4d5 100644 --- a/core/utils/file.go +++ b/core/utils/file.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "io" "io/fs" @@ -211,13 +210,6 @@ func _Compress(file *os.File, prefix string, zw *zip.Writer) error { return nil } -func TrimFileData(data []byte) (res []byte) { - if string(data) == constants.EmptyFileData { - return res - } - return data -} - func ZipDirectory(dir, zipfile string) error { zipFile, err := os.Create(zipfile) if err != nil {