git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: code cleanup

Browse Source
This commit is contained in:
Marvin Zhang
2024-10-20 16:55:57 +08:00
parent 1b852fb96a
commit 4ac92d7eaf
33 changed files with 66 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/config/config.go
View File

@@ -1,11 +1,11 @@
package config
import (
"strings"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/trace"
"github.com/fsnotify/fsnotify"
"github.com/spf13/viper"
"strings"
)
func init() {
@@ -65,7 +65,6 @@ func (c *Config) Init() (err error) {
// read in config
if err := viper.ReadInConfig(); err != nil {
log.Errorf("Error reading config file, %s", err)
trace.PrintError(err)
return err
}

62
core/config/config_test.go
View File

@@ -1,11 +1,65 @@
package config
import (
"github.com/stretchr/testify/require"
"os"
"path/filepath"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInitConfig(t *testing.T) {
err := InitConfig()
require.Nil(t, err)
func init() {
// Set test environment variables
viper.Set("CRAWLAB_TEST_STRING", "test_string_value")
viper.Set("CRAWLAB_TEST_INT", 42)
viper.Set("CRAWLAB_TEST_BOOL", true)
viper.Set("CRAWLAB_TEST_NESTED_KEY", "nested_value")
}
func TestInitConfig(t *testing.T) {
// Create a temporary directory for the test
tempDir, err := os.MkdirTemp("", "crawlab-config-test")
require.NoError(t, err, "Failed to create temp directory")
defer os.RemoveAll(tempDir)
// Create a temporary config file
configContent := []byte(`
log:
level: info
test:
string: default_string_value
int: 0
bool: false
nested:
key: default_nested_value
`)
configPath := filepath.Join(tempDir, "config.yaml")
err = os.WriteFile(configPath, configContent, 0644)
require.NoError(t, err, "Failed to write config file")
// Set up the test environment
oldConfigPath := viper.ConfigFileUsed()
defer viper.SetConfigFile(oldConfigPath)
viper.SetConfigFile(configPath)
// Create a new Config instance
c := Config{Name: configPath}
// Initialize the config
err = c.Init()
require.NoError(t, err, "Failed to initialize config")
// Test config values
assert.Equal(t, "default_string_value", viper.GetString("test.string"), "Unexpected value for test.string")
assert.Equal(t, 0, viper.GetInt("test.int"), "Unexpected value for test.int")
assert.False(t, viper.GetBool("test.bool"), "Unexpected value for test.bool")
assert.Equal(t, "default_nested_value", viper.GetString("test.nested.key"), "Unexpected value for test.nested.key")
assert.Empty(t, viper.GetString("non.existent.key"), "Non-existent key should return empty string")
// Test environment variable override
os.Setenv("CRAWLAB_TEST_STRING", "env_string_value")
defer os.Unsetenv("CRAWLAB_TEST_STRING")
assert.Equal(t, "env_string_value", viper.GetString("test.string"), "Environment variable should override config value")
}

8
core/constants/action.go
View File

@@ -1,8 +0,0 @@
package constants
const (
ActionTypeVisit = "visit"
ActionTypeInstallDep = "install_dep"
ActionTypeInstallLang = "install_lang"
ActionTypeViewDisclaimer = "view_disclaimer"
)

8
core/constants/anchor.go
View File

@@ -1,8 +0,0 @@
package constants
const (
AnchorStartStage = "START_STAGE"
AnchorStartUrl = "START_URL"
AnchorItems = "ITEMS"
AnchorParsers = "PARSERS"
)

7
core/constants/auth.go
View File

@@ -1,7 +0,0 @@
package constants
const (
OwnerTypeAll = "all"
OwnerTypeMe = "me"
OwnerTypePublic = "public"
)

8
core/constants/cache.go
View File

@@ -1,8 +0,0 @@
package constants
const (
CacheColName = "cache"
CacheColKey = "k"
CacheColValue = "v"
CacheColTime = "t"
)

9
core/constants/channels.go
View File

@@ -1,9 +0,0 @@
package constants
const (
ChannelAllNode = "nodes:public"
ChannelWorkerNode = "nodes:"
ChannelMasterNode = "nodes:master"
)

6
core/constants/config_spider.go
View File

@@ -1,6 +0,0 @@
package constants
const (
EngineScrapy = "scrapy"
EngineColly = "colly"
)

5
core/constants/data_collection.go
View File

@@ -1,5 +0,0 @@
package constants
const (
DataCollectionKey = "_col"
)

12
core/constants/data_field.go
View File

@@ -1,12 +0,0 @@
package constants
const (
DataFieldTypeGeneral = "general"
DataFieldTypeNumeric = "numeric"
DataFieldTypeDate = "date"
DataFieldTypeCurrency = "currency"
DataFieldTypeUrl = "url"
DataFieldTypeImage = "image"
DataFieldTypeAudio = "audio"
DataFieldTypeVideo = "video"
)

5
core/constants/database.go
View File

@@ -1,5 +0,0 @@
package constants
const (
ColJob = "jobs"
)

1
core/constants/delegate.go
View File

@@ -1 +0,0 @@
package constants

31
core/constants/ds.go
View File

@@ -1,31 +0,0 @@
package constants
const (
DataSourceTypeMongo = "mongo"
DataSourceTypeMysql = "mysql"
DataSourceTypePostgresql = "postgresql"
DataSourceTypeMssql = "mssql"
DataSourceTypeSqlite = "sqlite"
DataSourceTypeCockroachdb = "cockroachdb"
DataSourceTypeElasticSearch = "elasticsearch"
DataSourceTypeKafka = "kafka"
)
const (
DefaultHost = "localhost"
)
const (
DefaultMongoPort = 27017
DefaultMysqlPort = 3306
DefaultPostgresqlPort = 5432
DefaultMssqlPort = 1433
DefaultCockroachdbPort = 26257
DefaultElasticsearchPort = 9200
DefaultKafkaPort = 9092
)
const (
DataSourceStatusOnline = "on"
DataSourceStatusOffline = "off"
)

28
core/constants/errors.go
View File

@@ -5,26 +5,10 @@ import (
)
var (
//ErrorMongoError = e.NewSystemOPError(1001, "system error:[mongo]%s", http.StatusInternalServerError)
//ErrorUserNotFound = e.NewBusinessError(10001, "user not found.", http.StatusUnauthorized)
//ErrorUsernameOrPasswordInvalid = e.NewBusinessError(11001, "username or password invalid", http.StatusUnauthorized)
ErrAlreadyExists = errors.New("already exists")
ErrNotExists = errors.New("not exists")
ErrForbidden = errors.New("forbidden")
ErrInvalidOperation = errors.New("invalid operation")
ErrInvalidOptions = errors.New("invalid options")
ErrNoTasksAvailable = errors.New("no tasks available")
ErrInvalidType = errors.New("invalid type")
ErrInvalidSignal = errors.New("invalid signal")
ErrEmptyValue = errors.New("empty value")
ErrTaskError = errors.New("task error")
ErrTaskLost = errors.New("task lost")
ErrTaskCancelled = errors.New("task cancelled")
ErrUnableToCancel = errors.New("unable to cancel")
ErrUnableToDispose = errors.New("unable to dispose")
ErrAlreadyDisposed = errors.New("already disposed")
ErrStopped = errors.New("stopped")
ErrMissingCol = errors.New("missing col")
ErrInvalidValue = errors.New("invalid value")
ErrInvalidCronSpec = errors.New("invalid cron spec")
ErrNotExists = errors.New("not exists")
ErrInvalidOptions = errors.New("invalid options")
ErrInvalidSignal = errors.New("invalid signal")
ErrTaskError = errors.New("task error")
ErrTaskLost = errors.New("task lost")
ErrTaskCancelled = errors.New("task cancelled")
)

6
core/constants/event.go
View File

@@ -1,6 +0,0 @@
package constants
const (
GrpcEventServiceTypeRegister = "register"
GrpcEventServiceTypeSend = "send"
)

5
core/constants/file.go
View File

@@ -1,5 +0,0 @@
package constants
const EmptyFileData = " "
const FsKeepFileName = ".gitkeep"

5
core/constants/filer.go
View File

@@ -1,5 +0,0 @@
package constants
const (
DefaultFilerAuthKey = "Crawlab2021!"
)

6
core/constants/filter.go
View File

@@ -6,12 +6,6 @@ const (
FilterQueryFieldFilter = "filter"
)
const (
FilterObjectTypeString = "string"
FilterObjectTypeNumber = "number"
FilterObjectTypeBoolean = "boolean"
)
const (
FilterOpNotSet = "ns"
FilterOpContains = "c"

15
core/constants/git.go
View File

@@ -1,16 +1 @@
package constants
const (
GitAuthTypeHttp = "http"
GitAuthTypeSsh = "ssh"
)
const (
GitRemoteNameUpstream = "upstream"
GitRemoteNameOrigin = "origin"
)
const (
GitBranchMaster = "master"
GitBranchMain = "main"
)

4
core/constants/grpc.go
View File

@@ -11,7 +11,3 @@ const (
const (
GrpcHeaderAuthorization = "authorization"
)
const (
GrpcSubscribeTypeNode = "node"
)

4
core/constants/http.go
View File

@@ -5,7 +5,3 @@ const (
HttpResponseMessageSuccess = "success"
HttpResponseMessageError = "error"
)
const (
HttpContentTypeApplicationJson = "application/json"
)

5
core/constants/log.go
View File

@@ -1,5 +0,0 @@
package constants
const (
ErrorRegexPattern = "(?:[ :,.]|^)((?:error|exception|traceback)s?)(?:[ :,.]|$)"
)

9
core/constants/message.go
View File

@@ -1,9 +0,0 @@
package constants
const (
MsgTypeGetLog = "get-log"
MsgTypeGetSystemInfo = "get-sys-info"
MsgTypeCancelTask = "cancel-task"
MsgTypeRemoveLog = "remove-log"
MsgTypeRemoveSpider = "remove-spider"
)

8
core/constants/register.go
View File

@@ -1,8 +0,0 @@
package constants
const (
RegisterTypeMac = "mac"
RegisterTypeIp = "ip"
RegisterTypeHostname = "hostname"
RegisterTypeCustomName = "customName"
)

10
core/constants/results.go
View File

@@ -1,10 +0,0 @@
package constants
const (
HashKey = "_h"
)
const (
DedupTypeIgnore = "ignore"
DedupTypeOverwrite = "overwrite"
)

12
core/constants/rpc.go
View File

@@ -1,12 +0,0 @@
package constants
const (
RpcInstallLang = "install_lang"
RpcInstallDep = "install_dep"
RpcUninstallDep = "uninstall_dep"
RpcGetInstalledDepList = "get_installed_dep_list"
RpcGetLang = "get_lang"
RpcCancelTask = "cancel_task"
RpcGetSystemInfoService = "get_system_info"
RpcRemoveSpider = "remove_spider"
)

10
core/constants/schedule.go
View File

@@ -1,10 +0,0 @@
package constants
const (
ScheduleStatusStop = "stopped"
ScheduleStatusRunning = "running"
ScheduleStatusError = "error"
ScheduleStatusErrorNotFoundNode = "Not Found Node"
ScheduleStatusErrorNotFoundSpider = "Not Found Spider"
)

5
core/constants/scrapy.go
View File

@@ -1,5 +0,0 @@
package constants
const ScrapyProtectedStageNames = ""
const ScrapyProtectedFieldNames = "_id,task_id,ts"

5
core/constants/signal.go
View File

@@ -1,5 +0,0 @@
package constants
const (
SignalQuit = iota
)

25
core/constants/system.go
View File

@@ -1,25 +0,0 @@
package constants
const (
Windows = "windows"
Linux = "linux"
Darwin = "darwin"
)
const (
Python = "python"
Nodejs = "node"
Java = "java"
)
const (
InstallStatusNotInstalled = "not-installed"
InstallStatusInstalling = "installing"
InstallStatusInstallingOther = "installing-other"
InstallStatusInstalled = "installed"
)
const (
LangTypeLang = "lang"
LangTypeWebDriver = "webdriver"
)

9
core/constants/variable.go
View File

@@ -1,9 +0,0 @@
package constants
const (
String = "string"
Number = "number"
Boolean = "boolean"
Array = "array"
Object = "object"
)

11
core/container/container.go
View File

@@ -1,11 +0,0 @@
package container
import (
"go.uber.org/dig"
)
var c = dig.New()
func GetContainer() *dig.Container {
return c
}

8
core/utils/file.go
View File

@@ -6,7 +6,6 @@ import (
"encoding/hex"
"fmt"
"github.com/apex/log"
"github.com/crawlab-team/crawlab/core/constants"
"github.com/crawlab-team/crawlab/core/entity"
"io"
"io/fs"
@@ -211,13 +210,6 @@ func _Compress(file *os.File, prefix string, zw *zip.Writer) error {
return nil
}
func TrimFileData(data []byte) (res []byte) {
if string(data) == constants.EmptyFileData {
return res
}
return data
}
func ZipDirectory(dir, zipfile string) error {
zipFile, err := os.Create(zipfile)
if err != nil {