git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-31 18:10:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: enhance Spider model and string utility functions

Browse Source 
- Updated the Spider model to introduce a new SpiderTemplateParams struct for improved template handling.
- Refactored string utility functions in utils/string.go to include a new replaceChars function, streamlining character replacement across multiple functions.
- Enhanced ToSnakeCase and ToKebabCase functions to utilize the new replaceChars function for better maintainability and readability.
- Added splitStringWithQuotes function to facilitate string manipulation with quotes, improving overall utility in string processing.
This commit is contained in:
Marvin Zhang
2025-01-07 13:21:16 +08:00
parent c3c629a7d7
commit c937e0f45f
2 changed files with 71 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
core/models/models/spider.go
View File

@@ -7,25 +7,21 @@ import (
type Spider struct {
any `collection:"spiders"`
BaseModel[Spider] `bson:",inline"`
Name string `json:"name" bson:"name"` // spider name
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future
ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name
DbName string `json:"db_name,omitempty" bson:"db_name"` // database name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *Git `json:"git,omitempty" bson:"-"`
Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template
TemplateParams *struct {
SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty"`
StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty"`
Domains string `json:"domains,omitempty" bson:"domains,omitempty"`
} `json:"template_params,omitempty" bson:"template_params,omitempty"`
Name string `json:"name" bson:"name"` // spider name
ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future
ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name
DbName string `json:"db_name,omitempty" bson:"db_name"` // database name
DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id
DataSource *Database `json:"data_source,omitempty" bson:"-"` // data source
Description string `json:"description" bson:"description"` // description
ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id
Mode string `json:"mode" bson:"mode"` // default Task.Mode
NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds
GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id
GitRootPath string `json:"git_root_path" bson:"git_root_path"`
Git *Git `json:"git,omitempty" bson:"-"`
Template string `json:"template,omitempty" bson:"template,omitempty"` // spider template
TemplateParams *SpiderTemplateParams `json:"template_params,omitempty" bson:"template_params,omitempty"`
// stats
Stat *SpiderStat `json:"stat,omitempty" bson:"-"`
@@ -36,3 +32,10 @@ type Spider struct {
Priority int `json:"priority" bson:"priority"`
AutoInstall bool `json:"auto_install" bson:"auto_install"`
}
type SpiderTemplateParams struct {
ProjectName string `json:"project_name,omitempty" bson:"project_name,omitempty"`
SpiderName string `json:"spider_name,omitempty" bson:"spider_name,omitempty"`
StartUrls string `json:"start_urls,omitempty" bson:"start_urls,omitempty"`
AllowedDomains string `json:"allowed_domains,omitempty" bson:"allowed_domains,omitempty"`
}