git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-25 17:42:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: code cleanup

Browse Source
This commit is contained in:
Marvin Zhang
2024-10-20 17:17:08 +08:00
parent 9b5ed188a6
commit 52e16badb7
28 changed files with 80 additions and 5849 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
core/entity/color.go
View File

@@ -1,18 +0,0 @@
package entity
type Color struct {
Name string `json:"name"`
Hex string `json:"hex"`
}
func (c *Color) GetHex() string {
return c.Hex
}
func (c *Color) GetName() string {
return c.Name
}
func (c *Color) Value() interface{} {
return c
}

17
core/entity/common.go
View File

@@ -1,17 +0,0 @@
package entity
import "strconv"
type Page struct {
Skip int
Limit int
PageNum int
PageSize int
}
func (p *Page) GetPage(pageNum string, pageSize string) {
p.PageNum, _ = strconv.Atoi(pageNum)
p.PageSize, _ = strconv.Atoi(pageSize)
p.Skip = p.PageSize * (p.PageNum - 1)
p.Limit = p.PageSize
}

40
core/entity/config_spider.go
View File

@@ -1,40 +0,0 @@
package entity
type ConfigSpiderData struct {
// 通用
Name string `yaml:"name" json:"name"`
DisplayName string `yaml:"display_name" json:"display_name"`
Col string `yaml:"col" json:"col"`
Remark string `yaml:"remark" json:"remark"`
Type string `yaml:"type" bson:"type"`
// 可配置爬虫
Engine string `yaml:"engine" json:"engine"`
StartUrl string `yaml:"start_url" json:"start_url"`
StartStage string `yaml:"start_stage" json:"start_stage"`
Stages []Stage `yaml:"stages" json:"stages"`
Settings map[string]string `yaml:"settings" json:"settings"`
// 自定义爬虫
Cmd string `yaml:"cmd" json:"cmd"`
}
type Stage struct {
Name string `yaml:"name" json:"name"`
IsList bool `yaml:"is_list" json:"is_list"`
ListCss string `yaml:"list_css" json:"list_css"`
ListXpath string `yaml:"list_xpath" json:"list_xpath"`
PageCss string `yaml:"page_css" json:"page_css"`
PageXpath string `yaml:"page_xpath" json:"page_xpath"`
PageAttr string `yaml:"page_attr" json:"page_attr"`
Fields []Field `yaml:"fields" json:"fields"`
}
type Field struct {
Name string `yaml:"name" json:"name"`
Css string `yaml:"css" json:"css"`
Xpath string `yaml:"xpath" json:"xpath"`
Attr string `yaml:"attr" json:"attr"`
NextStage string `yaml:"next_stage" json:"next_stage"`
Remark string `yaml:"remark" json:"remark"`
}

8
core/entity/doc.go
View File

@@ -1,8 +0,0 @@
package entity
type DocItem struct {
Title string `json:"title"`
Url string `json:"url"`
Path string `json:"path"`
Children []DocItem `json:"children"`
}

54
core/entity/es.go
View File

@@ -1,54 +0,0 @@
package entity
/* ElasticsearchResponseData JSON format
{
"took" : 6,
"timed_out" : false,
"_shards" : {
"total" : 1,
"successful" : 1,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 60,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "test_table",
"_id" : "c39ad9a2-9a37-49fb-b7ea-f1b55913e0af",
"_score" : 1.0,
"_source" : {
"_tid" : "62524ac7f5f99e7ef594de64",
"author" : "James Baldwin",
"tags" : [
"love"
],
"text" : "“Love does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.”"
}
}
]
}
}
*/
type ElasticsearchResponseData struct {
Took int64 `json:"took"`
Timeout bool `json:"timeout"`
Hits struct {
Total struct {
Value int64 `json:"value"`
Relation string `json:"relation"`
} `json:"total"`
MaxScore float64 `json:"max_score"`
Hits []struct {
Index string `json:"_index"`
Id string `json:"_id"`
Score float64 `json:"_score"`
Source interface{} `json:"_source"`
} `json:"hits"`
} `json:"hits"`
}

14
core/entity/event.go
View File

@@ -1,14 +0,0 @@
package entity
type EventData struct {
Event string
Data interface{}
}
func (d *EventData) GetEvent() string {
return d.Event
}
func (d *EventData) GetData() interface{} {
return d.Data
}

12
core/entity/git.go
View File

@@ -1,12 +0,0 @@
package entity
type GitPayload struct {
Paths []string `json:"paths"`
CommitMessage string `json:"commit_message"`
Branch string `json:"branch"`
Tag string `json:"tag"`
}
type GitConfig struct {
Url string `json:"url" bson:"url"`
}

29
core/entity/grpc_base_service_message.go
View File

@@ -1,29 +0,0 @@
package entity
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/trace"
)
type GrpcBaseServiceMessage struct {
ModelId interfaces.ModelId `json:"id"`
Data []byte `json:"d"`
}
func (msg *GrpcBaseServiceMessage) GetModelId() interfaces.ModelId {
return msg.ModelId
}
func (msg *GrpcBaseServiceMessage) GetData() []byte {
return msg.Data
}
func (msg *GrpcBaseServiceMessage) ToBytes() (data []byte) {
data, err := json.Marshal(*msg)
if err != nil {
_ = trace.TraceError(err)
return data
}
return data
}

23
core/entity/grpc_base_service_params.go
View File

@@ -1,23 +0,0 @@
package entity
import (
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/db/mongo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type GrpcBaseServiceParams struct {
Query bson.M `json:"q"`
Id primitive.ObjectID `json:"id"`
Update bson.M `json:"u"`
Doc interfaces.Model `json:"d"`
Fields []string `json:"f"`
FindOptions *mongo.FindOptions `json:"o"`
Docs []interface{} `json:"dl"`
User interfaces.User `json:"U"`
}
func (params *GrpcBaseServiceParams) Value() interface{} {
return params
}

34
core/entity/grpc_delegate_message.go
View File

@@ -1,34 +0,0 @@
package entity
import (
"encoding/json"
"github.com/crawlab-team/crawlab/core/interfaces"
"github.com/crawlab-team/crawlab/trace"
)
type GrpcDelegateMessage struct {
ModelId interfaces.ModelId `json:"id"`
Method interfaces.ModelDelegateMethod `json:"m"`
Data []byte `json:"d"`
}
func (msg *GrpcDelegateMessage) GetModelId() interfaces.ModelId {
return msg.ModelId
}
func (msg *GrpcDelegateMessage) GetMethod() interfaces.ModelDelegateMethod {
return msg.Method
}
func (msg *GrpcDelegateMessage) GetData() []byte {
return msg.Data
}
func (msg *GrpcDelegateMessage) ToBytes() (data []byte) {
data, err := json.Marshal(*msg)
if err != nil {
_ = trace.TraceError(err)
return data
}
return data
}

8
core/entity/grpc_event_service_message.go
View File

@@ -1,8 +0,0 @@
package entity
type GrpcEventServiceMessage struct {
Type string `json:"type"`
Events []string `json:"events"`
Key string `json:"key"`
Data []byte `json:"data"`
}

20
core/entity/http.go
View File

@@ -17,26 +17,6 @@ type ListResponse struct {
Error string `json:"error"`
}
type ListRequestData struct {
PageNum int `form:"page_num" json:"page_num"`
PageSize int `form:"page_size" json:"page_size"`
SortKey string `form:"sort_key" json:"sort_key"`
Status string `form:"status" json:"status"`
Keyword string `form:"keyword" json:"keyword"`
}
type BatchRequestPayload struct {
Ids []primitive.ObjectID `form:"ids" json:"ids"`
}
type BatchRequestPayloadWithStringData struct {
Ids []primitive.ObjectID `form:"ids" json:"ids"`
Data string `form:"data" json:"data"`
Fields []string `form:"fields" json:"fields"`
}
type FileRequestPayload struct {
Path string `json:"path" form:"path"`
NewPath string `json:"new_path" form:"new_path"`
Data string `json:"data" form:"data"`
}

11
core/entity/model_delegate.go
View File

@@ -1,11 +0,0 @@
package entity
import "github.com/crawlab-team/crawlab/core/interfaces"
type ModelDelegate struct {
Id interfaces.ModelId `json:"id"`
ColName string `json:"col_name"`
Doc interfaces.Model `json:"doc"`
Artifact interfaces.ModelArtifact `json:"a"`
User interfaces.User `json:"u"`
}

8
core/entity/model_info.go
View File

@@ -1,8 +0,0 @@
package entity
import "github.com/crawlab-team/crawlab/core/interfaces"
type ModelInfo struct {
Id interfaces.ModelId
ColName string
}

11
core/entity/rpc.go
View File

@@ -1,11 +0,0 @@
package entity
type RpcMessage struct {
Id string `json:"id"` // 消息ID
Method string `json:"method"` // 消息方法
NodeId string `json:"node_id"` // 节点ID
Params map[string]string `json:"params"` // 参数
Timeout int `json:"timeout"` // 超时
Result string `json:"result"` // 结果
Error string `json:"error"` // 错误
}

17
core/entity/spider.go
View File

@@ -1,17 +0,0 @@
package entity
type SpiderType struct {
Type string `json:"type" bson:"_id"`
Count int `json:"count" bson:"count"`
}
type ScrapySettingParam struct {
Key string `json:"key"`
Value interface{} `json:"value"`
Type string `json:"type"`
}
type ScrapyItem struct {
Name string `json:"name"`
Fields []string `json:"fields"`
}

5
core/entity/stats.go
View File

@@ -5,8 +5,3 @@ type StatsDailyItem struct {
Tasks int64 `json:"tasks" bson:"tasks"`
Results int64 `json:"results" bson:"results"`
}
type StatsTasksByStatusItem struct {
Status string `json:"status" bson:"_id"`
Tasks int64 `json:"tasks" bson:"tasks"`
}

19
core/entity/task.go
View File

@@ -1,28 +1,9 @@
package entity
import (
"encoding/json"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type TaskMessage struct {
Id primitive.ObjectID `json:"id"`
Key string `json:"key"`
Cmd string `json:"cmd"`
Param string `json:"param"`
}
func (m *TaskMessage) ToString() (string, error) {
data, err := json.Marshal(&m)
if err != nil {
return "", err
}
return string(data), err
}
type TaskRunOptions struct {
}
type StreamMessageTaskData struct {
TaskId primitive.ObjectID `json:"task_id"`
Records []Result `json:"data"`

11
core/entity/translation.go
View File

@@ -1,11 +0,0 @@
package entity
type Translation struct {
Lang string `json:"lang"`
Key string `json:"key"`
Value string `json:"value"`
}
func (t Translation) GetLang() (l string) {
return t.Lang
}

58
core/entity/ttl_map.go
View File

@@ -1,58 +0,0 @@
package entity
import (
"sync"
"time"
)
type TTLMap struct {
TTL time.Duration
data sync.Map
}
type expireEntry struct {
ExpiresAt time.Time
Value interface{}
}
func (t *TTLMap) Store(key string, val interface{}) {
t.data.Store(key, expireEntry{
ExpiresAt: time.Now().Add(t.TTL),
Value: val,
})
}
func (t *TTLMap) Load(key string) (val interface{}) {
entry, ok := t.data.Load(key)
if !ok {
return nil
}
expireEntry := entry.(expireEntry)
if expireEntry.ExpiresAt.After(time.Now()) {
return nil
}
return expireEntry.Value
}
func NewTTLMap(ttl time.Duration) (m *TTLMap) {
m = &TTLMap{
TTL: ttl,
}
go func() {
for now := range time.Tick(time.Second) {
m.data.Range(func(k, v interface{}) bool {
expiresAt := v.(expireEntry).ExpiresAt
if expiresAt.Before(now) {
m.data.Delete(k)
}
return true
})
}
}()
return
}

23
core/entity/version.go
View File

@@ -1,23 +0,0 @@
package entity
type Release struct {
Name string `json:"name"`
Draft bool `json:"draft"`
PreRelease bool `json:"pre_release"`
PublishedAt string `json:"published_at"`
Body string `json:"body"`
}
type ReleaseSlices []Release
func (r ReleaseSlices) Len() int {
return len(r)
}
func (r ReleaseSlices) Less(i, j int) bool {
return r[i].PublishedAt < r[j].PublishedAt
}
func (r ReleaseSlices) Swap(i, j int) {
r[i], r[j] = r[j], r[i]
}