git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-31 18:10:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入可配置爬虫

Browse Source
This commit is contained in:
marvzhang
2019-11-24 17:57:12 +08:00
parent c03c296436
commit 38d103da39
20 changed files with 638 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
backend/entity/config_spider.go
View File

@@ -1,22 +1,23 @@
package entity
type Field struct {
Name string `yaml:"name" json:"name"`
Css string `yaml:"css" json:"css"`
Xpath string `yaml:"xpath" json:"xpath"`
Attr string `yaml:"attr" json:"attr"`
Stage string `yaml:"stage" json:"stage"`
Name string `yaml:"name" json:"name"`
Css string `yaml:"css" json:"css"`
Xpath string `yaml:"xpath" json:"xpath"`
Attr string `yaml:"attr" json:"attr"`
NextStage string `yaml:"next_stage" json:"next_stage"`
}
type Stage struct {
List bool `yaml:"list" json:"list"`
Css string `yaml:"css" json:"css"`
Xpath string `yaml:"xpath" json:"xpath"`
Fields []Field `yaml:"fields" json:"fields"`
IsList bool `yaml:"is_list" json:"is_list"`
ListCss string `yaml:"list_css" json:"list_css"`
PageCss string `yaml:"page_css" json:"page_css"`
Fields []Field `yaml:"fields" json:"fields"`
}
type ConfigSpiderData struct {
Version string `yaml:"version" json:"version"`
StartUrl string `yaml:"startUrl" json:"start_url"`
Engine string `yaml:"engine" json:"engine"`
StartUrl string `yaml:"start_url" json:"start_url"`
Stages map[string]Stage `yaml:"stages" json:"stages"`
}