mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-28 17:50:56 +01:00
增加当前节点本地定时缓存,修改部分潜在BUG,启动时Mongo或者redis无法正常连接时,进入启动等待
This commit is contained in:
30
backend/vendor/github.com/imroc/req/README.md
generated
vendored
30
backend/vendor/github.com/imroc/req/README.md
generated
vendored
@@ -67,7 +67,8 @@ Examples
|
||||
[Cookie](#Cookie)
|
||||
[Set Timeout](#Set-Timeout)
|
||||
[Set Proxy](#Set-Proxy)
|
||||
[Customize Client](#Customize-Client)
|
||||
[Customize Client](#Customize-Client)
|
||||
[Set context.Context](#Context)
|
||||
|
||||
## <a name="Basic">Basic</a>
|
||||
``` go
|
||||
@@ -104,6 +105,25 @@ header.Set("Accept", "application/json")
|
||||
req.Get("https://www.baidu.com", header)
|
||||
```
|
||||
|
||||
You can also set header from struct, use `HeaderFromStruct` func to parse your struct
|
||||
``` go
|
||||
type HeaderStruct struct {
|
||||
UserAgent string `json:"User-Agent"`
|
||||
Authorization string `json:"Authorization"`
|
||||
}
|
||||
|
||||
func main(){
|
||||
h := HeaderStruct{
|
||||
"V1.0.0",
|
||||
"roc",
|
||||
}
|
||||
|
||||
authHeader := req.HeaderFromStruct(h)
|
||||
req.Get("https://www.baidu.com", authHeader, req.Header{"User-Agent": "V1.1"})
|
||||
}
|
||||
```
|
||||
> Note: Please add tag 'json' to your argument in struct to let you customize the key name of your header
|
||||
|
||||
## <a name="Set-Param">Set Param</a>
|
||||
Use `req.Param` (it is actually a `map[string]interface{}`)
|
||||
``` go
|
||||
@@ -163,7 +183,7 @@ Output in simple way (default format)
|
||||
``` go
|
||||
r, _ := req.Get(url, param)
|
||||
log.Printf("%v\n", r) // GET http://foo.bar/api?name=roc&cmd=add {"code":"0","msg":"success"}
|
||||
log.Prinln(r) // smae as above
|
||||
log.Prinln(r) // same as above
|
||||
```
|
||||
|
||||
### `%-v` or `%-s`
|
||||
@@ -281,6 +301,12 @@ Set a simple proxy (use fixed proxy url for every request)
|
||||
req.SetProxyUrl("http://my.proxy.com:23456")
|
||||
```
|
||||
|
||||
## <a name="Context">Set context.Context</a>
|
||||
You can pass context.Context in simple way:
|
||||
```go
|
||||
r, _ := req.Get(url, context.Background())
|
||||
```
|
||||
|
||||
## <a name="Customize-Client">Customize Client</a>
|
||||
Use `SetClient` to change the default underlying `*http.Client`
|
||||
``` go
|
||||
|
||||
49
backend/vendor/github.com/imroc/req/req.go
generated
vendored
49
backend/vendor/github.com/imroc/req/req.go
generated
vendored
@@ -34,20 +34,6 @@ const (
|
||||
LstdFlags = LreqHead | LreqBody | LrespHead | LrespBody
|
||||
)
|
||||
|
||||
// Header represents http request header
|
||||
type Header map[string]string
|
||||
|
||||
func (h Header) Clone() Header {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
hh := Header{}
|
||||
for k, v := range h {
|
||||
hh[k] = v
|
||||
}
|
||||
return hh
|
||||
}
|
||||
|
||||
// Param represents http request param
|
||||
type Param map[string]interface{}
|
||||
|
||||
@@ -121,15 +107,17 @@ func BodyXML(v interface{}) *bodyXml {
|
||||
|
||||
// Req is a convenient client for initiating requests
|
||||
type Req struct {
|
||||
client *http.Client
|
||||
jsonEncOpts *jsonEncOpts
|
||||
xmlEncOpts *xmlEncOpts
|
||||
flag int
|
||||
client *http.Client
|
||||
jsonEncOpts *jsonEncOpts
|
||||
xmlEncOpts *xmlEncOpts
|
||||
flag int
|
||||
progressInterval time.Duration
|
||||
}
|
||||
|
||||
// New create a new *Req
|
||||
func New() *Req {
|
||||
return &Req{flag: LstdFlags}
|
||||
// default progress reporting interval is 200 milliseconds
|
||||
return &Req{flag: LstdFlags, progressInterval: 200 * time.Millisecond}
|
||||
}
|
||||
|
||||
type param struct {
|
||||
@@ -277,9 +265,10 @@ func (r *Req) Do(method, rawurl string, vs ...interface{}) (resp *Resp, err erro
|
||||
up = UploadProgress(progress)
|
||||
}
|
||||
multipartHelper := &multipartHelper{
|
||||
form: formParam.Values,
|
||||
uploads: uploads,
|
||||
uploadProgress: up,
|
||||
form: formParam.Values,
|
||||
uploads: uploads,
|
||||
uploadProgress: up,
|
||||
progressInterval: resp.r.progressInterval,
|
||||
}
|
||||
multipartHelper.Upload(req)
|
||||
resp.multipartHelper = multipartHelper
|
||||
@@ -484,10 +473,11 @@ func (b *bodyWrapper) Read(p []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
type multipartHelper struct {
|
||||
form url.Values
|
||||
uploads []FileUpload
|
||||
dump []byte
|
||||
uploadProgress UploadProgress
|
||||
form url.Values
|
||||
uploads []FileUpload
|
||||
dump []byte
|
||||
uploadProgress UploadProgress
|
||||
progressInterval time.Duration
|
||||
}
|
||||
|
||||
func (m *multipartHelper) Upload(req *http.Request) {
|
||||
@@ -514,6 +504,11 @@ func (m *multipartHelper) Upload(req *http.Request) {
|
||||
var current int64
|
||||
buf := make([]byte, 1024)
|
||||
var lastTime time.Time
|
||||
|
||||
defer func() {
|
||||
m.uploadProgress(current, total)
|
||||
}()
|
||||
|
||||
upload = func(w io.Writer, r io.Reader) error {
|
||||
for {
|
||||
n, err := r.Read(buf)
|
||||
@@ -523,7 +518,7 @@ func (m *multipartHelper) Upload(req *http.Request) {
|
||||
return _err
|
||||
}
|
||||
current += int64(n)
|
||||
if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
|
||||
if now := time.Now(); now.Sub(lastTime) > m.progressInterval {
|
||||
lastTime = now
|
||||
m.uploadProgress(current, total)
|
||||
}
|
||||
|
||||
7
backend/vendor/github.com/imroc/req/resp.go
generated
vendored
7
backend/vendor/github.com/imroc/req/resp.go
generated
vendored
@@ -124,6 +124,11 @@ func (r *Resp) download(file *os.File) error {
|
||||
total := r.resp.ContentLength
|
||||
var current int64
|
||||
var lastTime time.Time
|
||||
|
||||
defer func() {
|
||||
r.downloadProgress(current, total)
|
||||
}()
|
||||
|
||||
for {
|
||||
l, err := b.Read(p)
|
||||
if l > 0 {
|
||||
@@ -132,7 +137,7 @@ func (r *Resp) download(file *os.File) error {
|
||||
return _err
|
||||
}
|
||||
current += int64(l)
|
||||
if now := time.Now(); now.Sub(lastTime) > 200*time.Millisecond {
|
||||
if now := time.Now(); now.Sub(lastTime) > r.r.progressInterval {
|
||||
lastTime = now
|
||||
r.downloadProgress(current, total)
|
||||
}
|
||||
|
||||
12
backend/vendor/github.com/imroc/req/setting.go
generated
vendored
12
backend/vendor/github.com/imroc/req/setting.go
generated
vendored
@@ -234,3 +234,15 @@ func (r *Req) SetXMLIndent(prefix, indent string) {
|
||||
func SetXMLIndent(prefix, indent string) {
|
||||
std.SetXMLIndent(prefix, indent)
|
||||
}
|
||||
|
||||
// SetProgressInterval sets the progress reporting interval of both
|
||||
// UploadProgress and DownloadProgress handler
|
||||
func (r *Req) SetProgressInterval(interval time.Duration) {
|
||||
r.progressInterval = interval
|
||||
}
|
||||
|
||||
// SetProgressInterval sets the progress reporting interval of both
|
||||
// UploadProgress and DownloadProgress handler for the default client
|
||||
func SetProgressInterval(interval time.Duration) {
|
||||
std.SetProgressInterval(interval)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user