From 3eee129a7c387a1109a19087f6a88d774ef11931 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 4 Sep 2019 20:13:12 +0800 Subject: [PATCH] added param to task --- backend/model/schedule.go | 1 + backend/model/task.go | 1 + backend/services/schedule.go | 1 + backend/services/task.go | 6 +- .../dgrijalva/jwt-go/request/doc.go | 7 -- .../dgrijalva/jwt-go/request/extractor.go | 81 ------------------- .../dgrijalva/jwt-go/request/oauth2.go | 28 ------- .../dgrijalva/jwt-go/request/request.go | 68 ---------------- backend/vendor/modules.txt | 23 +++++- 9 files changed, 29 insertions(+), 187 deletions(-) delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/request.go diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 9f77c452..1c8db0bd 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -18,6 +18,7 @@ type Schedule struct { NodeId bson.ObjectId `json:"node_id" bson:"node_id"` Cron string `json:"cron" bson:"cron"` EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Param string `json:"param" bson:"param"` // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` diff --git a/backend/model/task.go b/backend/model/task.go index 968055a6..4957b577 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -19,6 +19,7 @@ type Task struct { NodeId bson.ObjectId `json:"node_id" bson:"node_id"` LogPath string `json:"log_path" bson:"log_path"` Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` Error string `json:"error" bson:"error"` ResultCount int `json:"result_count" bson:"result_count"` WaitDuration float64 `json:"wait_duration" bson:"wait_duration"` diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 916e42d0..1c08e0fd 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -28,6 +28,7 @@ func AddTask(s model.Schedule) func() { SpiderId: s.SpiderId, NodeId: nodeId, Status: constants.StatusPending, + Param: s.Param, } // 将任务存入数据库 diff --git a/backend/services/task.go b/backend/services/task.go index 06e54572..dbaa2800 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -325,8 +325,10 @@ func ExecuteTask(id int) { // 执行命令 cmd := spider.Cmd - if t.Cmd != "" { - cmd = t.Cmd + + // 加入参数 + if t.Param != "" { + cmd += " " + t.Param } // 任务赋值 diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go deleted file mode 100644 index c01069c9..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Utility package for extracting JWT tokens from -// HTTP requests. -// -// The main function is ParseFromRequest and it's WithClaims variant. -// See examples for how to use the various Extractor implementations -// or roll your own. -package request diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go deleted file mode 100644 index 14414fe2..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go +++ /dev/null @@ -1,81 +0,0 @@ -package request - -import ( - "errors" - "net/http" -) - -// Errors -var ( - ErrNoTokenInRequest = errors.New("no token present in request") -) - -// Interface for extracting a token from an HTTP request. -// The ExtractToken method should return a token string or an error. -// If no token is present, you must return ErrNoTokenInRequest. -type Extractor interface { - ExtractToken(*http.Request) (string, error) -} - -// Extractor for finding a token in a header. Looks at each specified -// header in order until there's a match -type HeaderExtractor []string - -func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) { - // loop over header names and return the first one that contains data - for _, header := range e { - if ah := req.Header.Get(header); ah != "" { - return ah, nil - } - } - return "", ErrNoTokenInRequest -} - -// Extract token from request arguments. This includes a POSTed form or -// GET URL arguments. Argument names are tried in order until there's a match. -// This extractor calls `ParseMultipartForm` on the request -type ArgumentExtractor []string - -func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) { - // Make sure form is parsed - req.ParseMultipartForm(10e6) - - // loop over arg names and return the first one that contains data - for _, arg := range e { - if ah := req.Form.Get(arg); ah != "" { - return ah, nil - } - } - - return "", ErrNoTokenInRequest -} - -// Tries Extractors in order until one returns a token string or an error occurs -type MultiExtractor []Extractor - -func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) { - // loop over header names and return the first one that contains data - for _, extractor := range e { - if tok, err := extractor.ExtractToken(req); tok != "" { - return tok, nil - } else if err != ErrNoTokenInRequest { - return "", err - } - } - return "", ErrNoTokenInRequest -} - -// Wrap an Extractor in this to post-process the value before it's handed off. -// See AuthorizationHeaderExtractor for an example -type PostExtractionFilter struct { - Extractor - Filter func(string) (string, error) -} - -func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) { - if tok, err := e.Extractor.ExtractToken(req); tok != "" { - return e.Filter(tok) - } else { - return "", err - } -} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go deleted file mode 100644 index 5948694a..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go +++ /dev/null @@ -1,28 +0,0 @@ -package request - -import ( - "strings" -) - -// Strips 'Bearer ' prefix from bearer token string -func stripBearerPrefixFromTokenString(tok string) (string, error) { - // Should be a bearer token - if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " { - return tok[7:], nil - } - return tok, nil -} - -// Extract bearer token from Authorization header -// Uses PostExtractionFilter to strip "Bearer " prefix from header -var AuthorizationHeaderExtractor = &PostExtractionFilter{ - HeaderExtractor{"Authorization"}, - stripBearerPrefixFromTokenString, -} - -// Extractor for OAuth2 access tokens. Looks in 'Authorization' -// header then 'access_token' argument for a token. -var OAuth2Extractor = &MultiExtractor{ - AuthorizationHeaderExtractor, - ArgumentExtractor{"access_token"}, -} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go deleted file mode 100644 index 70525cfa..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go +++ /dev/null @@ -1,68 +0,0 @@ -package request - -import ( - "github.com/dgrijalva/jwt-go" - "net/http" -) - -// Extract and parse a JWT token from an HTTP request. -// This behaves the same as Parse, but accepts a request and an extractor -// instead of a token string. The Extractor interface allows you to define -// the logic for extracting a token. Several useful implementations are provided. -// -// You can provide options to modify parsing behavior -func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) { - // Create basic parser struct - p := &fromRequestParser{req, extractor, nil, nil} - - // Handle options - for _, option := range options { - option(p) - } - - // Set defaults - if p.claims == nil { - p.claims = jwt.MapClaims{} - } - if p.parser == nil { - p.parser = &jwt.Parser{} - } - - // perform extract - tokenString, err := p.extractor.ExtractToken(req) - if err != nil { - return nil, err - } - - // perform parse - return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc) -} - -// ParseFromRequest but with custom Claims type -// DEPRECATED: use ParseFromRequest and the WithClaims option -func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) { - return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims)) -} - -type fromRequestParser struct { - req *http.Request - extractor Extractor - claims jwt.Claims - parser *jwt.Parser -} - -type ParseFromRequestOption func(*fromRequestParser) - -// Parse with custom claims -func WithClaims(claims jwt.Claims) ParseFromRequestOption { - return func(p *fromRequestParser) { - p.claims = claims - } -} - -// Parse using a custom parser -func WithParser(parser *jwt.Parser) ParseFromRequestOption { - return func(p *fromRequestParser) { - p.parser = parser - } -} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 57c7d3f1..634c337b 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -2,7 +2,6 @@ github.com/apex/log # github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go -github.com/dgrijalva/jwt-go/request # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 @@ -18,11 +17,18 @@ github.com/globalsign/mgo/bson github.com/globalsign/mgo/internal/sasl github.com/globalsign/mgo/internal/scram github.com/globalsign/mgo/internal/json +# github.com/go-playground/locales v0.12.1 +github.com/go-playground/locales +github.com/go-playground/locales/currency +# github.com/go-playground/universal-translator v0.16.0 +github.com/go-playground/universal-translator # github.com/golang/protobuf v1.3.1 github.com/golang/protobuf/proto # github.com/gomodule/redigo v2.0.0+incompatible github.com/gomodule/redigo/redis github.com/gomodule/redigo/internal +# github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 +github.com/gopherjs/gopherjs/js # github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl github.com/hashicorp/hcl/hcl/printer @@ -36,6 +42,10 @@ github.com/hashicorp/hcl/json/scanner github.com/hashicorp/hcl/json/token # github.com/json-iterator/go v1.1.6 github.com/json-iterator/go +# github.com/jtolds/gls v4.20.0+incompatible +github.com/jtolds/gls +# github.com/leodido/go-urn v1.1.0 +github.com/leodido/go-urn # github.com/magiconair/properties v1.8.0 github.com/magiconair/properties # github.com/mattn/go-isatty v0.0.8 @@ -52,6 +62,15 @@ github.com/pelletier/go-toml github.com/pkg/errors # github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid +# github.com/smartystreets/assertions v1.0.0 +github.com/smartystreets/assertions +github.com/smartystreets/assertions/internal/go-diff/diffmatchpatch +github.com/smartystreets/assertions/internal/go-render/render +github.com/smartystreets/assertions/internal/oglematchers +# github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 +github.com/smartystreets/goconvey/convey +github.com/smartystreets/goconvey/convey/reporting +github.com/smartystreets/goconvey/convey/gotest # github.com/spf13/afero v1.1.2 github.com/spf13/afero github.com/spf13/afero/mem @@ -72,5 +91,7 @@ golang.org/x/text/transform golang.org/x/text/unicode/norm # gopkg.in/go-playground/validator.v8 v8.18.2 gopkg.in/go-playground/validator.v8 +# gopkg.in/go-playground/validator.v9 v9.29.1 +gopkg.in/go-playground/validator.v9 # gopkg.in/yaml.v2 v2.2.2 gopkg.in/yaml.v2