git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

added param to task

Browse Source
This commit is contained in:
Marvin Zhang
2019-09-04 20:13:12 +08:00
parent 992f3f4d86
commit 3eee129a7c
9 changed files with 29 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/model/schedule.go
View File

@@ -18,6 +18,7 @@ type Schedule struct {
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
Cron string `json:"cron" bson:"cron"`
EntryId cron.EntryID `json:"entry_id" bson:"entry_id"`
Param string `json:"param" bson:"param"`
// 前端展示
SpiderName string `json:"spider_name" bson:"spider_name"`

1
backend/model/task.go
View File

@@ -19,6 +19,7 @@ type Task struct {
NodeId bson.ObjectId `json:"node_id" bson:"node_id"`
LogPath string `json:"log_path" bson:"log_path"`
Cmd string `json:"cmd" bson:"cmd"`
Param string `json:"param" bson:"param"`
Error string `json:"error" bson:"error"`
ResultCount int `json:"result_count" bson:"result_count"`
WaitDuration float64 `json:"wait_duration" bson:"wait_duration"`

1
backend/services/schedule.go
View File

@@ -28,6 +28,7 @@ func AddTask(s model.Schedule) func() {
SpiderId: s.SpiderId,
NodeId: nodeId,
Status: constants.StatusPending,
Param: s.Param,
}
// 将任务存入数据库

6
backend/services/task.go
View File

@@ -325,8 +325,10 @@ func ExecuteTask(id int) {
// 执行命令
cmd := spider.Cmd
if t.Cmd != "" {
cmd = t.Cmd
// 加入参数
if t.Param != "" {
cmd += " " + t.Param
}
// 任务赋值

7
backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go generated vendored
View File

@@ -1,7 +0,0 @@
// Utility package for extracting JWT tokens from
// HTTP requests.
//
// The main function is ParseFromRequest and it's WithClaims variant.
// See examples for how to use the various Extractor implementations
// or roll your own.
package request

81
backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go generated vendored
View File

@@ -1,81 +0,0 @@
package request
import (
"errors"
"net/http"
)
// Errors
var (
ErrNoTokenInRequest = errors.New("no token present in request")
)
// Interface for extracting a token from an HTTP request.
// The ExtractToken method should return a token string or an error.
// If no token is present, you must return ErrNoTokenInRequest.
type Extractor interface {
ExtractToken(*http.Request) (string, error)
}
// Extractor for finding a token in a header. Looks at each specified
// header in order until there's a match
type HeaderExtractor []string
func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) {
// loop over header names and return the first one that contains data
for _, header := range e {
if ah := req.Header.Get(header); ah != "" {
return ah, nil
}
}
return "", ErrNoTokenInRequest
}
// Extract token from request arguments. This includes a POSTed form or
// GET URL arguments. Argument names are tried in order until there's a match.
// This extractor calls `ParseMultipartForm` on the request
type ArgumentExtractor []string
func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) {
// Make sure form is parsed
req.ParseMultipartForm(10e6)
// loop over arg names and return the first one that contains data
for _, arg := range e {
if ah := req.Form.Get(arg); ah != "" {
return ah, nil
}
}
return "", ErrNoTokenInRequest
}
// Tries Extractors in order until one returns a token string or an error occurs
type MultiExtractor []Extractor
func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) {
// loop over header names and return the first one that contains data
for _, extractor := range e {
if tok, err := extractor.ExtractToken(req); tok != "" {
return tok, nil
} else if err != ErrNoTokenInRequest {
return "", err
}
}
return "", ErrNoTokenInRequest
}
// Wrap an Extractor in this to post-process the value before it's handed off.
// See AuthorizationHeaderExtractor for an example
type PostExtractionFilter struct {
Extractor
Filter func(string) (string, error)
}
func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) {
if tok, err := e.Extractor.ExtractToken(req); tok != "" {
return e.Filter(tok)
} else {
return "", err
}
}

28
backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go generated vendored
View File

@@ -1,28 +0,0 @@
package request
import (
"strings"
)
// Strips 'Bearer ' prefix from bearer token string
func stripBearerPrefixFromTokenString(tok string) (string, error) {
// Should be a bearer token
if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " {
return tok[7:], nil
}
return tok, nil
}
// Extract bearer token from Authorization header
// Uses PostExtractionFilter to strip "Bearer " prefix from header
var AuthorizationHeaderExtractor = &PostExtractionFilter{
HeaderExtractor{"Authorization"},
stripBearerPrefixFromTokenString,
}
// Extractor for OAuth2 access tokens. Looks in 'Authorization'
// header then 'access_token' argument for a token.
var OAuth2Extractor = &MultiExtractor{
AuthorizationHeaderExtractor,
ArgumentExtractor{"access_token"},
}

68
backend/vendor/github.com/dgrijalva/jwt-go/request/request.go generated vendored
View File

@@ -1,68 +0,0 @@
package request
import (
"github.com/dgrijalva/jwt-go"
"net/http"
)
// Extract and parse a JWT token from an HTTP request.
// This behaves the same as Parse, but accepts a request and an extractor
// instead of a token string. The Extractor interface allows you to define
// the logic for extracting a token. Several useful implementations are provided.
//
// You can provide options to modify parsing behavior
func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) {
// Create basic parser struct
p := &fromRequestParser{req, extractor, nil, nil}
// Handle options
for _, option := range options {
option(p)
}
// Set defaults
if p.claims == nil {
p.claims = jwt.MapClaims{}
}
if p.parser == nil {
p.parser = &jwt.Parser{}
}
// perform extract
tokenString, err := p.extractor.ExtractToken(req)
if err != nil {
return nil, err
}
// perform parse
return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc)
}
// ParseFromRequest but with custom Claims type
// DEPRECATED: use ParseFromRequest and the WithClaims option
func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) {
return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims))
}
type fromRequestParser struct {
req *http.Request
extractor Extractor
claims jwt.Claims
parser *jwt.Parser
}
type ParseFromRequestOption func(*fromRequestParser)
// Parse with custom claims
func WithClaims(claims jwt.Claims) ParseFromRequestOption {
return func(p *fromRequestParser) {
p.claims = claims
}
}
// Parse using a custom parser
func WithParser(parser *jwt.Parser) ParseFromRequestOption {
return func(p *fromRequestParser) {
p.parser = parser
}
}

23
backend/vendor/modules.txt vendored
View File

@@ -2,7 +2,6 @@
github.com/apex/log
# github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgrijalva/jwt-go
github.com/dgrijalva/jwt-go/request
# github.com/fsnotify/fsnotify v1.4.7
github.com/fsnotify/fsnotify
# github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3
@@ -18,11 +17,18 @@ github.com/globalsign/mgo/bson
github.com/globalsign/mgo/internal/sasl
github.com/globalsign/mgo/internal/scram
github.com/globalsign/mgo/internal/json
# github.com/go-playground/locales v0.12.1
github.com/go-playground/locales
github.com/go-playground/locales/currency
# github.com/go-playground/universal-translator v0.16.0
github.com/go-playground/universal-translator
# github.com/golang/protobuf v1.3.1
github.com/golang/protobuf/proto
# github.com/gomodule/redigo v2.0.0+incompatible
github.com/gomodule/redigo/redis
github.com/gomodule/redigo/internal
# github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1
github.com/gopherjs/gopherjs/js
# github.com/hashicorp/hcl v1.0.0
github.com/hashicorp/hcl
github.com/hashicorp/hcl/hcl/printer
@@ -36,6 +42,10 @@ github.com/hashicorp/hcl/json/scanner
github.com/hashicorp/hcl/json/token
# github.com/json-iterator/go v1.1.6
github.com/json-iterator/go
# github.com/jtolds/gls v4.20.0+incompatible
github.com/jtolds/gls
# github.com/leodido/go-urn v1.1.0
github.com/leodido/go-urn
# github.com/magiconair/properties v1.8.0
github.com/magiconair/properties
# github.com/mattn/go-isatty v0.0.8
@@ -52,6 +62,15 @@ github.com/pelletier/go-toml
github.com/pkg/errors
# github.com/satori/go.uuid v1.2.0
github.com/satori/go.uuid
# github.com/smartystreets/assertions v1.0.0
github.com/smartystreets/assertions
github.com/smartystreets/assertions/internal/go-diff/diffmatchpatch
github.com/smartystreets/assertions/internal/go-render/render
github.com/smartystreets/assertions/internal/oglematchers
# github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
github.com/smartystreets/goconvey/convey
github.com/smartystreets/goconvey/convey/reporting
github.com/smartystreets/goconvey/convey/gotest
# github.com/spf13/afero v1.1.2
github.com/spf13/afero
github.com/spf13/afero/mem
@@ -72,5 +91,7 @@ golang.org/x/text/transform
golang.org/x/text/unicode/norm
# gopkg.in/go-playground/validator.v8 v8.18.2
gopkg.in/go-playground/validator.v8
# gopkg.in/go-playground/validator.v9 v9.29.1
gopkg.in/go-playground/validator.v9
# gopkg.in/yaml.v2 v2.2.2
gopkg.in/yaml.v2