mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
Merge branch 'develop' of https://github.com/crawlab-team/crawlab into develop
This commit is contained in:
marvzhang
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,4 +1,5 @@
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
|
||||
@@ -23,8 +23,9 @@ server:
|
||||
master: "Y"
|
||||
secret: "crawlab"
|
||||
register:
|
||||
# mac地址/ip地址/hostname, 如果是ip，则需要手动指定IP
|
||||
# type 填 mac/ip/customName, 如果是ip，则需要手动指定IP, 如果是 customName, 需填写你的 customNodeName
|
||||
type: "mac"
|
||||
customNodeName: "" # 自定义节点名称, default node1,只有在type = customName 时生效
|
||||
ip: ""
|
||||
lang: # 安装语言环境, Y 为安装，N 为不安装
|
||||
python: "Y"
|
||||
@@ -40,12 +41,16 @@ other:
|
||||
tmppath: "/tmp"
|
||||
version: 0.4.10
|
||||
setting:
|
||||
crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient
|
||||
crawlabLogIndex: "crawlab-log"
|
||||
allowRegister: "N"
|
||||
enableTutorial: "N"
|
||||
runOnMaster: "Y"
|
||||
demoSpiders: "N"
|
||||
checkScrapy: "Y"
|
||||
autoInstall: "Y"
|
||||
esClient: "" # Your ES client, for example, http://192.168.1.1:9200 or http://your-domain.com, if not use es, set empty
|
||||
spiderLogIndex: "spider-log" # Index pattern for kibana, need to config on kibana
|
||||
notification:
|
||||
mail:
|
||||
server: ''
|
||||
|
||||
@@ -53,3 +53,5 @@ func InitConfig(cfg string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -4,4 +4,5 @@ const (
|
||||
RegisterTypeMac = "mac"
|
||||
RegisterTypeIp = "ip"
|
||||
RegisterTypeHostname = "hostname"
|
||||
RegisterTypeCustomName = "customName"
|
||||
)
|
||||
|
||||
44
backend/database/es_base.go
Normal file
44
backend/database/es_base.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/apex/log"
|
||||
"github.com/olivere/elastic/v7"
|
||||
"github.com/satori/go.uuid"
|
||||
"github.com/spf13/viper"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var doOnce sync.Once
|
||||
var ctx context.Context
|
||||
var ESClient *elastic.Client
|
||||
|
||||
func InitEsClient() {
|
||||
esClientStr := viper.GetString("setting.esClient")
|
||||
ctx = context.Background()
|
||||
ESClient, _ = elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
|
||||
}
|
||||
|
||||
// WriteMsg will write the msg and level into es
|
||||
func WriteMsgToES(when time.Time, msg chan string, index string) {
|
||||
doOnce.Do(InitEsClient)
|
||||
vals := make(map[string]interface{})
|
||||
vals["@timestamp"] = when.Format(time.RFC3339)
|
||||
for {
|
||||
select {
|
||||
case vals["@msg"] = <-msg:
|
||||
uid := uuid.NewV4().String()
|
||||
_, err := ESClient.Index().Index(index).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
log.Error("send msg log to es error")
|
||||
return
|
||||
}
|
||||
case <-time.After(6 * time.Second):
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
4808
backend/docs/docs.go
Normal file
4808
backend/docs/docs.go
Normal file
File diff suppressed because it is too large
Load Diff
4740
backend/docs/swagger.json
Normal file
4740
backend/docs/swagger.json
Normal file
File diff suppressed because it is too large
Load Diff
3177
backend/docs/swagger.yaml
Normal file
3177
backend/docs/swagger.yaml
Normal file
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,11 @@ module crawlab
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/Masterminds/semver v1.4.2 // indirect
|
||||
github.com/Masterminds/sprig v2.16.0+incompatible // indirect
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
|
||||
github.com/aokoli/goutils v1.0.1 // indirect
|
||||
github.com/apex/log v1.1.1
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/fsnotify/fsnotify v1.4.7
|
||||
@@ -12,15 +16,23 @@ require (
|
||||
github.com/go-playground/locales v0.12.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.16.0 // indirect
|
||||
github.com/gomodule/redigo v2.0.0+incompatible
|
||||
github.com/huandu/xstrings v1.2.0 // indirect
|
||||
github.com/imdario/mergo v0.3.6 // indirect
|
||||
github.com/imroc/req v0.2.4
|
||||
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect
|
||||
github.com/leodido/go-urn v1.1.0 // indirect
|
||||
github.com/matcornic/hermes v1.2.0
|
||||
github.com/matcornic/hermes/v2 v2.0.2 // indirect
|
||||
github.com/pkg/errors v0.8.1
|
||||
github.com/royeo/dingrobot v1.0.0 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.3 // indirect
|
||||
github.com/olekukonko/tablewriter v0.0.1 // indirect
|
||||
github.com/olivere/elastic/v7 v7.0.14
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
|
||||
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
|
||||
github.com/spf13/viper v1.4.0
|
||||
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
|
||||
github.com/swaggo/gin-swagger v1.2.0
|
||||
github.com/swaggo/swag v1.5.1
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/go-playground/validator.v9 v9.29.1
|
||||
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737
|
||||
|
||||
@@ -6,11 +6,18 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
|
||||
github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY=
|
||||
github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
|
||||
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
|
||||
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
|
||||
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
|
||||
github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
|
||||
@@ -19,8 +26,10 @@ github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA
|
||||
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
|
||||
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
@@ -41,29 +50,50 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
|
||||
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
|
||||
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc=
|
||||
github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w=
|
||||
github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
|
||||
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g=
|
||||
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
|
||||
github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ=
|
||||
github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
|
||||
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
|
||||
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
|
||||
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
|
||||
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
|
||||
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
|
||||
github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
|
||||
github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
|
||||
github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
|
||||
github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
|
||||
github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880=
|
||||
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
|
||||
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
|
||||
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
|
||||
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
|
||||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
|
||||
@@ -72,8 +102,10 @@ github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNu
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
@@ -97,8 +129,10 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
|
||||
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
@@ -120,11 +154,14 @@ github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
|
||||
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
|
||||
github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
|
||||
github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU=
|
||||
github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc=
|
||||
github.com/matcornic/hermes/v2 v2.0.2/go.mod h1:iVsJWSIS4NtMNtgan22sy6lt7pImok7bATGPWCoaKNY=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
|
||||
@@ -145,14 +182,19 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
|
||||
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/olivere/elastic/v7 v7.0.14 h1:89dYPg6kD3WJx42ZtO4U6WDIzRy69FvQqz/yRiwekuM=
|
||||
github.com/olivere/elastic/v7 v7.0.14/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
@@ -166,9 +208,6 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/royeo/dingrobot v1.0.0 h1:K4GrF+fOecNX0yi+oBKpfh7z0XP/8TzaIIHu1B2kKUQ=
|
||||
github.com/royeo/dingrobot v1.0.0/go.mod h1:RqDM8E/hySCVwI2aUFRJAUGDcHHRnIhzNmbNG3bamQs=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
@@ -205,6 +244,13 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/swaggo/gin-swagger v1.2.0 h1:YskZXEiv51fjOMTsXrOetAjrMDfFaXD79PEoQBOe2W0=
|
||||
github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI=
|
||||
github.com/swaggo/swag v1.5.1 h1:2Agm8I4K5qb00620mHq0VJ05/KT4FtmALPIcQR9lEZM=
|
||||
github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y=
|
||||
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
@@ -212,52 +258,69 @@ github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKw
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/ugorji/go v1.1.5-pre h1:jyJKFOSEbdOc2HODrf2qcCkYOdq7zzXqA9bhW5oV4fM=
|
||||
github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0=
|
||||
github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/ugorji/go/codec v1.1.5-pre h1:5YV9PsFAN+ndcCtTM7s60no7nY7eTG3LPtxhSwuxzCs=
|
||||
github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
|
||||
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
|
||||
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
@@ -268,12 +331,21 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
|
||||
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
|
||||
@@ -295,6 +367,7 @@ gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8
|
||||
gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
|
||||
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
|
||||
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
|
||||
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
|
||||
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
|
||||
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"crawlab/config"
|
||||
"crawlab/database"
|
||||
_ "crawlab/docs"
|
||||
"crawlab/lib/validate_bridge"
|
||||
"crawlab/middlewares"
|
||||
"crawlab/model"
|
||||
@@ -14,7 +15,10 @@ import (
|
||||
"github.com/apex/log"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gin-gonic/gin/binding"
|
||||
"github.com/olivere/elastic/v7"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/swaggo/gin-swagger"
|
||||
"github.com/swaggo/gin-swagger/swaggerFiles"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -24,9 +28,17 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
var swagHandler gin.HandlerFunc
|
||||
|
||||
func init() {
|
||||
swagHandler = ginSwagger.WrapHandler(swaggerFiles.Handler)
|
||||
}
|
||||
func main() {
|
||||
binding.Validator = new(validate_bridge.DefaultValidator)
|
||||
app := gin.Default()
|
||||
if swagHandler != nil {
|
||||
app.GET("/swagger/*any", swagHandler)
|
||||
}
|
||||
|
||||
// 初始化配置
|
||||
if err := config.InitConfig(""); err != nil {
|
||||
@@ -133,6 +145,15 @@ func main() {
|
||||
// 以下为主节点服务
|
||||
if model.IsMaster() {
|
||||
// 中间件
|
||||
esClientStr := viper.GetString("setting.esClient")
|
||||
if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
|
||||
ctx := context.Background()
|
||||
esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
|
||||
if err != nil {
|
||||
log.Error("Init es client Error:" + err.Error())
|
||||
}
|
||||
app.Use(middlewares.EsLog(ctx, esClient))
|
||||
}
|
||||
app.Use(middlewares.CORSMiddleware())
|
||||
anonymousGroup := app.Group("/")
|
||||
{
|
||||
|
||||
54
backend/middlewares/es_log.go
Normal file
54
backend/middlewares/es_log.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package middlewares
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/olivere/elastic/v7"
|
||||
"github.com/satori/go.uuid"
|
||||
"github.com/spf13/viper"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
func EsLog(ctx context.Context, esClient *elastic.Client) gin.HandlerFunc {
|
||||
|
||||
return func(c *gin.Context) {
|
||||
// 开始时间
|
||||
crawlabIndex := viper.GetString("setting.crawlabLogIndex")
|
||||
start := time.Now()
|
||||
// 处理请求
|
||||
c.Next()
|
||||
// 结束时间
|
||||
end := time.Now()
|
||||
//执行时间
|
||||
latency := strconv.FormatInt(int64(end.Sub(start).Milliseconds()), 10)
|
||||
path := c.Request.URL.Path
|
||||
|
||||
clientIP := c.ClientIP()
|
||||
method := c.Request.Method
|
||||
statusCode := strconv.Itoa(c.Writer.Status())
|
||||
buf := new(bytes.Buffer)
|
||||
buf.ReadFrom(c.Request.Body)
|
||||
b := buf.String()
|
||||
accessLog := "costTime:" + latency + "ms--" + "StatusCode:" + statusCode + "--" + "Method:" + method + "--" + "ClientIp:" + clientIP + "--" +
|
||||
"RequestURI:" + path + "--" + "Host:" + c.Request.Host + "--" + "UserAgent--" + c.Request.UserAgent() + "--RequestBody:" +
|
||||
string(b)
|
||||
WriteMsg(ctx, crawlabIndex, esClient, time.Now(), accessLog)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// WriteMsg will write the msg and level into es
|
||||
func WriteMsg(ctx context.Context, crawlabIndex string, es *elastic.Client, when time.Time, msg string) error {
|
||||
vals := make(map[string]interface{})
|
||||
vals["@timestamp"] = when.Format(time.RFC3339)
|
||||
vals["@msg"] = msg
|
||||
uid := uuid.NewV4().String()
|
||||
_, err := es.Index().Index(crawlabIndex).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -80,7 +80,7 @@ func GetCurrentNode() (Node, error) {
|
||||
Key: key,
|
||||
Id: bson.NewObjectId(),
|
||||
Ip: ip,
|
||||
Name: ip,
|
||||
Name: key,
|
||||
Mac: mac,
|
||||
Hostname: hostname,
|
||||
IsMaster: true,
|
||||
|
||||
@@ -76,10 +76,14 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) {
|
||||
|
||||
// 获取爬虫名称
|
||||
spider, err := GetSpider(schedule.SpiderId)
|
||||
if err != nil && err == mgo.ErrNotFound {
|
||||
if err != nil {
|
||||
log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error())
|
||||
schedule.Status = constants.ScheduleStatusError
|
||||
schedule.Message = constants.ScheduleStatusErrorNotFoundSpider
|
||||
if err == mgo.ErrNotFound {
|
||||
schedule.Message = constants.ScheduleStatusErrorNotFoundSpider
|
||||
} else {
|
||||
schedule.Message = err.Error()
|
||||
}
|
||||
} else {
|
||||
schedule.SpiderName = spider.Name
|
||||
}
|
||||
|
||||
@@ -508,3 +508,4 @@ func UpdateTaskErrorLogs(taskId string, errorRegexPattern string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,17 @@ import (
|
||||
)
|
||||
|
||||
// 添加可配置爬虫
|
||||
|
||||
// @Summary Put config spider
|
||||
// @Description Put config spider
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param spider body model.Spider true "spider item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders [put]
|
||||
func PutConfigSpider(c *gin.Context) {
|
||||
var spider model.Spider
|
||||
if err := c.ShouldBindJSON(&spider); err != nil {
|
||||
@@ -104,6 +115,18 @@ func PostConfigSpider(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 上传可配置爬虫Spiderfile
|
||||
|
||||
// @Summary Upload config spider
|
||||
// @Description Upload config spider
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param spider body model.Spider true "spider item"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders/{id}/upload [post]
|
||||
func UploadConfigSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -190,6 +213,16 @@ func UploadConfigSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post config spider
|
||||
// @Description Post config spider
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders/{id}/spiderfile [post]
|
||||
func PostConfigSpiderSpiderfile(c *gin.Context) {
|
||||
type Body struct {
|
||||
Content string `json:"content"`
|
||||
@@ -249,6 +282,17 @@ func PostConfigSpiderSpiderfile(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post config spider config
|
||||
// @Description Post config spider config
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param spider body model.Spider true "spider item"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders/{id}/config [post]
|
||||
func PostConfigSpiderConfig(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -296,6 +340,16 @@ func PostConfigSpiderConfig(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get config spider
|
||||
// @Description Get config spider
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders/{id}/config [get]
|
||||
func GetConfigSpiderConfig(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -319,6 +373,17 @@ func GetConfigSpiderConfig(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 获取模版名称列表
|
||||
|
||||
// @Summary Get config spider template list
|
||||
// @Description Get config spider template list
|
||||
// @Tags config spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /config_spiders_templates [get]
|
||||
func GetConfigSpiderTemplateList(c *gin.Context) {
|
||||
var data []string
|
||||
for _, fInfo := range utils.ListDir("./template/spiderfile") {
|
||||
|
||||
@@ -8,6 +8,14 @@ import (
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
// @Summary Get docs
|
||||
// @Description Get docs
|
||||
// @Tags docs
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /docs [get]
|
||||
func GetDocs(c *gin.Context) {
|
||||
type ResData struct {
|
||||
String string `json:"string"`
|
||||
|
||||
@@ -7,6 +7,14 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// @Summary Get file
|
||||
// @Description Get file
|
||||
// @Tags file
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /file [get]
|
||||
func GetFile(c *gin.Context) {
|
||||
path := c.Query("path")
|
||||
fileBytes, err := ioutil.ReadFile(path)
|
||||
|
||||
@@ -8,6 +8,14 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// @Summary Get nodes
|
||||
// @Description Get nodes
|
||||
// @Tags node
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes [get]
|
||||
func GetNodeList(c *gin.Context) {
|
||||
nodes, err := model.GetNodeList(nil)
|
||||
if err != nil {
|
||||
@@ -26,6 +34,15 @@ func GetNodeList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get node
|
||||
// @Description Get node
|
||||
// @Tags node
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param id path string true "id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id} [get]
|
||||
func GetNode(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -54,6 +71,17 @@ func Ping(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// @Summary Post node
|
||||
// @Description Post node
|
||||
// @Tags node
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param id path string true "post node"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /nodes/{id} [post]
|
||||
func PostNode(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -81,6 +109,15 @@ func PostNode(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get tasks on node
|
||||
// @Description Get tasks on node
|
||||
// @Tags node
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/tasks [get]
|
||||
func GetNodeTaskList(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -97,6 +134,15 @@ func GetNodeTaskList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get system info
|
||||
// @Description Get system info
|
||||
// @Tags node
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/system [get]
|
||||
func GetSystemInfo(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -109,6 +155,15 @@ func GetSystemInfo(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete node
|
||||
// @Description Delete node
|
||||
// @Tags node
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id} [delete]
|
||||
func DeleteNode(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
node, err := model.GetNode(bson.ObjectIdHex(id))
|
||||
|
||||
@@ -10,6 +10,15 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// @Summary Get projects
|
||||
// @Description Get projects
|
||||
// @Tags project
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param tag query string true "projects"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /projects [get]
|
||||
func GetProjectList(c *gin.Context) {
|
||||
tag := c.Query("tag")
|
||||
|
||||
@@ -70,6 +79,16 @@ func GetProjectList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put project
|
||||
// @Description Put project
|
||||
// @Tags project
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "With the bearer started"
|
||||
// @Param p body model.Project true "post project"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /projects [put]
|
||||
func PutProject(c *gin.Context) {
|
||||
// 绑定请求数据
|
||||
var p model.Project
|
||||
@@ -92,6 +111,17 @@ func PutProject(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post project
|
||||
// @Description Post project
|
||||
// @Tags project
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "project id"
|
||||
// @Param item body model.Project true "project item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /projects/{id} [post]
|
||||
func PostProject(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -116,6 +146,15 @@ func PostProject(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete project
|
||||
// @Description Delete project
|
||||
// @Tags project
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "project id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /projects/{id} [delete]
|
||||
func DeleteProject(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -154,6 +193,14 @@ func DeleteProject(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get project tags
|
||||
// @Description Get projects tags
|
||||
// @Tags project
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /projects/tags [get]
|
||||
func GetProjectTags(c *gin.Context) {
|
||||
type Result struct {
|
||||
Tag string `json:"tag" bson:"tag"`
|
||||
|
||||
@@ -8,6 +8,14 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// @Summary Get schedule list
|
||||
// @Description Get schedule list
|
||||
// @Tags schedule
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /schedules [get]
|
||||
func GetScheduleList(c *gin.Context) {
|
||||
query := bson.M{}
|
||||
|
||||
@@ -22,6 +30,15 @@ func GetScheduleList(c *gin.Context) {
|
||||
HandleSuccessData(c, results)
|
||||
}
|
||||
|
||||
// @Summary Get schedule by id
|
||||
// @Description Get schedule by id
|
||||
// @Tags schedule
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /schedules/{id} [get]
|
||||
func GetSchedule(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -34,6 +51,17 @@ func GetSchedule(c *gin.Context) {
|
||||
HandleSuccessData(c, result)
|
||||
}
|
||||
|
||||
// @Summary Post schedule
|
||||
// @Description Post schedule
|
||||
// @Tags schedule
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Param newItem body model.Schedule true "schedule item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /schedules/{id} [post]
|
||||
func PostSchedule(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -66,6 +94,16 @@ func PostSchedule(c *gin.Context) {
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// @Summary Put schedule
|
||||
// @Description Put schedule
|
||||
// @Tags schedule
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param item body model.Schedule true "schedule item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /schedules [put]
|
||||
func PutSchedule(c *gin.Context) {
|
||||
var item model.Schedule
|
||||
|
||||
@@ -99,6 +137,15 @@ func PutSchedule(c *gin.Context) {
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// @Summary Delete schedule
|
||||
// @Description Delete schedule
|
||||
// @Tags schedule
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /schedules/{id} [delete]
|
||||
func DeleteSchedule(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -118,6 +165,16 @@ func DeleteSchedule(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 停止定时任务
|
||||
// @Summary disable schedule
|
||||
// @Description disable schedule
|
||||
// @Tags schedule
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /schedules/{id}/disable [post]
|
||||
func DisableSchedule(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if err := services.Sched.Disable(bson.ObjectIdHex(id)); err != nil {
|
||||
@@ -128,6 +185,16 @@ func DisableSchedule(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 运行定时任务
|
||||
// @Summary enable schedule
|
||||
// @Description enable schedule
|
||||
// @Tags schedule
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /schedules/{id}/enable [post]
|
||||
func EnableSchedule(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
if err := services.Sched.Enable(bson.ObjectIdHex(id)); err != nil {
|
||||
|
||||
@@ -13,6 +13,14 @@ type SettingBody struct {
|
||||
EnableDemoSpiders string `json:"enable_demo_spiders"`
|
||||
}
|
||||
|
||||
// @Summary Get version
|
||||
// @Description Get version
|
||||
// @Tags setting
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /version [get]
|
||||
func GetVersion(c *gin.Context) {
|
||||
version := viper.GetString("version")
|
||||
|
||||
@@ -23,6 +31,14 @@ func GetVersion(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get setting
|
||||
// @Description Get setting
|
||||
// @Tags setting
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /setting [get]
|
||||
func GetSetting(c *gin.Context) {
|
||||
body := SettingBody{
|
||||
AllowRegister: viper.GetString("setting.allowRegister"),
|
||||
|
||||
@@ -28,6 +28,22 @@ import (
|
||||
|
||||
// ======== 爬虫管理 ========
|
||||
|
||||
// @Summary Get spider list
|
||||
// @Description Get spider list
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param page_num query string false "page num"
|
||||
// @Param page_size query string false "page size"
|
||||
// @Param keyword query string false "keyword"
|
||||
// @Param project_id query string false "project_id"
|
||||
// @Param type query string false "type"
|
||||
// @Param sort_key query string false "sort_key"
|
||||
// @Param sort_direction query string false "sort_direction"
|
||||
// @Param owner_type query string false "owner_type"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /schedules [get]
|
||||
func GetSpiderList(c *gin.Context) {
|
||||
pageNum := c.Query("page_num")
|
||||
pageSize := c.Query("page_size")
|
||||
@@ -109,6 +125,15 @@ func GetSpiderList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get spider by id
|
||||
// @Description Get spider by id
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id} [get]
|
||||
func GetSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -129,6 +154,17 @@ func GetSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post spider
|
||||
// @Description Post spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Param item body model.Spider true "spider item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders/{id} [post]
|
||||
func PostSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -177,6 +213,16 @@ func PostSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Publish spider
|
||||
// @Description Publish spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders/{id}/publish [post]
|
||||
func PublishSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -198,6 +244,16 @@ func PublishSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put spider
|
||||
// @Description Put spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param spider body model.Spider true "spider item"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders [put]
|
||||
func PutSpider(c *gin.Context) {
|
||||
var spider model.Spider
|
||||
if err := c.ShouldBindJSON(&spider); err != nil {
|
||||
@@ -279,6 +335,16 @@ func PutSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Copy spider
|
||||
// @Description Copy spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "schedule id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders/{id}/copy [post]
|
||||
func CopySpider(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
Name string `json:"name"`
|
||||
@@ -326,6 +392,20 @@ func CopySpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Upload spider
|
||||
// @Description Upload spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param file formData file true "spider file to upload"
|
||||
// @Param name formData string true "spider name"
|
||||
// @Param display_name formData string true "display name"
|
||||
// @Param col formData string true "col"
|
||||
// @Param cmd formData string true "cmd"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders [post]
|
||||
func UploadSpider(c *gin.Context) {
|
||||
// 从body中获取文件
|
||||
uploadFile, err := c.FormFile("file")
|
||||
@@ -467,6 +547,17 @@ func UploadSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Upload spider by id
|
||||
// @Description Upload spider by id
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param file formData file true "spider file to upload"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders/{id}/upload [post]
|
||||
func UploadSpiderFromId(c *gin.Context) {
|
||||
// TODO: 与 UploadSpider 部分逻辑重复，需要优化代码
|
||||
// 爬虫ID
|
||||
@@ -560,6 +651,15 @@ func UploadSpiderFromId(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete spider by id
|
||||
// @Description Delete spider by id
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id} [delete]
|
||||
func DeleteSpider(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -585,6 +685,15 @@ func DeleteSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary delete spider
|
||||
// @Description delete spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders [post]
|
||||
func DeleteSelectedSpider(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
SpiderIds []string `json:"spider_ids"`
|
||||
@@ -615,6 +724,15 @@ func DeleteSelectedSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary cancel spider
|
||||
// @Description cancel spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders-cancel [post]
|
||||
func CancelSelectedSpider(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
SpiderIds []string `json:"spider_ids"`
|
||||
@@ -639,6 +757,15 @@ func CancelSelectedSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary run spider
|
||||
// @Description run spider
|
||||
// @Tags spider
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 500 json string Response
|
||||
// @Router /spiders-run [post]
|
||||
func RunSelectedSpider(c *gin.Context) {
|
||||
type TaskParam struct {
|
||||
SpiderId bson.ObjectId `json:"spider_id"`
|
||||
@@ -734,6 +861,15 @@ func RunSelectedSpider(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get task list
|
||||
// @Description Get task list
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/tasks [get]
|
||||
func GetSpiderTasks(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -756,6 +892,15 @@ func GetSpiderTasks(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get spider stats
|
||||
// @Description Get spider stats
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/stats [get]
|
||||
func GetSpiderStats(c *gin.Context) {
|
||||
type Overview struct {
|
||||
TaskCount int `json:"task_count" bson:"task_count"`
|
||||
@@ -876,6 +1021,15 @@ func GetSpiderStats(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get schedules
|
||||
// @Description Get schedules
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/schedules [get]
|
||||
func GetSpiderSchedules(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -902,6 +1056,16 @@ func GetSpiderSchedules(c *gin.Context) {
|
||||
|
||||
// ======== 爬虫文件管理 ========
|
||||
|
||||
// @Summary Get spider dir
|
||||
// @Description Get spider dir
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param path query string true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/dir [get]
|
||||
func GetSpiderDir(c *gin.Context) {
|
||||
// 爬虫ID
|
||||
id := c.Param("id")
|
||||
@@ -949,6 +1113,16 @@ type SpiderFileReqBody struct {
|
||||
NewPath string `json:"new_path"`
|
||||
}
|
||||
|
||||
// @Summary Get spider file
|
||||
// @Description Get spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param path query string true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file [get]
|
||||
func GetSpiderFile(c *gin.Context) {
|
||||
// 爬虫ID
|
||||
id := c.Param("id")
|
||||
@@ -977,6 +1151,15 @@ func GetSpiderFile(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get spider dir
|
||||
// @Description Get spider dir
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file/tree [get]
|
||||
func GetSpiderFileTree(c *gin.Context) {
|
||||
// 爬虫ID
|
||||
id := c.Param("id")
|
||||
@@ -1007,6 +1190,16 @@ func GetSpiderFileTree(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post spider file
|
||||
// @Description Post spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqBody body routes.SpiderFileReqBody true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file [post]
|
||||
func PostSpiderFile(c *gin.Context) {
|
||||
// 爬虫ID
|
||||
id := c.Param("id")
|
||||
@@ -1044,6 +1237,16 @@ func PostSpiderFile(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put spider file
|
||||
// @Description Put spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqBody body routes.SpiderFileReqBody true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file [post]
|
||||
func PutSpiderFile(c *gin.Context) {
|
||||
spiderId := c.Param("id")
|
||||
var reqBody SpiderFileReqBody
|
||||
@@ -1084,6 +1287,16 @@ func PutSpiderFile(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post spider dir
|
||||
// @Description Post spider dir
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqBody body routes.SpiderFileReqBody true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file [put]
|
||||
func PutSpiderDir(c *gin.Context) {
|
||||
spiderId := c.Param("id")
|
||||
var reqBody SpiderFileReqBody
|
||||
@@ -1124,6 +1337,16 @@ func PutSpiderDir(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete spider file
|
||||
// @Description Delete spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqBody body routes.SpiderFileReqBody true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file [delete]
|
||||
func DeleteSpiderFile(c *gin.Context) {
|
||||
spiderId := c.Param("id")
|
||||
var reqBody SpiderFileReqBody
|
||||
@@ -1154,6 +1377,16 @@ func DeleteSpiderFile(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Rename spider file
|
||||
// @Description Rename spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqBody body routes.SpiderFileReqBody true "path"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/file/rename [post]
|
||||
func RenameSpiderFile(c *gin.Context) {
|
||||
spiderId := c.Param("id")
|
||||
var reqBody SpiderFileReqBody
|
||||
@@ -1203,6 +1436,15 @@ func RenameSpiderFile(c *gin.Context) {
|
||||
|
||||
// ======== Scrapy 部分 ========
|
||||
|
||||
// @Summary Get scrapy spider file
|
||||
// @Description Get scrapy spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/spiders [get]
|
||||
func GetSpiderScrapySpiders(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1230,6 +1472,15 @@ func GetSpiderScrapySpiders(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put scrapy spider file
|
||||
// @Description Put scrapy spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/spiders [put]
|
||||
func PutSpiderScrapySpiders(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
Name string `json:"name"`
|
||||
@@ -1267,6 +1518,15 @@ func PutSpiderScrapySpiders(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get scrapy spider settings
|
||||
// @Description Get scrapy spider settings
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/settings [get]
|
||||
func GetSpiderScrapySettings(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1294,6 +1554,16 @@ func GetSpiderScrapySettings(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get scrapy spider file
|
||||
// @Description Get scrapy spider file
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqData body []entity.ScrapySettingParam true "req data"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/settings [post]
|
||||
func PostSpiderScrapySettings(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1325,6 +1595,15 @@ func PostSpiderScrapySettings(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get scrapy spider items
|
||||
// @Description Get scrapy spider items
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/items [get]
|
||||
func GetSpiderScrapyItems(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1352,6 +1631,16 @@ func GetSpiderScrapyItems(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post scrapy spider items
|
||||
// @Description Post scrapy spider items
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Param reqData body []entity.ScrapyItem true "req data"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/items [post]
|
||||
func PostSpiderScrapyItems(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1383,6 +1672,16 @@ func PostSpiderScrapyItems(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// @Summary Get scrapy spider pipelines
|
||||
// @Description Get scrapy spider pipelines
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/pipelines [get]
|
||||
func GetSpiderScrapyPipelines(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1410,6 +1709,15 @@ func GetSpiderScrapyPipelines(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get scrapy spider file path
|
||||
// @Description Get scrapy spider file path
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/scrapy/spider/filepath [get]
|
||||
func GetSpiderScrapySpiderFilepath(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1447,6 +1755,15 @@ func GetSpiderScrapySpiderFilepath(c *gin.Context) {
|
||||
|
||||
// ======== Git 部分 ========
|
||||
|
||||
// @Summary Post spider sync git
|
||||
// @Description Post spider sync git
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/git/sync [post]
|
||||
func PostSpiderSyncGit(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -1472,6 +1789,15 @@ func PostSpiderSyncGit(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post spider reset git
|
||||
// @Description Post spider reset git
|
||||
// @Tags spider
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "spider id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /spiders/{id}/git/reset [post]
|
||||
func PostSpiderResetGit(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
|
||||
@@ -9,6 +9,14 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// @Summary Get home stats
|
||||
// @Description Get home stats
|
||||
// @Tags version
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /stats/home [get]
|
||||
func GetHomeStats(c *gin.Context) {
|
||||
type DataOverview struct {
|
||||
TaskCount int `json:"task_count"`
|
||||
|
||||
@@ -11,6 +11,15 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// @Summary Get language list
|
||||
// @Description Get language list
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/langs [get]
|
||||
func GetLangList(c *gin.Context) {
|
||||
nodeId := c.Param("id")
|
||||
c.JSON(http.StatusOK, Response{
|
||||
@@ -20,6 +29,17 @@ func GetLangList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get dep list
|
||||
// @Description Get dep list
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Param lang query string true "language"
|
||||
// @Param dep_name query string true "dep name"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/deps [get]
|
||||
func GetDepList(c *gin.Context) {
|
||||
nodeId := c.Param("id")
|
||||
lang := c.Query("lang")
|
||||
@@ -52,6 +72,16 @@ func GetDepList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get installed dep list
|
||||
// @Description Get installed dep list
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Param lang query string true "language"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/deps/installed [get]
|
||||
func GetInstalledDepList(c *gin.Context) {
|
||||
nodeId := c.Param("id")
|
||||
lang := c.Query("lang")
|
||||
@@ -79,6 +109,16 @@ func GetInstalledDepList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get all dep list
|
||||
// @Description Get all dep list
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param lang path string true "language"
|
||||
// @Param dep_nane query string true "dep name"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /system/deps/:lang [get]
|
||||
func GetAllDepList(c *gin.Context) {
|
||||
lang := c.Param("lang")
|
||||
depName := c.Query("dep_name")
|
||||
@@ -121,6 +161,15 @@ func GetAllDepList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Install dep
|
||||
// @Description Install dep
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/deps/install [Post]
|
||||
func InstallDep(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
Lang string `json:"lang"`
|
||||
@@ -153,6 +202,15 @@ func InstallDep(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Uninstall dep
|
||||
// @Description Uninstall dep
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/deps/uninstall [Post]
|
||||
func UninstallDep(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
Lang string `json:"lang"`
|
||||
@@ -184,6 +242,16 @@ func UninstallDep(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get dep json
|
||||
// @Description Get dep json
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param lang path string true "language"
|
||||
// @Param dep_name path string true "dep name"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /system/deps/{lang}/{dep_name}/json [get]
|
||||
func GetDepJson(c *gin.Context) {
|
||||
depName := c.Param("dep_name")
|
||||
lang := c.Param("lang")
|
||||
@@ -209,6 +277,15 @@ func GetDepJson(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Install language
|
||||
// @Description Install language
|
||||
// @Tags system
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "node id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /nodes/{id}/langs/install [Post]
|
||||
func InstallLang(c *gin.Context) {
|
||||
type ReqBody struct {
|
||||
Lang string `json:"lang"`
|
||||
|
||||
@@ -26,6 +26,15 @@ type TaskResultsRequestData struct {
|
||||
PageSize int `form:"page_size"`
|
||||
}
|
||||
|
||||
// @Summary Get task list
|
||||
// @Description Get task list
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param data body routes.TaskListRequestData true "req data"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks [get]
|
||||
func GetTaskList(c *gin.Context) {
|
||||
// 绑定数据
|
||||
data := TaskListRequestData{}
|
||||
@@ -81,6 +90,15 @@ func GetTaskList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get task
|
||||
// @Description Get task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id} [get]
|
||||
func GetTask(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -92,6 +110,14 @@ func GetTask(c *gin.Context) {
|
||||
HandleSuccessData(c, result)
|
||||
}
|
||||
|
||||
// @Summary Put task
|
||||
// @Description Put task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks [put]
|
||||
func PutTask(c *gin.Context) {
|
||||
type TaskRequestBody struct {
|
||||
SpiderId bson.ObjectId `json:"spider_id"`
|
||||
@@ -177,6 +203,15 @@ func PutTask(c *gin.Context) {
|
||||
HandleSuccessData(c, taskIds)
|
||||
}
|
||||
|
||||
// @Summary Delete task
|
||||
// @Description Delete task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param status query string true "task status"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks_by_status [delete]
|
||||
func DeleteTaskByStatus(c *gin.Context) {
|
||||
status := c.Query("status")
|
||||
|
||||
@@ -196,6 +231,15 @@ func DeleteTaskByStatus(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 删除多个任务
|
||||
|
||||
// @Summary Delete tasks
|
||||
// @Description Delete tasks
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks [delete]
|
||||
func DeleteSelectedTask(c *gin.Context) {
|
||||
ids := make(map[string][]string)
|
||||
if err := c.ShouldBindJSON(&ids); err != nil {
|
||||
@@ -217,6 +261,16 @@ func DeleteSelectedTask(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 删除单个任务
|
||||
|
||||
// @Summary Delete task
|
||||
// @Description Delete task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /task/{id} [delete]
|
||||
func DeleteTask(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -233,6 +287,15 @@ func DeleteTask(c *gin.Context) {
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// @Summary Get task log
|
||||
// @Description Get task log
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/log [delete]
|
||||
func GetTaskLog(c *gin.Context) {
|
||||
type RequestData struct {
|
||||
PageNum int `form:"page_num"`
|
||||
@@ -258,6 +321,15 @@ func GetTaskLog(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get task error log
|
||||
// @Description Get task error log
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/error-log [delete]
|
||||
func GetTaskErrorLog(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
u := services.GetCurrentUser(c)
|
||||
@@ -273,6 +345,16 @@ func GetTaskErrorLog(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get task list
|
||||
// @Description Get task list
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param data body routes.TaskResultsRequestData true "req data"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/results [get]
|
||||
func GetTaskResults(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -305,6 +387,16 @@ func GetTaskResults(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
// @Summary Get task results
|
||||
// @Description Get task results
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/results/download [get]
|
||||
func DownloadTaskResultsCsv(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -374,6 +466,16 @@ func DownloadTaskResultsCsv(c *gin.Context) {
|
||||
c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes())
|
||||
}
|
||||
|
||||
|
||||
// @Summary Cancel task
|
||||
// @Description Cancel task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/cancel [post]
|
||||
func CancelTask(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -384,6 +486,15 @@ func CancelTask(c *gin.Context) {
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
// @Summary Restart task
|
||||
// @Description Restart task
|
||||
// @Tags task
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "task id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tasks/{id}/restart [post]
|
||||
func RestartTask(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
|
||||
@@ -9,6 +9,14 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// @Summary Get token
|
||||
// @Description token
|
||||
// @Tags token
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tokens [get]
|
||||
func GetTokens(c *gin.Context) {
|
||||
u := services.GetCurrentUser(c)
|
||||
|
||||
@@ -25,6 +33,14 @@ func GetTokens(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put token
|
||||
// @Description token
|
||||
// @Tags token
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tokens [put]
|
||||
func PutToken(c *gin.Context) {
|
||||
u := services.GetCurrentUser(c)
|
||||
|
||||
@@ -53,6 +69,15 @@ func PutToken(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete token
|
||||
// @Description Delete token
|
||||
// @Tags token
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "token id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /tokens/{id} [delete]
|
||||
func DeleteToken(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
|
||||
@@ -25,6 +25,15 @@ type UserRequestData struct {
|
||||
Email string `json:"email"`
|
||||
}
|
||||
|
||||
// @Summary Get user
|
||||
// @Description user
|
||||
// @Tags user
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "user id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /users/{id} [get]
|
||||
func GetUser(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -41,6 +50,15 @@ func GetUser(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Get user list
|
||||
// @Description Get user list
|
||||
// @Tags token
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param data body routes.UserListRequestData true "data body"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /users [get]
|
||||
func GetUserList(c *gin.Context) {
|
||||
// 绑定数据
|
||||
data := UserListRequestData{}
|
||||
@@ -82,6 +100,15 @@ func GetUserList(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Put user
|
||||
// @Description Put user
|
||||
// @Tags user
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param reqData body routes.UserRequestData true "reqData body"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /users [put]
|
||||
func PutUser(c *gin.Context) {
|
||||
// 绑定请求数据
|
||||
var reqData UserRequestData
|
||||
@@ -115,6 +142,16 @@ func PutUser(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Post user
|
||||
// @Description Post user
|
||||
// @Tags user
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param item body model.User true "user body"
|
||||
// @Param id path string true "user id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /users/{id} [post]
|
||||
func PostUser(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
@@ -143,6 +180,15 @@ func PostUser(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// @Summary Delete user
|
||||
// @Description Delete user
|
||||
// @Tags user
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "user id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /users/{id} [delete]
|
||||
func DeleteUser(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
|
||||
@@ -8,6 +8,16 @@ import (
|
||||
)
|
||||
|
||||
// 新增
|
||||
|
||||
// @Summary Put variable
|
||||
// @Description Put variable
|
||||
// @Tags variable
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param variable body model.Variable true "reqData body"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /variable [put]
|
||||
func PutVariable(c *gin.Context) {
|
||||
var variable model.Variable
|
||||
if err := c.ShouldBindJSON(&variable); err != nil {
|
||||
@@ -22,6 +32,17 @@ func PutVariable(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 修改
|
||||
|
||||
// @Summary Post variable
|
||||
// @Description Post variable
|
||||
// @Tags variable
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param variable body model.Variable true "reqData body"
|
||||
// @Param id path string true "variable id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /variable/{id} [post]
|
||||
func PostVariable(c *gin.Context) {
|
||||
var id = c.Param("id")
|
||||
var variable model.Variable
|
||||
@@ -38,6 +59,16 @@ func PostVariable(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 删除
|
||||
|
||||
// @Summary Delete variable
|
||||
// @Description Delete variable
|
||||
// @Tags variable
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Param id path string true "variable id"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /variable/{id} [delete]
|
||||
func DeleteVariable(c *gin.Context) {
|
||||
var idStr = c.Param("id")
|
||||
var id = bson.ObjectIdHex(idStr)
|
||||
@@ -56,6 +87,15 @@ func DeleteVariable(c *gin.Context) {
|
||||
}
|
||||
|
||||
// 列表
|
||||
|
||||
// @Summary Get variable list
|
||||
// @Description Get variable list
|
||||
// @Tags variable
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /variables [get]
|
||||
func GetVariableList(c *gin.Context) {
|
||||
list := model.GetVariableList()
|
||||
HandleSuccessData(c, list)
|
||||
|
||||
@@ -8,6 +8,14 @@ import (
|
||||
"runtime/debug"
|
||||
)
|
||||
|
||||
// @Summary Get latest release
|
||||
// @Description Get latest release
|
||||
// @Tags version
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization token"
|
||||
// @Success 200 json string Response
|
||||
// @Failure 400 json string Response
|
||||
// @Router /releases/latest [get]
|
||||
func GetLatestRelease(c *gin.Context) {
|
||||
latestRelease, err := services.GetLatestRelease()
|
||||
if err != nil {
|
||||
|
||||
@@ -25,6 +25,7 @@ type Register interface {
|
||||
GetMac() (string, error)
|
||||
// 注册节点的Hostname
|
||||
GetHostname() (string, error)
|
||||
GetCustomName() (string, error)
|
||||
}
|
||||
|
||||
// ===================== mac 地址注册 =====================
|
||||
@@ -50,11 +51,50 @@ func (mac *MacRegister) GetHostname() (string, error) {
|
||||
return getHostname()
|
||||
}
|
||||
|
||||
func (mac *MacRegister) GetCustomName() (string, error) {
|
||||
return getMac()
|
||||
}
|
||||
|
||||
// ===================== ip 地址注册 =====================
|
||||
type IpRegister struct {
|
||||
Ip string
|
||||
}
|
||||
|
||||
func (ip *IpRegister) GetCustomName() (string, error) {
|
||||
return ip.Ip, nil
|
||||
}
|
||||
|
||||
// ============= 自定义节点名称注册 ==============
|
||||
type CustomNameRegister struct {
|
||||
CustomName string
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetType() string {
|
||||
return "customName"
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetIp() (string, error) {
|
||||
return getIp()
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetMac() (string, error) {
|
||||
return getMac()
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetKey() (string, error) {
|
||||
return c.CustomName, nil
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetHostname() (string, error) {
|
||||
|
||||
return getHostname()
|
||||
}
|
||||
|
||||
func (c *CustomNameRegister) GetCustomName() (string, error) {
|
||||
return c.CustomName, nil
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
func (ip *IpRegister) GetType() string {
|
||||
return "ip"
|
||||
}
|
||||
@@ -98,6 +138,10 @@ func (h *HostnameRegister) GetHostname() (string, error) {
|
||||
return getHostname()
|
||||
}
|
||||
|
||||
func (h *HostnameRegister) GetCustomName() (string, error) {
|
||||
return getHostname()
|
||||
}
|
||||
|
||||
// ===================== 公共方法 =====================
|
||||
// 获取本机的IP地址
|
||||
// TODO: 考虑多个IP地址的情况
|
||||
@@ -158,9 +202,14 @@ var once sync.Once
|
||||
func GetRegister() Register {
|
||||
once.Do(func() {
|
||||
registerType := viper.GetString("server.register.type")
|
||||
if registerType == constants.RegisterTypeMac {
|
||||
|
||||
switch registerType {
|
||||
case constants.RegisterTypeMac:
|
||||
|
||||
register = &MacRegister{}
|
||||
} else if registerType == constants.RegisterTypeIp {
|
||||
|
||||
case constants.RegisterTypeIp:
|
||||
|
||||
ip := viper.GetString("server.register.ip")
|
||||
if ip == "" {
|
||||
log.Error("server.register.ip is empty")
|
||||
@@ -170,8 +219,22 @@ func GetRegister() Register {
|
||||
register = &IpRegister{
|
||||
Ip: ip,
|
||||
}
|
||||
} else if registerType == constants.RegisterTypeHostname {
|
||||
|
||||
case constants.RegisterTypeHostname:
|
||||
|
||||
register = &HostnameRegister{}
|
||||
|
||||
case constants.RegisterTypeCustomName:
|
||||
|
||||
customNodeName := viper.GetString("server.register.customNodeName")
|
||||
if customNodeName == "" {
|
||||
log.Error("server.register.customNodeName is empty")
|
||||
debug.PrintStack()
|
||||
register = nil
|
||||
}
|
||||
register = &CustomNameRegister{
|
||||
CustomName: customNodeName,
|
||||
}
|
||||
}
|
||||
log.Info("register type is :" + reflect.TypeOf(register).String())
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"github.com/apex/log"
|
||||
"github.com/globalsign/mgo/bson"
|
||||
"github.com/imroc/req"
|
||||
uuid "github.com/satori/go.uuid"
|
||||
"github.com/satori/go.uuid"
|
||||
"github.com/spf13/viper"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -166,7 +166,11 @@ func SetEnv(cmd *exec.Cmd, envs []model.Env, task model.Task, spider model.Spide
|
||||
return cmd
|
||||
}
|
||||
|
||||
func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
func SetLogConfig(wg *sync.WaitGroup, cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
|
||||
esChan := make(chan string, 1)
|
||||
esClientStr := viper.GetString("setting.esClient")
|
||||
spiderLogIndex := viper.GetString("setting.spiderLogIndex")
|
||||
// get stdout reader
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
readerStdout := bufio.NewReader(stdout)
|
||||
@@ -191,7 +195,9 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
isStderrFinished := false
|
||||
|
||||
// periodically (1 sec) insert log items
|
||||
wg.Add(3)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
_ = model.AddLogItems(logs)
|
||||
logs = []model.LogItem{}
|
||||
@@ -211,6 +217,7 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
|
||||
// read stdout
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
line, err := readerStdout.ReadString('\n')
|
||||
if err != nil {
|
||||
@@ -227,12 +234,18 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
Ts: time.Now(),
|
||||
ExpireTs: time.Now().Add(time.Duration(expireDuration) * time.Second),
|
||||
}
|
||||
esChan <- l.Message
|
||||
if esClientStr != "" {
|
||||
go database.WriteMsgToES(time.Now(), esChan, spiderLogIndex)
|
||||
}
|
||||
|
||||
logs = append(logs, l)
|
||||
}
|
||||
}()
|
||||
|
||||
// read stderr
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
line, err := readerStderr.ReadString('\n')
|
||||
if err != nil {
|
||||
@@ -249,10 +262,15 @@ func SetLogConfig(cmd *exec.Cmd, t model.Task, u model.User) error {
|
||||
Ts: time.Now(),
|
||||
ExpireTs: time.Now().Add(time.Duration(expireDuration) * time.Second),
|
||||
}
|
||||
esChan <- l.Message
|
||||
if esClientStr != "" {
|
||||
go database.WriteMsgToES(time.Now(), esChan, spiderLogIndex)
|
||||
}
|
||||
logs = append(logs, l)
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -337,6 +355,8 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
|
||||
log.Infof("cwd: %s", cwd)
|
||||
log.Infof("cmd: %s", cmdStr)
|
||||
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
// 生成执行命令
|
||||
var cmd *exec.Cmd
|
||||
if runtime.GOOS == constants.Windows {
|
||||
@@ -349,9 +369,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider, u
|
||||
cmd.Dir = cwd
|
||||
|
||||
// 日志配置
|
||||
if err := SetLogConfig(cmd, t, u); err != nil {
|
||||
return err
|
||||
}
|
||||
go SetLogConfig(wg, cmd, t, u)
|
||||
|
||||
// 环境变量配置
|
||||
envs := s.Envs
|
||||
|
||||
Reference in New Issue
Block a user