git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

added cli

Browse Source
This commit is contained in:
marvzhang
2021-04-27 11:34:12 +08:00
parent 98913cd3aa
commit be5a2207fd
10 changed files with 453 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/apps/api.go
View File

@@ -2,7 +2,6 @@ package apps
import (
"context"
"fmt"
"github.com/apex/log"
"github.com/crawlab-team/crawlab-core/config"
"github.com/crawlab-team/crawlab-core/controllers"
@@ -11,7 +10,6 @@ import (
"github.com/crawlab-team/crawlab-core/routes"
"github.com/crawlab-team/crawlab-db/mongo"
"github.com/crawlab-team/crawlab-db/redis"
"github.com/crawlab-team/go-trace"
"github.com/gin-gonic/gin"
"github.com/spf13/viper"
"net"
@@ -23,6 +21,7 @@ import (
)
type Api struct {
BaseApp
app *gin.Engine
}
@@ -82,16 +81,6 @@ func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (
})
}
func (app *Api) initModule(name string, fn func() error) (err error) {
if err := fn(); err != nil {
log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
_ = trace.TraceError(err)
panic(err)
}
log.Info(fmt.Sprintf("initialized %s successfully", name))
return nil
}
func NewApi() *Api {
app := gin.New()
return &Api{

27
backend/apps/base.go
View File

@@ -1,6 +1,33 @@
package apps
import (
"fmt"
"github.com/apex/log"
"github.com/crawlab-team/go-trace"
)
type App interface {
Init()
Run()
}
type BaseApp struct {
}
func (app *BaseApp) Init() {
panic("implement me")
}
func (app *BaseApp) Run() {
panic("implement me")
}
func (app *BaseApp) initModule(name string, fn func() error) (err error) {
if err := fn(); err != nil {
log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
_ = trace.TraceError(err)
panic(err)
}
log.Info(fmt.Sprintf("initialized %s successfully", name))
return nil
}

21
backend/apps/handler.go Normal file
View File

@@ -0,0 +1,21 @@
package apps
import (
"github.com/crawlab-team/crawlab-core/services"
)
type Handler struct {
BaseApp
}
func (app *Handler) Init() {
_ = app.initModule("task-service", services.InitTaskService)
}
func (app *Handler) Run() {
panic("implement me")
}
func NewHandler() *Handler {
return &Handler{}
}

21
backend/cmd/api.go Normal file
View File

@@ -0,0 +1,21 @@
package cmd
import (
"crawlab/apps"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(apiCmd)
}
var apiCmd = &cobra.Command{
Use: "api",
Short: "Start API server",
Long: `Start API server of Crawlab which serves data to frontend`,
Run: func(cmd *cobra.Command, args []string) {
api := apps.NewApi()
api.Init()
api.Run()
},
}

21
backend/cmd/handler.go Normal file
View File

@@ -0,0 +1,21 @@
package cmd
import (
"crawlab/apps"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(handlerCmd)
}
var handlerCmd = &cobra.Command{
Use: "handler",
Short: "Start API server",
Long: `Start API server of Crawlab which serves data to frontend`,
Run: func(cmd *cobra.Command, args []string) {
handler := apps.NewHandler()
handler.Init()
handler.Run()
},
}

60
backend/cmd/root.go Normal file
View File

@@ -0,0 +1,60 @@
package cmd
import (
"fmt"
"github.com/mitchellh/go-homedir"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var (
// Used for flags.
cfgFile string
rootCmd = &cobra.Command{
Use: "crawlab",
Short: "CLI tool for Crawlab",
Long: `The CLI tool is for controlling against Crawlab.
Crawlab is a distributed web crawler and task admin platform
aimed at making web crawling and task management easier.
`,
}
)
// Execute executes the root command.
func Execute() error {
return rootCmd.Execute()
}
func init() {
cobra.OnInitialize(initConfig)
//rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cobra.yaml)")
//rootCmd.PersistentFlags().StringP("author", "a", "YOUR NAME", "author name for copyright attribution")
//rootCmd.PersistentFlags().StringVarP(&userLicense, "license", "l", "", "name of license for the project")
//rootCmd.PersistentFlags().Bool("viper", true, "use Viper for configuration")
//viper.BindPFlag("author", rootCmd.PersistentFlags().Lookup("author"))
//viper.BindPFlag("useViper", rootCmd.PersistentFlags().Lookup("viper"))
}
func initConfig() {
if cfgFile != "" {
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
} else {
// Find home directory.
home, err := homedir.Dir()
cobra.CheckErr(err)
// Search config in home directory with name ".cobra" (without extension).
viper.AddConfigPath(home)
viper.SetConfigName(".cobra")
}
viper.AutomaticEnv()
if err := viper.ReadInConfig(); err == nil {
fmt.Println("Using config file:", viper.ConfigFileUsed())
}
}

4
backend/go.mod
View File

@@ -19,7 +19,7 @@ require (
github.com/crawlab-team/crawlab-db v0.0.2
github.com/crawlab-team/go-trace v0.0.0
github.com/gin-gonic/gin v1.6.3
github.com/go-playground/validator/v10 v10.3.0
github.com/olivere/elastic/v7 v7.0.15
github.com/mitchellh/go-homedir v1.1.0
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1
)

76
backend/go.sum
View File

@@ -19,9 +19,9 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0
github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY=
github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
@@ -63,9 +63,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/crawlab-team/crawlab-fs v0.0.0 h1:RGPWKB7ORgGEYLS5uplS/dIbdY8ktKmfBHM2MxxwQKQ=
github.com/crawlab-team/crawlab-fs v0.0.0/go.mod h1:k2VXprQspLAmbgO5sSpqMjg/xP4iKDkW4RyTWY8eTZM=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -79,8 +77,10 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fasthttp/websocket v1.4.2 h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM=
github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -90,6 +90,7 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gavv/httpexpect/v2 v2.2.0 h1:0VwaEBmQaNFHX9x591A8Up+8shCwdF/nF0qlRd/nI48=
github.com/gavv/httpexpect/v2 v2.2.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -98,12 +99,11 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
@@ -111,6 +111,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -187,7 +188,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -203,6 +206,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -229,16 +233,17 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E=
github.com/imkira/go-interpol v1.0.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U=
github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
@@ -249,6 +254,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
@@ -263,6 +269,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
@@ -282,17 +289,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/linxGnu/goseaweedfs v0.1.5 h1:7dChPdq8+fsPH0yqxKEofPhiosaar4LWePm4M+1Taz0=
github.com/linxGnu/goseaweedfs v0.1.5/go.mod h1:Zwe/7H7FJaPQyMTNKXgv6fhVDw6qi34MMJQp1K0VLNc=
github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M=
github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
@@ -307,6 +312,7 @@ github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFD
github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
@@ -340,6 +346,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
@@ -347,12 +354,13 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW
github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE=
github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
@@ -382,9 +390,11 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY=
github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -414,19 +424,21 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -437,6 +449,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
@@ -450,7 +463,9 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
@@ -460,13 +475,20 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y=
github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww=
@@ -492,8 +514,6 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -531,8 +551,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -571,14 +589,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg=
golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -611,8 +625,6 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -669,8 +681,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
@@ -679,12 +693,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
@@ -697,6 +706,8 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -705,5 +716,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo=
moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

251
backend/main.go
View File

@@ -1,252 +1,9 @@
package main
import "crawlab/apps"
import (
"crawlab/cmd"
)
func main() {
api := apps.NewApi()
api.Init()
api.Run()
//if model.IsMaster() {
// // 初始化定时任务
// if err := services.InitScheduler(); err != nil {
// log.Error("init scheduler error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized schedule successfully")
//
// // 初始化用户服务
// if err := services.InitUserService(); err != nil {
// log.Error("init user service error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized user service successfully")
//
// // 初始化依赖服务
// if err := services.InitDepsFetcher(); err != nil {
// log.Error("init dependency fetcher error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized dependency fetcher successfully")
//
// // 初始化清理服务
// if err := services.InitCleanService(); err != nil {
// log.Error("init clean service error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized clean service successfully")
//}
//
//// 初始化任务执行器
//if err := services.InitTaskExecutor(); err != nil {
// log.Error("init task executor error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized task executor successfully")
//
//// 初始化爬虫服务
//if err := services.InitSpiderService(); err != nil {
// log.Error("init spider service error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized spider service successfully")
//
//// 初始化RPC服务
//if err := rpc.InitRpcService(); err != nil {
// log.Error("init rpc service error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized rpc service successfully")
// 以下为主节点服务
//if model.IsMaster() {
// // 中间件
// esClientStr := viper.GetString("setting.esClient")
// if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
// ctx := context.Background()
// esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
// if err != nil {
// log.Error("Init es client Error:" + err.Error())
// }
// app.Use(middlewares.EsLog(ctx, esClient))
// }
// app.Use(middlewares.CORSMiddleware())
// anonymousGroup := app.Group("/")
// {
// anonymousGroup.POST("/login", routes.Login) // 用户登录
// anonymousGroup.PUT("/users", routes.PutUser) // 添加用户
// anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
// // release版本
// anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本
// anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本
// // 文档
// anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据
// }
// authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
// {
// // 节点
// {
// authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
// authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
// authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
// authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
// authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
// authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
// authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
// authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
// authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
// authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
// authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
// authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
// }
// // 爬虫
// {
// authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
// authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
// authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
// authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
// authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
// authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
// authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
// authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫
// authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
// authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫
// authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
// authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
// authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
// authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
// authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
// authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
// authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
// authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
// authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
// authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
// authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
// authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
// authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫
// authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
// authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置
// authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items
// authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items
// authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines
// authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
// authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步
// authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
// authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务
// authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫
// authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目
// }
// // 可配置爬虫
// {
// authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
// authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
// authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
// authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
// authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
// authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
// authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
// }
// // 任务
// {
// authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
// authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
// authGroup.PUT("/tasks", routes.PutTask) // 派发任务
// authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务
// authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
// authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务
// //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务
// authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
// authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
// authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志
// authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
// authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
// authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务
// authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务
// authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务
// }
// // 系统任务/脚本
// {
// authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务
// authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表
// }
// // 定时任务
// {
// authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
// authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
// authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
// authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务
// authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
// authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
// authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务
// authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
// authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
// authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态
// }
// // 用户
// {
// authGroup.GET("/users", routes.GetUserList) // 用户列表
// authGroup.GET("/users/:id", routes.GetUser) // 用户详情
// authGroup.POST("/users/:id", routes.PostUser) // 更改用户
// authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
// authGroup.PUT("/users-add", routes.PutUser) // 添加用户
// authGroup.GET("/me", routes.GetMe) // 获取自己账户
// authGroup.POST("/me", routes.PostMe) // 修改自己账户
// authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码
// }
// // 系统
// {
// authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
// authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
// }
// // 全局变量
// {
// authGroup.GET("/variables", routes.GetVariableList) // 列表
// authGroup.PUT("/variable", routes.PutVariable) // 新增
// authGroup.POST("/variable/:id", routes.PostVariable) // 修改
// authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除
// }
// // 项目
// {
// authGroup.GET("/projects", routes.GetProjectList) // 列表
// authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签
// authGroup.PUT("/projects", routes.PutProject) // 修改
// authGroup.POST("/projects/:id", routes.PostProject) // 新增
// authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除
// }
// // 操作
// {
// //authGroup.GET("/actions", routes.GetActionList) // 操作列表
// //authGroup.GET("/actions/:id", routes.GetAction) // 操作
// authGroup.PUT("/actions", routes.PutAction) // 新增操作
// //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作
// }
// // API Token
// {
// authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens
// authGroup.PUT("/tokens", routes.PutToken) // 添加 Token
// authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token
// }
// // 统计数据
// authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
// // 文件
// authGroup.GET("/file", routes.GetFile) // 获取文件
// // Git
// authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支
// authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
// authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits
// authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits
// }
//}
// 路由ping
//app.GET("/ping", routes.Ping)
// 运行服务器
_ = cmd.Execute()
}

252
backend/main_legacy.go Normal file
View File

@@ -0,0 +1,252 @@
package main
import "crawlab/apps"
func main() {
api := apps.NewApi()
api.Init()
api.Run()
//if model.IsMaster() {
// // 初始化定时任务
// if err := services.InitScheduler(); err != nil {
// log.Error("init scheduler error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized schedule successfully")
//
// // 初始化用户服务
// if err := services.InitUserService(); err != nil {
// log.Error("init user service error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized user service successfully")
//
// // 初始化依赖服务
// if err := services.InitDepsFetcher(); err != nil {
// log.Error("init dependency fetcher error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized dependency fetcher successfully")
//
// // 初始化清理服务
// if err := services.InitCleanService(); err != nil {
// log.Error("init clean service error:" + err.Error())
// debug.PrintStack()
// panic(err)
// }
// log.Info("initialized clean service successfully")
//}
//
//// 初始化任务执行器
//if err := services.InitTaskExecutor(); err != nil {
// log.Error("init task executor error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized task executor successfully")
//
//// 初始化爬虫服务
//if err := services.InitSpiderService(); err != nil {
// log.Error("init spider service error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized spider service successfully")
//
//// 初始化RPC服务
//if err := rpc.InitRpcService(); err != nil {
// log.Error("init rpc service error:" + err.Error())
// debug.PrintStack()
// panic(err)
//}
//log.Info("initialized rpc service successfully")
// 以下为主节点服务
//if model.IsMaster() {
// // 中间件
// esClientStr := viper.GetString("setting.esClient")
// if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
// ctx := context.Background()
// esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
// if err != nil {
// log.Error("Init es client Error:" + err.Error())
// }
// app.Use(middlewares.EsLog(ctx, esClient))
// }
// app.Use(middlewares.CORSMiddleware())
// anonymousGroup := app.Group("/")
// {
// anonymousGroup.POST("/login", routes.Login) // 用户登录
// anonymousGroup.PUT("/users", routes.PutUser) // 添加用户
// anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
// // release版本
// anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本
// anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本
// // 文档
// anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据
// }
// authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
// {
// // 节点
// {
// authGroup.GET("/nodes", routes.GetNodeList) // 节点列表
// authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情
// authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点
// authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
// authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表
// authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点
// authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表
// authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表
// authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
// authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖
// authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖
// authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言
// }
// // 爬虫
// {
// authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表
// authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情
// authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫
// authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫
// authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫
// authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
// authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID）
// authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫
// authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫
// authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫
// authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表
// authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取
// authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取
// authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改
// authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建
// authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建
// authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除
// authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名
// authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录
// authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据
// authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务
// authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表
// authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫
// authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置
// authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置
// authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items
// authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items
// authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines
// authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
// authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步
// authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置
// authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务
// authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫
// authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目
// }
// // 可配置爬虫
// {
// authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置
// authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置
// authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫
// authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫
// authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫
// authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
// authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表
// }
// // 任务
// {
// authGroup.GET("/tasks", routes.GetTaskList) // 任务列表
// authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情
// authGroup.PUT("/tasks", routes.PutTask) // 派发任务
// authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务
// authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务
// authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务
// //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务
// authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务
// authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志
// authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志
// authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果
// authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
// authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务
// authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务
// authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务
// }
// // 系统任务/脚本
// {
// authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务
// authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表
// }
// // 定时任务
// {
// authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表
// authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情
// authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务
// authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务
// authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务
// authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务
// authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务
// authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务
// authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务
// authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态
// }
// // 用户
// {
// authGroup.GET("/users", routes.GetUserList) // 用户列表
// authGroup.GET("/users/:id", routes.GetUser) // 用户详情
// authGroup.POST("/users/:id", routes.PostUser) // 更改用户
// authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
// authGroup.PUT("/users-add", routes.PutUser) // 添加用户
// authGroup.GET("/me", routes.GetMe) // 获取自己账户
// authGroup.POST("/me", routes.PostMe) // 修改自己账户
// authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码
// }
// // 系统
// {
// authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
// authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
// }
// // 全局变量
// {
// authGroup.GET("/variables", routes.GetVariableList) // 列表
// authGroup.PUT("/variable", routes.PutVariable) // 新增
// authGroup.POST("/variable/:id", routes.PostVariable) // 修改
// authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除
// }
// // 项目
// {
// authGroup.GET("/projects", routes.GetProjectList) // 列表
// authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签
// authGroup.PUT("/projects", routes.PutProject) // 修改
// authGroup.POST("/projects/:id", routes.PostProject) // 新增
// authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除
// }
// // 操作
// {
// //authGroup.GET("/actions", routes.GetActionList) // 操作列表
// //authGroup.GET("/actions/:id", routes.GetAction) // 操作
// authGroup.PUT("/actions", routes.PutAction) // 新增操作
// //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作
// }
// // API Token
// {
// authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens
// authGroup.PUT("/tokens", routes.PutToken) // 添加 Token
// authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token
// }
// // 统计数据
// authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
// // 文件
// authGroup.GET("/file", routes.GetFile) // 获取文件
// // Git
// authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支
// authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
// authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits
// authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits
// }
//}
// 路由ping
//app.GET("/ping", routes.Ping)
// 运行服务器
}