diff --git a/backend/apps/api.go b/backend/apps/api.go index 11b1b2b2..f9f7eacd 100644 --- a/backend/apps/api.go +++ b/backend/apps/api.go @@ -2,7 +2,6 @@ package apps import ( "context" - "fmt" "github.com/apex/log" "github.com/crawlab-team/crawlab-core/config" "github.com/crawlab-team/crawlab-core/controllers" @@ -11,7 +10,6 @@ import ( "github.com/crawlab-team/crawlab-core/routes" "github.com/crawlab-team/crawlab-db/mongo" "github.com/crawlab-team/crawlab-db/redis" - "github.com/crawlab-team/go-trace" "github.com/gin-gonic/gin" "github.com/spf13/viper" "net" @@ -23,6 +21,7 @@ import ( ) type Api struct { + BaseApp app *gin.Engine } @@ -82,16 +81,6 @@ func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) ( }) } -func (app *Api) initModule(name string, fn func() error) (err error) { - if err := fn(); err != nil { - log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) - _ = trace.TraceError(err) - panic(err) - } - log.Info(fmt.Sprintf("initialized %s successfully", name)) - return nil -} - func NewApi() *Api { app := gin.New() return &Api{ diff --git a/backend/apps/base.go b/backend/apps/base.go index 2eb759f4..b07ee94f 100644 --- a/backend/apps/base.go +++ b/backend/apps/base.go @@ -1,6 +1,33 @@ package apps +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/go-trace" +) + type App interface { Init() Run() } + +type BaseApp struct { +} + +func (app *BaseApp) Init() { + panic("implement me") +} + +func (app *BaseApp) Run() { + panic("implement me") +} + +func (app *BaseApp) initModule(name string, fn func() error) (err error) { + if err := fn(); err != nil { + log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) + _ = trace.TraceError(err) + panic(err) + } + log.Info(fmt.Sprintf("initialized %s successfully", name)) + return nil +} diff --git a/backend/apps/handler.go b/backend/apps/handler.go new file mode 100644 index 00000000..dacb32c4 --- /dev/null +++ b/backend/apps/handler.go @@ -0,0 +1,21 @@ +package apps + +import ( + "github.com/crawlab-team/crawlab-core/services" +) + +type Handler struct { + BaseApp +} + +func (app *Handler) Init() { + _ = app.initModule("task-service", services.InitTaskService) +} + +func (app *Handler) Run() { + panic("implement me") +} + +func NewHandler() *Handler { + return &Handler{} +} diff --git a/backend/cmd/api.go b/backend/cmd/api.go new file mode 100644 index 00000000..fd265273 --- /dev/null +++ b/backend/cmd/api.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "crawlab/apps" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(apiCmd) +} + +var apiCmd = &cobra.Command{ + Use: "api", + Short: "Start API server", + Long: `Start API server of Crawlab which serves data to frontend`, + Run: func(cmd *cobra.Command, args []string) { + api := apps.NewApi() + api.Init() + api.Run() + }, +} diff --git a/backend/cmd/handler.go b/backend/cmd/handler.go new file mode 100644 index 00000000..747880a6 --- /dev/null +++ b/backend/cmd/handler.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "crawlab/apps" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(handlerCmd) +} + +var handlerCmd = &cobra.Command{ + Use: "handler", + Short: "Start API server", + Long: `Start API server of Crawlab which serves data to frontend`, + Run: func(cmd *cobra.Command, args []string) { + handler := apps.NewHandler() + handler.Init() + handler.Run() + }, +} diff --git a/backend/cmd/root.go b/backend/cmd/root.go new file mode 100644 index 00000000..6b2eb743 --- /dev/null +++ b/backend/cmd/root.go @@ -0,0 +1,60 @@ +package cmd + +import ( + "fmt" + + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var ( + // Used for flags. + cfgFile string + + rootCmd = &cobra.Command{ + Use: "crawlab", + Short: "CLI tool for Crawlab", + Long: `The CLI tool is for controlling against Crawlab. +Crawlab is a distributed web crawler and task admin platform +aimed at making web crawling and task management easier. +`, + } +) + +// Execute executes the root command. +func Execute() error { + return rootCmd.Execute() +} + +func init() { + cobra.OnInitialize(initConfig) + + //rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cobra.yaml)") + //rootCmd.PersistentFlags().StringP("author", "a", "YOUR NAME", "author name for copyright attribution") + //rootCmd.PersistentFlags().StringVarP(&userLicense, "license", "l", "", "name of license for the project") + //rootCmd.PersistentFlags().Bool("viper", true, "use Viper for configuration") + //viper.BindPFlag("author", rootCmd.PersistentFlags().Lookup("author")) + //viper.BindPFlag("useViper", rootCmd.PersistentFlags().Lookup("viper")) +} + +func initConfig() { + if cfgFile != "" { + // Use config file from the flag. + viper.SetConfigFile(cfgFile) + } else { + // Find home directory. + home, err := homedir.Dir() + cobra.CheckErr(err) + + // Search config in home directory with name ".cobra" (without extension). + viper.AddConfigPath(home) + viper.SetConfigName(".cobra") + } + + viper.AutomaticEnv() + + if err := viper.ReadInConfig(); err == nil { + fmt.Println("Using config file:", viper.ConfigFileUsed()) + } +} diff --git a/backend/go.mod b/backend/go.mod index f70efcd7..64e7f858 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -19,7 +19,7 @@ require ( github.com/crawlab-team/crawlab-db v0.0.2 github.com/crawlab-team/go-trace v0.0.0 github.com/gin-gonic/gin v1.6.3 - github.com/go-playground/validator/v10 v10.3.0 - github.com/olivere/elastic/v7 v7.0.15 + github.com/mitchellh/go-homedir v1.1.0 + github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 ) diff --git a/backend/go.sum b/backend/go.sum index 776c8761..236692df 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -19,9 +19,9 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw= -github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= @@ -63,9 +63,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/crawlab-team/crawlab-fs v0.0.0 h1:RGPWKB7ORgGEYLS5uplS/dIbdY8ktKmfBHM2MxxwQKQ= -github.com/crawlab-team/crawlab-fs v0.0.0/go.mod h1:k2VXprQspLAmbgO5sSpqMjg/xP4iKDkW4RyTWY8eTZM= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -79,8 +77,10 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fasthttp/websocket v1.4.2 h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM= github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -90,6 +90,7 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/gavv/httpexpect/v2 v2.2.0 h1:0VwaEBmQaNFHX9x591A8Up+8shCwdF/nF0qlRd/nI48= github.com/gavv/httpexpect/v2 v2.2.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -98,12 +99,11 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM= github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M= github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI= github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs= @@ -111,6 +111,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -187,7 +188,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -203,6 +206,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -229,16 +233,17 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E= github.com/imkira/go-interpol v1.0.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= @@ -249,6 +254,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= @@ -263,6 +269,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= @@ -282,17 +289,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/linxGnu/goseaweedfs v0.1.5 h1:7dChPdq8+fsPH0yqxKEofPhiosaar4LWePm4M+1Taz0= -github.com/linxGnu/goseaweedfs v0.1.5/go.mod h1:Zwe/7H7FJaPQyMTNKXgv6fhVDw6qi34MMJQp1K0VLNc= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k= @@ -307,6 +312,7 @@ github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFD github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -340,6 +346,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= @@ -347,12 +354,13 @@ github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXW github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= @@ -382,9 +390,11 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY= github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY= github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -414,19 +424,21 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -437,6 +449,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= @@ -450,7 +463,9 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= @@ -460,13 +475,20 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y= github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= @@ -492,8 +514,6 @@ golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -531,8 +551,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -571,14 +589,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg= golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -611,8 +625,6 @@ golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -669,8 +681,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= @@ -679,12 +693,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA= gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -697,6 +706,8 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -705,5 +716,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/backend/main.go b/backend/main.go index c514dc67..2c48caa1 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1,252 +1,9 @@ package main -import "crawlab/apps" +import ( + "crawlab/cmd" +) func main() { - api := apps.NewApi() - api.Init() - api.Run() - - //if model.IsMaster() { - // // 初始化定时任务 - // if err := services.InitScheduler(); err != nil { - // log.Error("init scheduler error:" + err.Error()) - // debug.PrintStack() - // panic(err) - // } - // log.Info("initialized schedule successfully") - // - // // 初始化用户服务 - // if err := services.InitUserService(); err != nil { - // log.Error("init user service error:" + err.Error()) - // debug.PrintStack() - // panic(err) - // } - // log.Info("initialized user service successfully") - // - // // 初始化依赖服务 - // if err := services.InitDepsFetcher(); err != nil { - // log.Error("init dependency fetcher error:" + err.Error()) - // debug.PrintStack() - // panic(err) - // } - // log.Info("initialized dependency fetcher successfully") - // - // // 初始化清理服务 - // if err := services.InitCleanService(); err != nil { - // log.Error("init clean service error:" + err.Error()) - // debug.PrintStack() - // panic(err) - // } - // log.Info("initialized clean service successfully") - //} - // - //// 初始化任务执行器 - //if err := services.InitTaskExecutor(); err != nil { - // log.Error("init task executor error:" + err.Error()) - // debug.PrintStack() - // panic(err) - //} - //log.Info("initialized task executor successfully") - // - //// 初始化爬虫服务 - //if err := services.InitSpiderService(); err != nil { - // log.Error("init spider service error:" + err.Error()) - // debug.PrintStack() - // panic(err) - //} - //log.Info("initialized spider service successfully") - // - //// 初始化RPC服务 - //if err := rpc.InitRpcService(); err != nil { - // log.Error("init rpc service error:" + err.Error()) - // debug.PrintStack() - // panic(err) - //} - //log.Info("initialized rpc service successfully") - - // 以下为主节点服务 - //if model.IsMaster() { - // // 中间件 - // esClientStr := viper.GetString("setting.esClient") - // if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" { - // ctx := context.Background() - // esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false)) - // if err != nil { - // log.Error("Init es client Error:" + err.Error()) - // } - // app.Use(middlewares.EsLog(ctx, esClient)) - // } - // app.Use(middlewares.CORSMiddleware()) - // anonymousGroup := app.Group("/") - // { - // anonymousGroup.POST("/login", routes.Login) // 用户登录 - // anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 - // anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 - // // release版本 - // anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 - // anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 - // // 文档 - // anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据 - // } - // authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) - // { - // // 节点 - // { - // authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 - // authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 - // authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 - // authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - // authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - // authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 - // authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 - // authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 - // authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 - // authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 - // authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 - // authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 - // } - // // 爬虫 - // { - // authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - // authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - // authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - // authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - // authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - // authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - // authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - // authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫 - // authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - // authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫 - // authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - // authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - // authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - // authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - // authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - // authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - // authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - // authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - // authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - // authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - // authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 - // authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 - // authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 - // authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 - // authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 - // authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items - // authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items - // authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines - // authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines - // authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 - // authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 - // authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 - // authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫 - // authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目 - // } - // // 可配置爬虫 - // { - // authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 - // authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 - // authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 - // authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 - // authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 - // authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 - // authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 - // } - // // 任务 - // { - // authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 - // authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 - // authGroup.PUT("/tasks", routes.PutTask) // 派发任务 - // authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务 - // authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 - // authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务 - // //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务 - // authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 - // authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 - // authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志 - // authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 - // authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 - // authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务 - // authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务 - // authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务 - // } - // // 系统任务/脚本 - // { - // authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务 - // authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表 - // } - // // 定时任务 - // { - // authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 - // authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 - // authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 - // authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务 - // authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 - // authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 - // authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务 - // authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 - // authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 - // authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态 - // } - // // 用户 - // { - // authGroup.GET("/users", routes.GetUserList) // 用户列表 - // authGroup.GET("/users/:id", routes.GetUser) // 用户详情 - // authGroup.POST("/users/:id", routes.PostUser) // 更改用户 - // authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 - // authGroup.PUT("/users-add", routes.PutUser) // 添加用户 - // authGroup.GET("/me", routes.GetMe) // 获取自己账户 - // authGroup.POST("/me", routes.PostMe) // 修改自己账户 - // authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码 - // } - // // 系统 - // { - // authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 - // authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON - // } - // // 全局变量 - // { - // authGroup.GET("/variables", routes.GetVariableList) // 列表 - // authGroup.PUT("/variable", routes.PutVariable) // 新增 - // authGroup.POST("/variable/:id", routes.PostVariable) // 修改 - // authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除 - // } - // // 项目 - // { - // authGroup.GET("/projects", routes.GetProjectList) // 列表 - // authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 - // authGroup.PUT("/projects", routes.PutProject) // 修改 - // authGroup.POST("/projects/:id", routes.PostProject) // 新增 - // authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 - // } - // // 操作 - // { - // //authGroup.GET("/actions", routes.GetActionList) // 操作列表 - // //authGroup.GET("/actions/:id", routes.GetAction) // 操作 - // authGroup.PUT("/actions", routes.PutAction) // 新增操作 - // //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 - // } - // // API Token - // { - // authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens - // authGroup.PUT("/tokens", routes.PutToken) // 添加 Token - // authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token - // } - // // 统计数据 - // authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 - // // 文件 - // authGroup.GET("/file", routes.GetFile) // 获取文件 - // // Git - // authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 - // authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 - // authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits - // authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits - // } - //} - - // 路由ping - //app.GET("/ping", routes.Ping) - - // 运行服务器 + _ = cmd.Execute() } diff --git a/backend/main_legacy.go b/backend/main_legacy.go new file mode 100644 index 00000000..c514dc67 --- /dev/null +++ b/backend/main_legacy.go @@ -0,0 +1,252 @@ +package main + +import "crawlab/apps" + +func main() { + api := apps.NewApi() + api.Init() + api.Run() + + //if model.IsMaster() { + // // 初始化定时任务 + // if err := services.InitScheduler(); err != nil { + // log.Error("init scheduler error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized schedule successfully") + // + // // 初始化用户服务 + // if err := services.InitUserService(); err != nil { + // log.Error("init user service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized user service successfully") + // + // // 初始化依赖服务 + // if err := services.InitDepsFetcher(); err != nil { + // log.Error("init dependency fetcher error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized dependency fetcher successfully") + // + // // 初始化清理服务 + // if err := services.InitCleanService(); err != nil { + // log.Error("init clean service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized clean service successfully") + //} + // + //// 初始化任务执行器 + //if err := services.InitTaskExecutor(); err != nil { + // log.Error("init task executor error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized task executor successfully") + // + //// 初始化爬虫服务 + //if err := services.InitSpiderService(); err != nil { + // log.Error("init spider service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized spider service successfully") + // + //// 初始化RPC服务 + //if err := rpc.InitRpcService(); err != nil { + // log.Error("init rpc service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized rpc service successfully") + + // 以下为主节点服务 + //if model.IsMaster() { + // // 中间件 + // esClientStr := viper.GetString("setting.esClient") + // if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" { + // ctx := context.Background() + // esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false)) + // if err != nil { + // log.Error("Init es client Error:" + err.Error()) + // } + // app.Use(middlewares.EsLog(ctx, esClient)) + // } + // app.Use(middlewares.CORSMiddleware()) + // anonymousGroup := app.Group("/") + // { + // anonymousGroup.POST("/login", routes.Login) // 用户登录 + // anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 + // anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 + // // release版本 + // anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + // anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 + // // 文档 + // anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据 + // } + // authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) + // { + // // 节点 + // { + // authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + // authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + // authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + // authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + // authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + // authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + // authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 + // authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + // authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 + // authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 + // authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 + // authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 + // } + // // 爬虫 + // { + // authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + // authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + // authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + // authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + // authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + // authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + // authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + // authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫 + // authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + // authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫 + // authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + // authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + // authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + // authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + // authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + // authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + // authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + // authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + // authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + // authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + // authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + // authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 + // authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 + // authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 + // authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 + // authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items + // authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items + // authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines + // authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines + // authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 + // authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 + // authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 + // authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫 + // authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目 + // } + // // 可配置爬虫 + // { + // authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 + // authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 + // authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 + // authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 + // authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 + // authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 + // authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 + // } + // // 任务 + // { + // authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 + // authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 + // authGroup.PUT("/tasks", routes.PutTask) // 派发任务 + // authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务 + // authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 + // authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务 + // //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务 + // authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 + // authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 + // authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志 + // authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 + // authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 + // authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务 + // authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务 + // authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务 + // } + // // 系统任务/脚本 + // { + // authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务 + // authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表 + // } + // // 定时任务 + // { + // authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 + // authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 + // authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 + // authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务 + // authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 + // authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 + // authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务 + // authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 + // authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 + // authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态 + // } + // // 用户 + // { + // authGroup.GET("/users", routes.GetUserList) // 用户列表 + // authGroup.GET("/users/:id", routes.GetUser) // 用户详情 + // authGroup.POST("/users/:id", routes.PostUser) // 更改用户 + // authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + // authGroup.PUT("/users-add", routes.PutUser) // 添加用户 + // authGroup.GET("/me", routes.GetMe) // 获取自己账户 + // authGroup.POST("/me", routes.PostMe) // 修改自己账户 + // authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码 + // } + // // 系统 + // { + // authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 + // authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON + // } + // // 全局变量 + // { + // authGroup.GET("/variables", routes.GetVariableList) // 列表 + // authGroup.PUT("/variable", routes.PutVariable) // 新增 + // authGroup.POST("/variable/:id", routes.PostVariable) // 修改 + // authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除 + // } + // // 项目 + // { + // authGroup.GET("/projects", routes.GetProjectList) // 列表 + // authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 + // authGroup.PUT("/projects", routes.PutProject) // 修改 + // authGroup.POST("/projects/:id", routes.PostProject) // 新增 + // authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 + // } + // // 操作 + // { + // //authGroup.GET("/actions", routes.GetActionList) // 操作列表 + // //authGroup.GET("/actions/:id", routes.GetAction) // 操作 + // authGroup.PUT("/actions", routes.PutAction) // 新增操作 + // //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 + // } + // // API Token + // { + // authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens + // authGroup.PUT("/tokens", routes.PutToken) // 添加 Token + // authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token + // } + // // 统计数据 + // authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 + // // 文件 + // authGroup.GET("/file", routes.GetFile) // 获取文件 + // // Git + // authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 + // authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 + // authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits + // authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits + // } + //} + + // 路由ping + //app.GET("/ping", routes.Ping) + + // 运行服务器 +}