Merge branch 'next' into develop

.gitmodules

@@ -0,0 +1,15 @@
+[submodule "backend/core"]
+	path = backend/core
+	url = https://github.com/crawlab-team/crawlab-core
+[submodule "backend/db"]
+	path = backend/db
+	url = https://github.com/crawlab-team/crawlab-db
+[submodule "backend/fs"]
+	path = backend/fs
+	url = https://github.com/crawlab-team/crawlab-fs
+[submodule "backend/vcs"]
+	path = backend/vcs
+	url = https://github.com/crawlab-team/crawlab-vcs
+[submodule "backend/log"]
+	path = backend/log
+	url = https://github.com/crawlab-team/crawlab-log

backend/apps/api.go

@@ -0,0 +1,83 @@
+package apps
+
+import (
+	"context"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-core/controllers"
+	"github.com/crawlab-team/crawlab-core/interfaces"
+	"github.com/crawlab-team/crawlab-core/middlewares"
+	"github.com/crawlab-team/crawlab-core/routes"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net"
+	"net/http"
+	"time"
+)
+
+type Api struct {
+	// dependencies
+	interfaces.WithConfigPath
+
+	// internals
+	app *gin.Engine
+	srv *http.Server
+}
+
+func (app *Api) Init() {
+	// initialize mongo
+	_ = initModule("mongo", mongo.InitMongo)
+
+	// initialize controllers
+	_ = initModule("controllers", controllers.InitControllers)
+
+	// initialize middlewares
+	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
+
+	// initialize routes
+	_ = app.initModuleWithApp("routes", routes.InitRoutes)
+}
+
+func (app *Api) Start() {
+	host := viper.GetString("server.host")
+	port := viper.GetString("server.port")
+	address := net.JoinHostPort(host, port)
+	app.srv = &http.Server{
+		Handler: app.app,
+		Addr:    address,
+	}
+	if err := app.srv.ListenAndServe(); err != nil {
+		if err != http.ErrServerClosed {
+			log.Error("run server error:" + err.Error())
+		} else {
+			log.Info("server graceful down")
+		}
+	}
+}
+
+func (app *Api) Wait() {
+	DefaultWait()
+}
+
+func (app *Api) Stop() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := app.srv.Shutdown(ctx); err != nil {
+		log.Error("run server error:" + err.Error())
+	}
+}
+
+func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
+	return initModule(name, func() error {
+		return fn(app.app)
+	})
+}
+
+func NewApi() *Api {
+	api := &Api{
+		app: gin.New(),
+	}
+	api.Init()
+	return api
+}

backend/apps/handler.go

@@ -0,0 +1,26 @@
+package apps
+
+import (
+	"github.com/apex/log"
+)
+
+type Handler struct {
+}
+
+func (app *Handler) Init() {
+}
+
+func (app *Handler) Start() {
+}
+
+func (app *Handler) Wait() {
+	DefaultWait()
+}
+
+func (app *Handler) Stop() {
+	log.Info("handler has stopped")
+}
+
+func NewHandler() *Handler {
+	return &Handler{}
+}

backend/apps/interfaces.go

@@ -0,0 +1,25 @@
+package apps
+
+import "github.com/crawlab-team/crawlab-core/interfaces"
+
+type App interface {
+	Init()
+	Start()
+	Wait()
+	Stop()
+}
+
+type NodeApp interface {
+	App
+	interfaces.WithConfigPath
+	SetGrpcAddress(address interfaces.Address)
+}
+
+type MasterApp interface {
+	NodeApp
+	SetRunOnMaster(ok bool)
+}
+
+type WorkerApp interface {
+	NodeApp
+}

backend/apps/master.go

@@ -0,0 +1,82 @@
+package apps
+
+import (
+	"github.com/crawlab-team/crawlab-core/config"
+	"github.com/crawlab-team/crawlab-core/interfaces"
+	"github.com/crawlab-team/crawlab-core/node/service"
+	"go.uber.org/dig"
+)
+
+type Master struct {
+	// settings
+	runOnMaster bool
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+	masterSvc interfaces.NodeMasterService
+
+	// modules
+	api *Api
+
+	// internals
+	quit chan int
+}
+
+func (app *Master) SetGrpcAddress(address interfaces.Address) {
+	app.grpcAddress = address
+}
+
+func (app *Master) SetRunOnMaster(ok bool) {
+	app.runOnMaster = ok
+}
+
+func (app *Master) Init() {
+}
+
+func (app *Master) Start() {
+	go app.api.Start()
+	go app.masterSvc.Start()
+}
+
+func (app *Master) Wait() {
+	<-app.quit
+}
+
+func (app *Master) Stop() {
+	app.api.Stop()
+	app.quit <- 1
+}
+
+func NewMaster(opts ...MasterOption) (app MasterApp) {
+	// master
+	m := &Master{
+		WithConfigPath: config.NewConfigPathService(),
+		api:            NewApi(),
+		quit:           make(chan int, 1),
+	}
+
+	// apply options
+	for _, opt := range opts {
+		opt(m)
+	}
+
+	// service options
+	var svcOpts []service.Option
+	if m.grpcAddress != nil {
+		svcOpts = append(svcOpts, service.WithAddress(m.grpcAddress))
+	}
+
+	// dependency injection
+	c := dig.New()
+	if err := c.Provide(service.ProvideMasterService(m.GetConfigPath(), svcOpts...)); err != nil {
+		panic(err)
+	}
+	if err := c.Invoke(func(masterSvc interfaces.NodeMasterService) {
+		m.masterSvc = masterSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return m
+}

backend/apps/options.go

@@ -0,0 +1,37 @@
+package apps
+
+import "github.com/crawlab-team/crawlab-core/interfaces"
+
+type MasterOption func(app MasterApp)
+
+func WithMasterConfigPath(path string) MasterOption {
+	return func(app MasterApp) {
+		app.SetConfigPath(path)
+	}
+}
+
+func WithMasterGrpcAddress(address interfaces.Address) MasterOption {
+	return func(app MasterApp) {
+		app.SetGrpcAddress(address)
+	}
+}
+
+func WithRunOnMaster(ok bool) MasterOption {
+	return func(app MasterApp) {
+		app.SetRunOnMaster(ok)
+	}
+}
+
+type WorkerOption func(app WorkerApp)
+
+func WithWorkerConfigPath(path string) WorkerOption {
+	return func(app WorkerApp) {
+		app.SetConfigPath(path)
+	}
+}
+
+func WithWorkerGrpcAddress(address interfaces.Address) WorkerOption {
+	return func(app WorkerApp) {
+		app.SetGrpcAddress(address)
+	}
+}

backend/apps/scheduler.go

@@ -0,0 +1,27 @@
+package apps
+
+import (
+	"github.com/apex/log"
+)
+
+type Scheduler struct {
+}
+
+func (app *Scheduler) Init() {
+}
+
+func (app *Scheduler) Start() {
+	log.Info("scheduler has started")
+}
+
+func (app *Scheduler) Wait() {
+	DefaultWait()
+}
+
+func (app *Scheduler) Stop() {
+	log.Info("scheduler has stopped")
+}
+
+func NewScheduler() *Scheduler {
+	return &Scheduler{}
+}

backend/apps/utils.go

@@ -0,0 +1,40 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-core/utils"
+	"github.com/crawlab-team/go-trace"
+)
+
+func Start(app App) {
+	start(app)
+}
+
+func start(app App) {
+	app.Init()
+	go app.Start()
+	app.Wait()
+	app.Stop()
+}
+
+func DefaultWait() {
+	utils.DefaultWait()
+}
+
+func initModule(name string, fn func() error) (err error) {
+	if err := fn(); err != nil {
+		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
+		_ = trace.TraceError(err)
+		panic(err)
+	}
+	log.Info(fmt.Sprintf("initialized %s successfully", name))
+	return nil
+}
+
+func initApp(name string, app App) {
+	_ = initModule(name, func() error {
+		app.Init()
+		return nil
+	})
+}

backend/apps/worker.go

@@ -0,0 +1,71 @@
+package apps
+
+import (
+	"github.com/crawlab-team/crawlab-core/config"
+	"github.com/crawlab-team/crawlab-core/interfaces"
+	"github.com/crawlab-team/crawlab-core/node/service"
+	"go.uber.org/dig"
+)
+
+type Worker struct {
+	// settings
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+	workerSvc interfaces.NodeWorkerService
+
+	// internals
+	quit chan int
+}
+
+func (app *Worker) SetGrpcAddress(address interfaces.Address) {
+	app.grpcAddress = address
+}
+
+func (app *Worker) Init() {
+}
+
+func (app *Worker) Start() {
+	go app.workerSvc.Start()
+}
+
+func (app *Worker) Wait() {
+	<-app.quit
+}
+
+func (app *Worker) Stop() {
+	app.quit <- 1
+}
+
+func NewWorker(opts ...WorkerOption) (app *Worker) {
+	// worker
+	w := &Worker{
+		WithConfigPath: config.NewConfigPathService(),
+		quit:           make(chan int, 1),
+	}
+
+	// apply options
+	for _, opt := range opts {
+		opt(w)
+	}
+
+	// service options
+	var svcOpts []service.Option
+	if w.grpcAddress != nil {
+		svcOpts = append(svcOpts, service.WithAddress(w.grpcAddress))
+	}
+
+	// dependency injection
+	c := dig.New()
+	if err := c.Provide(service.ProvideWorkerService(w.GetConfigPath(), svcOpts...)); err != nil {
+		panic(err)
+	}
+	if err := c.Invoke(func(workerSvc interfaces.NodeWorkerService) {
+		w.workerSvc = workerSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return w
+}

backend/cmd/api.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(apiCmd)
+}
+
+var apiCmd = &cobra.Command{
+	Use:     "api",
+	Aliases: []string{"A"},
+	Short:   "Start API server",
+	Long:    `Start API server of Crawlab which serves data to frontend`,
+	Run: func(cmd *cobra.Command, args []string) {
+		api := apps.NewApi()
+		apps.Start(api)
+	},
+}

backend/cmd/handler.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(handlerCmd)
+}
+
+var handlerCmd = &cobra.Command{
+	Use:     "handler",
+	Aliases: []string{"H"},
+	Short:   "Start handler",
+	Long: `Start a handler instance of Crawlab 
+which runs tasks with given parameters`,
+	Run: func(cmd *cobra.Command, args []string) {
+		handler := apps.NewHandler()
+		apps.Start(handler)
+	},
+}

backend/cmd/master.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"fmt"
+	"github.com/crawlab-team/crawlab-core/entity"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	runOnMaster       bool
+	masterConfigPath  string
+	masterGrpcAddress string
+	masterGrpcAuthKey string
+)
+
+func init() {
+	rootCmd.AddCommand(masterCmd)
+
+	masterCmd.PersistentFlags().StringVarP(&masterConfigPath, "config-path", "c", "", "Config path of master node")
+	_ = viper.BindPFlag("configPath", masterCmd.PersistentFlags().Lookup("configPath"))
+
+	masterCmd.PersistentFlags().StringVarP(&masterGrpcAddress, "grpc-address", "g", "", "gRPC address of master node")
+	_ = viper.BindPFlag("grpcAddress", masterCmd.PersistentFlags().Lookup("grpcAddress"))
+
+	masterCmd.PersistentFlags().StringVarP(&masterGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of master node")
+	_ = viper.BindPFlag("grpcAuthKey", masterCmd.PersistentFlags().Lookup("grpcAuthKey"))
+}
+
+var masterCmd = &cobra.Command{
+	Use:     "master",
+	Aliases: []string{"M"},
+	Short:   "Start master",
+	Long: `Start a master instance of Crawlab 
+which runs api and assign tasks to worker nodes`,
+	Run: func(cmd *cobra.Command, args []string) {
+		// options
+		var opts []apps.MasterOption
+		if masterConfigPath != "" {
+			opts = append(opts, apps.WithMasterConfigPath(masterConfigPath))
+			viper.Set("config.path", masterConfigPath)
+		}
+		opts = append(opts, apps.WithRunOnMaster(runOnMaster))
+		if masterGrpcAddress != "" {
+			address, err := entity.NewAddressFromString(masterGrpcAddress)
+			if err != nil {
+				fmt.Println(fmt.Sprintf("invalid grpc-address: %s", masterGrpcAddress))
+			}
+			opts = append(opts, apps.WithMasterGrpcAddress(address))
+			viper.Set("grpc.address", masterGrpcAddress)
+			viper.Set("grpc.server.address", masterGrpcAddress)
+		}
+		if masterGrpcAuthKey != "" {
+			viper.Set("grpc.authKey", masterGrpcAuthKey)
+		}
+
+		// app
+		master := apps.NewMaster(opts...)
+
+		// start
+		apps.Start(master)
+	},
+}

backend/cmd/root.go

@@ -0,0 +1,53 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	// Used for flags.
+	cfgFile string
+
+	rootCmd = &cobra.Command{
+		Use:   "crawlab",
+		Short: "CLI tool for Crawlab",
+		Long: `The CLI tool is for controlling against Crawlab.
+Crawlab is a distributed web crawler and task admin platform
+aimed at making web crawling and task management easier.
+`,
+	}
+)
+
+// Execute executes the root command.
+func Execute() error {
+	return rootCmd.Execute()
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+}
+
+func initConfig() {
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Find home directory.
+		home, err := homedir.Dir()
+		cobra.CheckErr(err)
+
+		// Search config in home directory with name ".cobra" (without extension).
+		viper.AddConfigPath(home)
+		viper.SetConfigName(".cobra")
+	}
+
+	viper.AutomaticEnv()
+
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	}
+}

backend/cmd/scheduler.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(schedulerCmd)
+}
+
+var schedulerCmd = &cobra.Command{
+	Use:     "scheduler",
+	Aliases: []string{"S"},
+	Short:   "Start scheduler",
+	Long: `Start a scheduler instance of Crawlab 
+which assigns tasks to worker nodes to execute`,
+	Run: func(cmd *cobra.Command, args []string) {
+		scheduler := apps.NewScheduler()
+		apps.Start(scheduler)
+	},
+}

backend/cmd/worker.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"crawlab/apps"
+	"fmt"
+	"github.com/crawlab-team/crawlab-core/entity"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	workerConfigPath  string
+	workerGrpcAddress string
+	workerGrpcAuthKey string
+)
+
+func init() {
+	rootCmd.AddCommand(workerCmd)
+
+	workerCmd.PersistentFlags().StringVarP(&workerConfigPath, "config-path", "c", "", "Config path of worker node")
+	_ = viper.BindPFlag("configPath", workerCmd.PersistentFlags().Lookup("configPath"))
+
+	workerCmd.PersistentFlags().StringVarP(&workerGrpcAddress, "grpc-address", "g", "", "gRPC address of worker node")
+	_ = viper.BindPFlag("grpcAddress", workerCmd.PersistentFlags().Lookup("grpcAddress"))
+
+	workerCmd.PersistentFlags().StringVarP(&workerGrpcAuthKey, "grpc-auth-key", "a", "", "gRPC auth key of worker node")
+	_ = viper.BindPFlag("grpcAuthKey", workerCmd.PersistentFlags().Lookup("grpcAuthKey"))
+}
+
+var workerCmd = &cobra.Command{
+	Use:     "worker",
+	Aliases: []string{"W"},
+	Short:   "Start worker",
+	Long: `Start a worker instance of Crawlab 
+serving in the worker node and executes tasks
+assigned by the master node`,
+	Run: func(cmd *cobra.Command, args []string) {
+		// options
+		var opts []apps.WorkerOption
+		if workerConfigPath != "" {
+			opts = append(opts, apps.WithWorkerConfigPath(workerConfigPath))
+			viper.Set("config.path", workerConfigPath)
+		}
+		if workerGrpcAddress != "" {
+			address, err := entity.NewAddressFromString(workerGrpcAddress)
+			if err != nil {
+				fmt.Println(fmt.Sprintf("invalid grpc-address: %s", workerGrpcAddress))
+				return
+			}
+			opts = append(opts, apps.WithWorkerGrpcAddress(address))
+			viper.Set("grpc.address", workerGrpcAddress)
+		}
+		if workerGrpcAuthKey != "" {
+			viper.Set("grpc.authKey", workerGrpcAuthKey)
+		}
+
+		// app
+		master := apps.NewWorker(opts...)
+
+		// start
+		apps.Start(master)
+	},
+}

backend/conf/config.yml

@@ -1,5 +1,3 @@
-api:
-  address: "localhost:8000"
 mongo:
   host: localhost
   port: 27017
@@ -7,61 +5,59 @@ mongo:
   username: ""
   password: ""
   authSource: "admin"
-redis:
-  address: localhost
-  password: ""
-  database: 1
-  port: 6379
-log:
-  level: info
-  path: "/var/logs/crawlab"
-  isDeletePeriodically: "N"
-  deleteFrequency: "@hourly"
 server:
   host: 0.0.0.0
   port: 8000
-  master: "Y"
-  secret: "crawlab"
-  register:
-    # type 填 mac/ip/customName, 如果是ip，则需要手动指定IP, 如果是 customName, 需填写你的 customNodeName
-    type: "mac"
-    customNodeName: "" # 自定义节点名称, default node1,只有在type = customName 时生效
-    ip: ""
-  lang: # 安装语言环境, Y 为安装，N 为不安装
-    python: "Y"
-    node: "N"
-    java: "N"
-    dotnet: "N"
-    php: "N"
-  scripts: "/app/backend/scripts"
+#  master: true
+#  secret: "crawlab"
+#  register:
+#    # type 填 mac/ip/customName, 如果是ip，则需要手动指定IP, 如果是 customName, 需填写你的 customNodeName
+#    type: "mac"
+#    customNodeName: "" # 自定义节点名称, default node1,只有在type = customName 时生效
+#    ip: ""
+#  lang: # 安装语言环境, Y 为安装，N 为不安装
+#    python: "Y"
+#    node: "N"
+#    java: "N"
+#    dotnet: "N"
+#    php: "N"
+#  scripts: "/app/backend/scripts"
 spider:
-  path: "/app/spiders"
+  fs: "/spiders"
+  workspace: "/workspace"
+  repo: "/repo"
 task:
   workers: 16
-other:
-  tmppath: "/tmp"
-version: 0.5.1
-setting:
-  crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient
-  crawlabLogIndex: "crawlab-log"
-  allowRegister: "N"
-  enableTutorial: "N"
-  runOnMaster: "Y"
-  demoSpiders: "N"
-  checkScrapy: "Y"
-  autoInstall: "Y"
-  esClient: "" # Your ES client, for example, http://192.168.1.1:9200 or http://your-domain.com, if not use es, set empty
-  spiderLogIndex: "spider-log" # Index pattern for kibana, need to config on kibana
-notification:
-  mail:
-    server: ''
-    port: ''
-    senderEmail: ''
-    senderIdentity: ''
-    smtp:
-      user: ''
-      password: ''
-repo:
-  apiUrl: "https://center.crawlab.cn/api"
-#  apiUrl: "http://localhost:8002"
-  ossUrl: "https://repo.crawlab.cn"
+  cancelWaitSeconds: 30
+# TODO: implement
+#setting:
+#  crawlabLogToES: "N" # Send crawlab runtime log to ES, open this option "Y", remember to set esClient
+#  crawlabLogIndex: "crawlab-log"
+#  allowRegister: "N"
+#  enableTutorial: "N"
+#  runOnMaster: "Y"
+#  demoSpiders: "N"
+#  checkScrapy: "Y"
+#  autoInstall: "Y"
+#  esClient: "" # Your ES client, for example, http://192.168.1.1:9200 or http://your-domain.com, if not use es, set empty
+#  spiderLogIndex: "spider-log" # Index pattern for kibana, need to config on kibana
+# TODO: implement
+#notification:
+#  mail:
+#    server: ''
+#    port: ''
+#    senderEmail: ''
+#    senderIdentity: ''
+#    smtp:
+#      user: ''
+#      password: ''
+grpc:
+  address: localhost:9666
+  server:
+    address: 0.0.0.0:9666
+  authKey: Crawlab2021!
+fs:
+  filer:
+    proxy: http://localhost:8888
+    url: http://localhost:8000/filer
+    authKey: Crawlab2021!

backend/config/config.go

@@ -1,57 +0,0 @@
-package config
-
-import (
-	"github.com/fsnotify/fsnotify"
-	"github.com/spf13/viper"
-	"log"
-	"strings"
-)
-
-type Config struct {
-	Name string
-}
-
-// 监控配置文件变化并热加载程序
-func (c *Config) WatchConfig() {
-	viper.WatchConfig()
-	viper.OnConfigChange(func(e fsnotify.Event) {
-		log.Printf("Config file changed: %s", e.Name)
-	})
-}
-
-func (c *Config) Init() error {
-	if c.Name != "" {
-		viper.SetConfigFile(c.Name) // 如果指定了配置文件，则解析指定的配置文件
-	} else {
-		viper.AddConfigPath("./conf") // 如果没有指定配置文件，则解析默认的配置文件
-		viper.SetConfigName("config")
-	}
-	viper.SetConfigType("yaml")   // 设置配置文件格式为YAML
-	viper.AutomaticEnv()          // 读取匹配的环境变量
-	viper.SetEnvPrefix("CRAWLAB") // 读取环境变量的前缀为CRAWLAB
-	replacer := strings.NewReplacer(".", "_")
-	viper.SetEnvKeyReplacer(replacer)
-	if err := viper.ReadInConfig(); err != nil { // viper解析配置文件
-		return err
-	}
-
-	return nil
-}
-
-func InitConfig(cfg string) error {
-	c := Config{
-		Name: cfg,
-	}
-
-	// 初始化配置文件
-	if err := c.Init(); err != nil {
-		return err
-	}
-
-	// 监控配置文件变化并热加载程序
-	c.WatchConfig()
-
-	return nil
-}
-
-

backend/config/config_test.go

@@ -1,16 +0,0 @@
-package config
-
-import (
-	. "github.com/smartystreets/goconvey/convey"
-	"testing"
-)
-
-func TestInitConfig(t *testing.T) {
-	Convey("Test InitConfig func", t, func() {
-		x := InitConfig("../conf/config.yml")
-
-		Convey("The value should be nil", func() {
-			So(x, ShouldEqual, nil)
-		})
-	})
-}

backend/constants/action.go

@@ -1,8 +0,0 @@
-package constants
-
-const (
-	ActionTypeVisit          = "visit"
-	ActionTypeInstallDep     = "install_dep"
-	ActionTypeInstallLang    = "install_lang"
-	ActionTypeViewDisclaimer = "view_disclaimer"
-)

backend/constants/anchor.go

@@ -1,8 +0,0 @@
-package constants
-
-const (
-	AnchorStartStage = "START_STAGE"
-	AnchorStartUrl   = "START_URL"
-	AnchorItems      = "ITEMS"
-	AnchorParsers    = "PARSERS"
-)

backend/constants/auth.go

@@ -1,7 +0,0 @@
-package constants
-
-const (
-	OwnerTypeAll    = "all"
-	OwnerTypeMe     = "me"
-	OwnerTypePublic = "public"
-)

backend/constants/challenge.go

@@ -1,20 +0,0 @@
-package constants
-
-const (
-	ChallengeLogin7d                  = "login_7d"
-	ChallengeLogin30d                 = "login_30d"
-	ChallengeLogin90d                 = "login_90d"
-	ChallengeLogin180d                = "login_180d"
-	ChallengeCreateCustomizedSpider   = "create_customized_spider"
-	ChallengeCreateConfigurableSpider = "create_configurable_spider"
-	ChallengeCreateSchedule           = "create_schedule"
-	ChallengeCreateNodes              = "create_nodes"
-	ChallengeCreateUser         = "create_user"
-	ChallengeRunRandom                = "run_random"
-	ChallengeScrape1k                 = "scrape_1k"
-	ChallengeScrape10k                = "scrape_10k"
-	ChallengeScrape100k               = "scrape_100k"
-	ChallengeInstallDep               = "install_dep"
-	ChallengeInstallLang              = "install_lang"
-	ChallengeViewDisclaimer           = "view_disclaimer"
-)

backend/constants/channels.go

@@ -1,9 +0,0 @@
-package constants
-
-const (
-	ChannelAllNode = "nodes:public"
-
-	ChannelWorkerNode = "nodes:"
-
-	ChannelMasterNode = "nodes:master"
-)

backend/constants/common.go

@@ -1,6 +0,0 @@
-package constants
-
-const (
-	ASCENDING = "ascending"
-	DESCENDING = "descending"
-)

backend/constants/config_spider.go

@@ -1,6 +0,0 @@
-package constants
-
-const (
-	EngineScrapy = "scrapy"
-	EngineColly  = "colly"
-)

backend/constants/context.go

@@ -1,5 +0,0 @@
-package constants
-
-const (
-	ContextUser = "currentUser"
-)

backend/constants/errors.go

@@ -1,13 +0,0 @@
-package constants
-
-import (
-	"crawlab/errors"
-	"net/http"
-)
-
-var (
-	ErrorMongoError = errors.NewSystemOPError(1001, "system error:[mongo]%s", http.StatusInternalServerError)
-	//users
-	ErrorUserNotFound              = errors.NewBusinessError(10001, "user not found.", http.StatusUnauthorized)
-	ErrorUsernameOrPasswordInvalid = errors.NewBusinessError(11001, "username or password invalid", http.StatusUnauthorized)
-)

backend/constants/log.go

@@ -1,5 +0,0 @@
-package constants
-
-const (
-	ErrorRegexPattern = "(?:[ :,.]|^)((?:error|exception|traceback)s?)(?:[ :,.]|$)"
-)

backend/constants/message.go

@@ -1,9 +0,0 @@
-package constants
-
-const (
-	MsgTypeGetLog        = "get-log"
-	MsgTypeGetSystemInfo = "get-sys-info"
-	MsgTypeCancelTask    = "cancel-task"
-	MsgTypeRemoveLog     = "remove-log"
-	MsgTypeRemoveSpider  = "remove-spider"
-)

backend/constants/model.go

@@ -1,6 +0,0 @@
-package constants
-
-const (
-	ObjectIdNull = "000000000000000000000000"
-	Infinite     = 999999999
-)

backend/constants/node.go

@@ -1,6 +0,0 @@
-package constants
-
-const (
-	StatusOnline  = "online"
-	StatusOffline = "offline"
-)

backend/constants/notification.go

@@ -1,13 +0,0 @@
-package constants
-
-const (
-	NotificationTriggerOnTaskEnd   = "notification_trigger_on_task_end"
-	NotificationTriggerOnTaskError = "notification_trigger_on_task_error"
-	NotificationTriggerNever       = "notification_trigger_never"
-)
-
-const (
-	NotificationTypeMail     = "notification_type_mail"
-	NotificationTypeDingTalk = "notification_type_ding_talk"
-	NotificationTypeWechat   = "notification_type_wechat"
-)

backend/constants/register.go

@@ -1,8 +0,0 @@
-package constants
-
-const (
-	RegisterTypeMac      = "mac"
-	RegisterTypeIp       = "ip"
-	RegisterTypeHostname = "hostname"
-	RegisterTypeCustomName = "customName"
-)

backend/constants/rpc.go

@@ -1,12 +0,0 @@
-package constants
-
-const (
-	RpcInstallLang          = "install_lang"
-	RpcInstallDep           = "install_dep"
-	RpcUninstallDep         = "uninstall_dep"
-	RpcGetInstalledDepList  = "get_installed_dep_list"
-	RpcGetLang              = "get_lang"
-	RpcCancelTask           = "cancel_task"
-	RpcGetSystemInfoService = "get_system_info"
-	RpcRemoveSpider         = "remove_spider"
-)

backend/constants/schedule.go

@@ -1,10 +0,0 @@
-package constants
-
-const (
-	ScheduleStatusStop    = "stopped"
-	ScheduleStatusRunning = "running"
-	ScheduleStatusError   = "error"
-
-	ScheduleStatusErrorNotFoundNode   = "Not Found Node"
-	ScheduleStatusErrorNotFoundSpider = "Not Found Spider"
-)

backend/constants/scrapy.go

@@ -1,5 +0,0 @@
-package constants
-
-const ScrapyProtectedStageNames = ""
-
-const ScrapyProtectedFieldNames = "_id,task_id,ts"

backend/constants/spider.go

@@ -1,7 +0,0 @@
-package constants
-
-const (
-	Customized   = "customized"
-	Configurable = "configurable"
-	Plugin       = "plugin"
-)

backend/constants/system.go

@@ -1,25 +0,0 @@
-package constants
-
-const (
-	Windows = "windows"
-	Linux   = "linux"
-	Darwin  = "darwin"
-)
-
-const (
-	Python = "python"
-	Nodejs = "node"
-	Java   = "java"
-)
-
-const (
-	InstallStatusNotInstalled    = "not-installed"
-	InstallStatusInstalling      = "installing"
-	InstallStatusInstallingOther = "installing-other"
-	InstallStatusInstalled       = "installed"
-)
-
-const (
-	LangTypeLang      = "lang"
-	LangTypeWebDriver = "webdriver"
-)

backend/constants/task.go

@@ -1,32 +0,0 @@
-package constants
-
-const (
-	// 调度中
-	StatusPending string = "pending"
-	// 运行中
-	StatusRunning string = "running"
-	// 已完成
-	StatusFinished string = "finished"
-	// 错误
-	StatusError string = "error"
-	// 取消
-	StatusCancelled string = "cancelled"
-	// 节点重启导致的异常终止
-	StatusAbnormal string = "abnormal"
-)
-
-const (
-	TaskFinish string = "finish"
-	TaskCancel string = "cancel"
-)
-
-const (
-	RunTypeAllNodes      string = "all-nodes"
-	RunTypeRandom        string = "random"
-	RunTypeSelectedNodes string = "selected-nodes"
-)
-
-const (
-	TaskTypeSpider string = "spider"
-	TaskTypeSystem string = "system"
-)

backend/constants/user.go

@@ -1,6 +0,0 @@
-package constants
-
-const (
-	RoleAdmin  = "admin"
-	RoleNormal = "normal"
-)

backend/constants/variable.go

@@ -1,9 +0,0 @@
-package constants
-
-const (
-	String  = "string"
-	Number  = "number"
-	Boolean = "boolean"
-	Array   = "array"
-	Object  = "object"
-)

backend/data/challenge_data.json

@@ -1,142 +0,0 @@
-[
-  {
-    "name": "login_7d",
-    "title_cn": "连续登录 7 天",
-    "title_en": "Logged-in for 7 days",
-    "description_cn": "连续 7 天登录 Crawlab，即可完成挑战！",
-    "description_en": "Logged-in for consecutive 7 days to complete the challenge",
-    "difficulty": 1
-  },
-  {
-    "name": "login_30d",
-    "title_cn": "连续登录 30 天",
-    "title_en": "Logged-in for 30 days",
-    "description_cn": "连续 30 天登录 Crawlab，即可完成挑战！",
-    "description_en": "Logged-in for consecutive 30 days to complete the challenge",
-    "difficulty": 2
-  },
-  {
-    "name": "login_90d",
-    "title_cn": "连续登录 90 天",
-    "title_en": "Logged-in for 90 days",
-    "description_cn": "连续 90 天登录 Crawlab，即可完成挑战！",
-    "description_en": "Logged-in for consecutive 90 days to complete the challenge",
-    "difficulty": 3
-  },
-  {
-    "name": "login_180d",
-    "title_cn": "连续登录 180 天",
-    "title_en": "Logged-in for 180 days",
-    "description_cn": "连续 180 天登录 Crawlab，即可完成挑战！",
-    "description_en": "Logged-in for consecutive 180 days to complete the challenge",
-    "difficulty": 4
-  },
-  {
-    "name": "create_customized_spider",
-    "title_cn": "创建 1 个自定义爬虫",
-    "title_en": "Create a customized spider",
-    "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '自定义爬虫'，输入相应的参数，点击添加，即可完成挑战！",
-    "description_en": "In Spider List page, click 'Add Spider', select 'Customized Spider', enter params, click 'Add' to finish the challenge.",
-    "difficulty": 1,
-    "path": "/spiders"
-  },
-  {
-    "name": "create_configurable_spider",
-    "title_cn": "创建 1 个可配置爬虫",
-    "title_en": "Create a configurable spider",
-    "description_cn": "在爬虫列表中，点击 '添加爬虫'，选择 '可配置爬虫'，输入相应的参数，点击添加，即可完成挑战！",
-    "description_en": "In Spider List page, click 'Add Spider', select 'Configurable Spider', enter params, click 'Add' to finish the challenge.",
-    "difficulty": 1,
-    "path": "/spiders"
-  },
-  {
-    "name": "run_random",
-    "title_cn": "用随机模式成功运行爬虫",
-    "title_en": "Run a spider in random mode successfully",
-    "description_cn": "在您创建好的爬虫中，导航到其对应的详情页（爬虫列表中点击爬虫），选择随机模式运行一个爬虫，并能运行成功。",
-    "description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode successfully.",
-    "difficulty": 1,
-    "path": "/spiders"
-  },
-  {
-    "name": "scrape_1k",
-    "title_cn": "抓取 1 千条数据",
-    "title_en": "Scrape 1k records",
-    "description_cn": "运行您创建好的爬虫，抓取 1 千条及以上的结果数据，即可完成挑战！",
-    "description_en": "Run your created spiders, scrape 1k and more results to finish the challenge.",
-    "difficulty": 2,
-    "path": "/spiders"
-  },
-  {
-    "name": "scrape_10k",
-    "title_cn": "抓取 1 万条数据",
-    "title_en": "Scrape 10k records",
-    "description_cn": "运行您创建好的爬虫，抓取 1 万条及以上的结果数据，即可完成挑战！",
-    "description_en": "Run your created spiders, scrape 10k and more results to finish the challenge.",
-    "difficulty": 3,
-    "path": "/spiders"
-  },
-  {
-    "name": "scrape_100k",
-    "title_cn": "抓取 10 万条数据",
-    "title_en": "Scrape 100k records",
-    "description_cn": "运行您创建好的爬虫，抓取 10 万条及以上的结果数据，即可完成挑战！",
-    "description_en": "Run your created spiders, scrape 100k and more results to finish the challenge.",
-    "difficulty": 4,
-    "path": "/spiders"
-  },
-  {
-    "name": "create_schedule",
-    "title_cn": "创建 1 个定时任务",
-    "title_en": "Create a schedule",
-    "description_cn": "在定时任务列表中，创建一个定时任务，正确设置好 Cron 表达式，即可完成挑战！",
-    "description_en": "In Schedule List page, create a schedule and configure cron expression to finish the task.",
-    "difficulty": 1,
-    "path": "/schedules"
-  },
-  {
-    "name": "create_nodes",
-    "title_cn": "创建 1 个节点集群",
-    "title_en": "Create a node cluster",
-    "description_cn": "按照文档的部署指南，部署含有 3 个节点的集群，即可完成挑战！",
-    "description_en": "Deploy a 3-node cluster according to the deployment guidance in documentation to finish the task.",
-    "difficulty": 3,
-    "path": "/nodes"
-  },
-  {
-    "name": "install_dep",
-    "title_cn": "安装 1 个依赖",
-    "title_en": "Install a dependency successfully",
-    "description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中，搜索并安装所需的 1 个依赖，即可完成挑战！",
-    "description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', search and install a dependency.",
-    "difficulty": 3,
-    "path": "/nodes"
-  },
-  {
-    "name": "install_lang",
-    "title_cn": "安装 1 个语言环境",
-    "title_en": "Install a language successfully",
-    "description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中，点击安装所需的 1 个语言环境，即可完成挑战！",
-    "description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', install a language.",
-    "difficulty": 3,
-    "path": "/nodes"
-  },
-  {
-    "name": "view_disclaimer",
-    "title_cn": "阅读免责声明",
-    "title_en": "View disclaimer",
-    "description_cn": "在左侧菜单栏，点击 '免责声明' 查看其内容，即可完成挑战！",
-    "description_en": "In the left side menu, click 'Disclaimer' and view its content to finish the challenge.",
-    "difficulty": 1,
-    "path": "/disclaimer"
-  },
-  {
-    "name": "create_user",
-    "title_cn": "创建 1 个用户",
-    "title_en": "Create a user",
-    "description_cn": "在用户管理页面中创建一个新用户，即可完成挑战！",
-    "description_en": "In User Admin page, create a new user to finish the challenge.",
-    "difficulty": 1,
-    "path": "/users"
-  }
-]

backend/database/es_base.go

@@ -1,44 +0,0 @@
-package database
-
-import (
-	"context"
-	"github.com/apex/log"
-	"github.com/olivere/elastic/v7"
-	"github.com/satori/go.uuid"
-	"github.com/spf13/viper"
-	"sync"
-	"time"
-)
-
-var doOnce sync.Once
-var ctx context.Context
-var ESClient *elastic.Client
-
-func InitEsClient() {
-	esClientStr := viper.GetString("setting.esClient")
-	ctx = context.Background()
-	ESClient, _ = elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
-}
-
-// WriteMsg will write the msg and level into es
-func WriteMsgToES(when time.Time, msg chan string, index string) {
-	doOnce.Do(InitEsClient)
-	vals := make(map[string]interface{})
-	vals["@timestamp"] = when.Format(time.RFC3339)
-	for {
-		select {
-		case vals["@msg"] = <-msg:
-			uid := uuid.NewV4().String()
-			_, err := ESClient.Index().Index(index).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
-			if err != nil {
-				log.Error(err.Error())
-				log.Error("send msg log to es error")
-				return
-			}
-		case <-time.After(6 * time.Second):
-			return
-		}
-	}
-
-	return
-}

backend/database/mongo.go

@@ -1,112 +0,0 @@
-package database
-
-import (
-	"crawlab/constants"
-	"github.com/apex/log"
-	"github.com/cenkalti/backoff/v4"
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-	"github.com/spf13/viper"
-	"net"
-	"reflect"
-	"time"
-)
-
-var Session *mgo.Session
-
-func GetSession() *mgo.Session {
-	return Session.Copy()
-}
-
-func GetDb() (*mgo.Session, *mgo.Database) {
-	s := GetSession()
-	return s, s.DB(viper.GetString("mongo.db"))
-}
-
-func GetCol(collectionName string) (*mgo.Session, *mgo.Collection) {
-	s := GetSession()
-	db := s.DB(viper.GetString("mongo.db"))
-	col := db.C(collectionName)
-	return s, col
-}
-
-func GetGridFs(prefix string) (*mgo.Session, *mgo.GridFS) {
-	s, db := GetDb()
-	gf := db.GridFS(prefix)
-	return s, gf
-}
-
-func FillNullObjectId(doc interface{}) {
-	t := reflect.TypeOf(doc)
-	if t.Kind() == reflect.Ptr {
-		t = t.Elem()
-	}
-	if t.Kind() != reflect.Struct {
-		return
-	}
-	v := reflect.ValueOf(doc)
-	for i := 0; i < t.NumField(); i++ {
-		ft := t.Field(i)
-		fv := v.Elem().Field(i)
-		val := fv.Interface()
-		switch val.(type) {
-		case bson.ObjectId:
-			if !val.(bson.ObjectId).Valid() {
-				v.FieldByName(ft.Name).Set(reflect.ValueOf(bson.ObjectIdHex(constants.ObjectIdNull)))
-			}
-		}
-	}
-}
-
-func InitMongo() error {
-	var mongoHost = viper.GetString("mongo.host")
-	var mongoPort = viper.GetString("mongo.port")
-	var mongoDb = viper.GetString("mongo.db")
-	var mongoUsername = viper.GetString("mongo.username")
-	var mongoPassword = viper.GetString("mongo.password")
-	var mongoAuth = viper.GetString("mongo.authSource")
-
-	if Session == nil {
-		var dialInfo mgo.DialInfo
-		addr := net.JoinHostPort(mongoHost, mongoPort)
-		timeout := time.Second * 10
-		dialInfo = mgo.DialInfo{
-			Addrs:         []string{addr},
-			Timeout:       timeout,
-			Database:      mongoDb,
-			PoolLimit:     100,
-			PoolTimeout:   timeout,
-			ReadTimeout:   timeout,
-			WriteTimeout:  timeout,
-			AppName:       "crawlab",
-			FailFast:      true,
-			MinPoolSize:   10,
-			MaxIdleTimeMS: 1000 * 30,
-		}
-		if mongoUsername != "" {
-			dialInfo.Username = mongoUsername
-			dialInfo.Password = mongoPassword
-			dialInfo.Source = mongoAuth
-		}
-		bp := backoff.NewExponentialBackOff()
-		var err error
-
-		err = backoff.Retry(func() error {
-			Session, err = mgo.DialWithInfo(&dialInfo)
-			if err != nil {
-				log.WithError(err).Warnf("waiting for connect mongo database, after %f seconds try again.", bp.NextBackOff().Seconds())
-			}
-			return err
-		}, bp)
-	}
-	//Add Unique index for 'key'
-	keyIndex := mgo.Index{
-		Key:    []string{"key"},
-		Unique: true,
-	}
-	s, c := GetCol("nodes")
-	defer s.Close()
-	c.EnsureIndex(keyIndex)
-
-	return nil
-}

backend/database/mongo_test.go

@@ -1,70 +0,0 @@
-package database
-
-import (
-	"crawlab/config"
-	"github.com/apex/log"
-	"github.com/globalsign/mgo"
-	. "github.com/smartystreets/goconvey/convey"
-	"github.com/spf13/viper"
-	"reflect"
-	"testing"
-)
-
-func init() {
-	if err := config.InitConfig("../conf/config.yml"); err != nil {
-		log.Fatal("Init config failed")
-	}
-	log.Infof("初始化配置成功")
-	err := InitMongo()
-	if err != nil {
-		log.Fatal("Init mongodb failed")
-	}
-
-}
-
-func TestGetDb(t *testing.T) {
-	Convey("Test GetDb", t, func() {
-		if err := config.InitConfig("../conf/config.yml"); err != nil {
-			t.Fatal("Init config failed")
-		}
-		t.Log("初始化配置成功")
-		err := InitMongo()
-		if err != nil {
-			t.Fatal("Init mongodb failed")
-		}
-		s, db := GetDb()
-		Convey("The value should be Session.Copy", func() {
-			So(s, ShouldResemble, Session.Copy())
-		})
-		Convey("The value should be reference of database", func() {
-			So(db, ShouldResemble, s.DB(viper.GetString("mongo.db")))
-		})
-	})
-}
-
-func TestGetCol(t *testing.T) {
-	var c = "nodes"
-	var colActual *mgo.Collection
-	Convey("Test GetCol", t, func() {
-		s, col := GetCol(c)
-		Convey("s should resemble Session.Copy", func() {
-			So(s, ShouldResemble, Session.Copy())
-			So(reflect.TypeOf(col), ShouldResemble, reflect.TypeOf(colActual))
-		})
-	})
-}
-
-func TestGetGridFs(t *testing.T) {
-	var prefix = "files"
-	var gfActual *mgo.GridFS
-
-	Convey("Test GetGridFs", t, func() {
-		s, gf := GetGridFs(prefix)
-		Convey("s should be session.copy", func() {
-			So(s, ShouldResemble, Session.Copy())
-		})
-		Convey("gf should be *mgo.GridFS", func() {
-			So(reflect.TypeOf(gf), ShouldResemble, reflect.TypeOf(gfActual))
-		})
-	})
-}

backend/database/pubsub.go

@@ -1,96 +0,0 @@
-package database
-
-import (
-	"context"
-	"crawlab/utils"
-	"fmt"
-	"github.com/apex/log"
-	"github.com/gomodule/redigo/redis"
-	errors2 "github.com/pkg/errors"
-	"time"
-)
-
-type ConsumeFunc func(message redis.Message) error
-
-func (r *Redis) Close() {
-	err := r.pool.Close()
-	if err != nil {
-		log.Errorf("redis close error.")
-	}
-}
-func (r *Redis) subscribe(ctx context.Context, consume ConsumeFunc, channel ...string) error {
-	psc := redis.PubSubConn{Conn: r.pool.Get()}
-	if err := psc.Subscribe(redis.Args{}.AddFlat(channel)...); err != nil {
-		return err
-	}
-	done := make(chan error, 1)
-	tick := time.NewTicker(time.Second * 3)
-	defer tick.Stop()
-	go func() {
-		defer utils.Close(psc)
-		for {
-			switch msg := psc.Receive().(type) {
-			case error:
-				done <- fmt.Errorf("redis pubsub receive err: %v", msg)
-				return
-			case redis.Message:
-				if err := consume(msg); err != nil {
-					fmt.Printf("redis pubsub consume message err: %v", err)
-					continue
-				}
-			case redis.Subscription:
-				if msg.Count == 0 {
-					// all channels are unsubscribed
-					return
-				}
-			}
-
-		}
-	}()
-	// start a new goroutine to receive message
-	for {
-		select {
-		case <-ctx.Done():
-			if err := psc.Unsubscribe(); err != nil {
-				fmt.Printf("redis pubsub unsubscribe err: %v \n", err)
-			}
-			done <- nil
-		case <-tick.C:
-			if err := psc.Ping(""); err != nil {
-				fmt.Printf("ping message error: %s \n", err)
-				//done <- err
-			}
-		case err := <-done:
-			close(done)
-			return err
-		}
-	}
-
-}
-func (r *Redis) Subscribe(ctx context.Context, consume ConsumeFunc, channel ...string) error {
-	index := 0
-	go func() {
-		for {
-			err := r.subscribe(ctx, consume, channel...)
-			fmt.Println(err)
-
-			if err == nil {
-				index = 0
-				break
-			}
-			time.Sleep(5 * time.Second)
-			index += 1
-			fmt.Printf("try reconnect %d times \n", index)
-		}
-	}()
-	return nil
-}
-func (r *Redis) Publish(channel, message string) (n int, err error) {
-	conn := r.pool.Get()
-	defer utils.Close(conn)
-	n, err = redis.Int(conn.Do("PUBLISH", channel, message))
-	if err != nil {
-		return 0, errors2.Wrapf(err, "redis publish %s %s", channel, message)
-	}
-	return
-}

backend/database/redis.go

@@ -1,290 +0,0 @@
-package database
-
-import (
-	"context"
-	"crawlab/entity"
-	"crawlab/utils"
-	"errors"
-	"github.com/apex/log"
-	"github.com/cenkalti/backoff/v4"
-	"github.com/gomodule/redigo/redis"
-	"github.com/spf13/viper"
-	"net/url"
-	"runtime/debug"
-	"strings"
-	"time"
-)
-
-var RedisClient *Redis
-
-type Redis struct {
-	pool *redis.Pool
-}
-
-type Mutex struct {
-	Name   string
-	expiry time.Duration
-	tries  int
-	delay  time.Duration
-	value  string
-}
-
-func NewRedisClient() *Redis {
-	return &Redis{pool: NewRedisPool()}
-}
-
-func (r *Redis) RPush(collection string, value interface{}) error {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	if _, err := c.Do("RPUSH", collection, value); err != nil {
-		log.Error(err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-
-func (r *Redis) LPush(collection string, value interface{}) error {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	if _, err := c.Do("RPUSH", collection, value); err != nil {
-		log.Error(err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-
-func (r *Redis) LPop(collection string) (string, error) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	value, err2 := redis.String(c.Do("LPOP", collection))
-	if err2 != nil {
-		return value, err2
-	}
-	return value, nil
-}
-
-func (r *Redis) HSet(collection string, key string, value string) error {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	if _, err := c.Do("HSET", collection, key, value); err != nil {
-		log.Error(err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-func (r *Redis) Ping() error {
-	c := r.pool.Get()
-	defer utils.Close(c)
-	_, err2 := redis.String(c.Do("PING"))
-	return err2
-}
-func (r *Redis) HGet(collection string, key string) (string, error) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-	value, err2 := redis.String(c.Do("HGET", collection, key))
-	if err2 != nil && err2 != redis.ErrNil {
-		log.Error(err2.Error())
-		debug.PrintStack()
-		return value, err2
-	}
-	return value, nil
-}
-
-func (r *Redis) HDel(collection string, key string) error {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	if _, err := c.Do("HDEL", collection, key); err != nil {
-		log.Error(err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-func (r *Redis) HScan(collection string) (results []string, err error) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-	var (
-		cursor int64
-		items  []string
-	)
-
-	for {
-		values, err := redis.Values(c.Do("HSCAN", collection, cursor))
-		if err != nil {
-			return results, err
-		}
-
-		values, err = redis.Scan(values, &cursor, &items)
-		if err != nil {
-			return results, err
-		}
-		for i := 0; i < len(items); i += 2 {
-			cur := items[i+1]
-			results = append(results, cur)
-		}
-		if cursor == 0 {
-			break
-		}
-	}
-	return results, err
-
-}
-func (r *Redis) HKeys(collection string) ([]string, error) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	value, err2 := redis.Strings(c.Do("HKEYS", collection))
-	if err2 != nil {
-		log.Error(err2.Error())
-		debug.PrintStack()
-		return []string{}, err2
-	}
-	return value, nil
-}
-
-func (r *Redis) BRPop(collection string, timeout int) (string, error) {
-	if timeout <= 0 {
-		timeout = 60
-	}
-	c := r.pool.Get()
-	defer utils.Close(c)
-
-	values, err := redis.Strings(c.Do("BRPOP", collection, timeout))
-	if err != nil {
-		return "", err
-	}
-	return values[1], nil
-}
-
-func NewRedisPool() *redis.Pool {
-	var address = viper.GetString("redis.address")
-	var port = viper.GetString("redis.port")
-	var database = viper.GetString("redis.database")
-	var password = url.QueryEscape(viper.GetString("redis.password"))
-
-	var redisUrl string
-	if password == "" {
-		redisUrl = "redis://" + address + ":" + port + "/" + database
-	} else {
-		redisUrl = "redis://x:" + password + "@" + address + ":" + port + "/" + database
-	}
-	return &redis.Pool{
-		Dial: func() (conn redis.Conn, e error) {
-			return redis.DialURL(redisUrl,
-				redis.DialConnectTimeout(time.Second*10),
-				redis.DialReadTimeout(time.Second*600),
-				redis.DialWriteTimeout(time.Second*10),
-			)
-		},
-		TestOnBorrow: func(c redis.Conn, t time.Time) error {
-			if time.Since(t) < time.Minute {
-				return nil
-			}
-			_, err := c.Do("PING")
-			return err
-		},
-		MaxIdle:         10,
-		MaxActive:       0,
-		IdleTimeout:     300 * time.Second,
-		Wait:            false,
-		MaxConnLifetime: 0,
-	}
-}
-
-func InitRedis() error {
-	RedisClient = NewRedisClient()
-	b := backoff.NewExponentialBackOff()
-	b.MaxInterval = 20 * time.Second
-	err := backoff.Retry(func() error {
-		err := RedisClient.Ping()
-
-		if err != nil {
-			log.WithError(err).Warnf("waiting for redis pool active connection. will after %f seconds try  again.", b.NextBackOff().Seconds())
-		}
-		return err
-	}, b)
-	return err
-}
-
-func Pub(channel string, msg entity.NodeMessage) error {
-	if _, err := RedisClient.Publish(channel, utils.GetJson(msg)); err != nil {
-		log.Errorf("publish redis error: %s", err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-
-func Sub(channel string, consume ConsumeFunc) error {
-	ctx := context.Background()
-	if err := RedisClient.Subscribe(ctx, consume, channel); err != nil {
-		log.Errorf("subscribe redis error: %s", err.Error())
-		debug.PrintStack()
-		return err
-	}
-	return nil
-}
-
-// 构建同步锁key
-func (r *Redis) getLockKey(lockKey string) string {
-	lockKey = strings.ReplaceAll(lockKey, ":", "-")
-	return "nodes:lock:" + lockKey
-}
-
-// 获得锁
-func (r *Redis) Lock(lockKey string) (int64, error) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-	lockKey = r.getLockKey(lockKey)
-
-	ts := time.Now().Unix()
-	ok, err := c.Do("SET", lockKey, ts, "NX", "PX", 30000)
-	if err != nil {
-		log.Errorf("get lock fail with error: %s", err.Error())
-		debug.PrintStack()
-		return 0, err
-	}
-	if ok == nil {
-		log.Errorf("the lockKey is locked: key=%s", lockKey)
-		return 0, errors.New("the lockKey is locked")
-	}
-	return ts, nil
-}
-
-func (r *Redis) UnLock(lockKey string, value int64) {
-	c := r.pool.Get()
-	defer utils.Close(c)
-	lockKey = r.getLockKey(lockKey)
-
-	getValue, err := redis.Int64(c.Do("GET", lockKey))
-	if err != nil {
-		log.Errorf("get lockKey error: %s", err.Error())
-		debug.PrintStack()
-		return
-	}
-
-	if getValue != value {
-		log.Errorf("the lockKey value diff: %d, %d", value, getValue)
-		return
-	}
-
-	v, err := redis.Int64(c.Do("DEL", lockKey))
-	if err != nil {
-		log.Errorf("unlock failed, error: %s", err.Error())
-		debug.PrintStack()
-		return
-	}
-
-	if v == 0 {
-		log.Errorf("unlock failed: key=%s", lockKey)
-		return
-	}
-}

backend/entity/common.go

@@ -1,17 +0,0 @@
-package entity
-
-import "strconv"
-
-type Page struct {
-	Skip     int
-	Limit    int
-	PageNum  int
-	PageSize int
-}
-
-func (p *Page) GetPage(pageNum string, pageSize string) {
-	p.PageNum, _ = strconv.Atoi(pageNum)
-	p.PageSize, _ = strconv.Atoi(pageSize)
-	p.Skip = p.PageSize * (p.PageNum - 1)
-	p.Limit = p.PageSize
-}

backend/entity/config_spider.go

@@ -1,40 +0,0 @@
-package entity
-
-type ConfigSpiderData struct {
-	// 通用
-	Name        string `yaml:"name" json:"name"`
-	DisplayName string `yaml:"display_name" json:"display_name"`
-	Col         string `yaml:"col" json:"col"`
-	Remark      string `yaml:"remark" json:"remark"`
-	Type        string `yaml:"type" bson:"type"`
-
-	// 可配置爬虫
-	Engine     string            `yaml:"engine" json:"engine"`
-	StartUrl   string            `yaml:"start_url" json:"start_url"`
-	StartStage string            `yaml:"start_stage" json:"start_stage"`
-	Stages     []Stage           `yaml:"stages" json:"stages"`
-	Settings   map[string]string `yaml:"settings" json:"settings"`
-
-	// 自定义爬虫
-	Cmd string `yaml:"cmd" json:"cmd"`
-}
-
-type Stage struct {
-	Name      string  `yaml:"name" json:"name"`
-	IsList    bool    `yaml:"is_list" json:"is_list"`
-	ListCss   string  `yaml:"list_css" json:"list_css"`
-	ListXpath string  `yaml:"list_xpath" json:"list_xpath"`
-	PageCss   string  `yaml:"page_css" json:"page_css"`
-	PageXpath string  `yaml:"page_xpath" json:"page_xpath"`
-	PageAttr  string  `yaml:"page_attr" json:"page_attr"`
-	Fields    []Field `yaml:"fields" json:"fields"`
-}
-
-type Field struct {
-	Name      string `yaml:"name" json:"name"`
-	Css       string `yaml:"css" json:"css"`
-	Xpath     string `yaml:"xpath" json:"xpath"`
-	Attr      string `yaml:"attr" json:"attr"`
-	NextStage string `yaml:"next_stage" json:"next_stage"`
-	Remark    string `yaml:"remark" json:"remark"`
-}

backend/entity/doc.go

@@ -1,8 +0,0 @@
-package entity
-
-type DocItem struct {
-	Title    string    `json:"title"`
-	Url      string    `json:"url"`
-	Path     string    `json:"path"`
-	Children []DocItem `json:"children"`
-}

backend/entity/node.go

@@ -1,28 +0,0 @@
-package entity
-
-type NodeMessage struct {
-	// 通信类别
-	Type string `json:"type"`
-
-	// 任务相关
-	TaskId string `json:"task_id"` // 任务ID
-
-	// 节点相关
-	NodeId string `json:"node_id"` // 节点ID
-
-	// 日志相关
-	LogPath string `json:"log_path"` // 日志路径
-	Log     string `json:"log"`      // 日志
-
-	// 系统信息
-	SysInfo SystemInfo `json:"sys_info"`
-
-	// 爬虫相关
-	SpiderId string `json:"spider_id"` //爬虫ID
-
-	// 语言相关
-	Lang Lang `json:"lang"`
-
-	// 错误相关
-	Error string `json:"error"`
-}

backend/entity/rpc.go

@@ -1,11 +0,0 @@
-package entity
-
-type RpcMessage struct {
-	Id      string            `json:"id"`      // 消息ID
-	Method  string            `json:"method"`  // 消息方法
-	NodeId  string            `json:"node_id"` // 节点ID
-	Params  map[string]string `json:"params"`  // 参数
-	Timeout int               `json:"timeout"` // 超时
-	Result  string            `json:"result"`  // 结果
-	Error   string            `json:"error"`   // 错误
-}

backend/entity/spider.go

@@ -1,17 +0,0 @@
-package entity
-
-type SpiderType struct {
-	Type  string `json:"type" bson:"_id"`
-	Count int    `json:"count" bson:"count"`
-}
-
-type ScrapySettingParam struct {
-	Key   string      `json:"key"`
-	Value interface{} `json:"value"`
-	Type  string      `json:"type"`
-}
-
-type ScrapyItem struct {
-	Name   string   `json:"name"`
-	Fields []string `json:"fields"`
-}

backend/entity/system.go

@@ -1,39 +0,0 @@
-package entity
-
-type SystemInfo struct {
-	ARCH        string       `json:"arch"`
-	OS          string       `json:"os"`
-	Hostname    string       `json:"host_name"`
-	NumCpu      int          `json:"num_cpu"`
-	Executables []Executable `json:"executables"`
-}
-
-type Executable struct {
-	Path        string `json:"path"`
-	FileName    string `json:"file_name"`
-	DisplayName string `json:"display_name"`
-}
-
-type Lang struct {
-	Name              string   `json:"name"`
-	ExecutableName    string   `json:"executable_name"`
-	ExecutablePaths   []string `json:"executable_paths"`
-	DepExecutablePath string   `json:"dep_executable_path"`
-	LockPath          string   `json:"lock_path"`
-	InstallScript     string   `json:"install_script"`
-	InstallStatus     string   `json:"install_status"`
-	DepFileName       string   `json:"dep_file_name"`
-	InstallDepArgs    string   `json:"install_dep_cmd"`
-	Type              string   `json:"type"`
-}
-
-type Dependency struct {
-	Name        string `json:"name"`
-	Version     string `json:"version"`
-	Description string `json:"description"`
-	Installed   bool   `json:"installed"`
-}
-
-type PackageJson struct {
-	Dependencies map[string]string `json:"dependencies"`
-}

backend/entity/version.go

@@ -1,23 +0,0 @@
-package entity
-
-type Release struct {
-	Name        string `json:"name"`
-	Draft       bool   `json:"draft"`
-	PreRelease  bool   `json:"pre_release"`
-	PublishedAt string `json:"published_at"`
-	Body        string `json:"body"`
-}
-
-type ReleaseSlices []Release
-
-func (r ReleaseSlices) Len() int {
-	return len(r)
-}
-
-func (r ReleaseSlices) Less(i, j int) bool {
-	return r[i].PublishedAt < r[j].PublishedAt
-}
-
-func (r ReleaseSlices) Swap(i, j int) {
-	r[i], r[j] = r[j], r[i]
-}

backend/errors/errors.go

@@ -1,54 +0,0 @@
-package errors
-
-import (
-	"fmt"
-	"net/http"
-)
-
-type Scope int
-
-const (
-	ScopeSystem   Scope = 1
-	ScopeBusiness Scope = 2
-)
-
-type OPError struct {
-	HttpCode int
-	Message  string
-	Code     int
-	Scope    Scope
-}
-
-func (O OPError) Error() string {
-	var scope string
-	switch O.Scope {
-	case ScopeSystem:
-		scope = "system"
-	case ScopeBusiness:
-		scope = "business"
-	}
-	return fmt.Sprintf("%s error: [%d]%s.", scope, O.Code, O.Message)
-}
-
-func NewSystemOPError(code int, message string, httpCodes ...int) *OPError {
-	httpCode := http.StatusOK
-	if len(httpCodes) > 0 {
-		httpCode = httpCodes[0]
-	}
-	return NewOpError(code, message, ScopeSystem, httpCode)
-}
-func NewOpError(code int, message string, scope Scope, httpCode int) *OPError {
-	return &OPError{
-		Message:  message,
-		Code:     code,
-		Scope:    scope,
-		HttpCode: httpCode,
-	}
-}
-func NewBusinessError(code int, message string, httpCodes ...int) *OPError {
-	httpCode := http.StatusOK
-	if len(httpCodes) > 0 {
-		httpCode = httpCodes[0]
-	}
-	return NewOpError(code, message, ScopeBusiness, httpCode)
-}

backend/go.mod

@@ -1,43 +1,15 @@
 module crawlab
 
-go 1.12
+go 1.15
 
 require (
-	github.com/Masterminds/semver v1.4.2 // indirect
-	github.com/Masterminds/sprig v2.16.0+incompatible // indirect
-	github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd
-	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
-	github.com/aokoli/goutils v1.0.1 // indirect
-	github.com/apex/log v1.1.4
-	github.com/cenkalti/backoff/v4 v4.0.2
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/fsnotify/fsnotify v1.4.9
+	github.com/apex/log v1.9.0
+	github.com/crawlab-team/crawlab-core v0.6.0-beta.20210715
+	github.com/crawlab-team/crawlab-db v0.1.0
+	github.com/crawlab-team/go-trace v0.1.0
 	github.com/gin-gonic/gin v1.6.3
-	github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
-	github.com/go-playground/validator/v10 v10.3.0
-	github.com/gomodule/redigo v2.0.0+incompatible
-	github.com/hashicorp/go-sockaddr v1.0.0
-	github.com/huandu/xstrings v1.2.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
-	github.com/imroc/req v0.3.0
-	github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect
-	github.com/matcornic/hermes v1.2.0
-	github.com/mattn/go-runewidth v0.0.3 // indirect
-	github.com/olekukonko/tablewriter v0.0.1 // indirect
-	github.com/olivere/elastic/v7 v7.0.15
-	github.com/pkg/errors v0.9.1
-	github.com/satori/go.uuid v1.2.0
-	github.com/smartystreets/goconvey v1.6.4
-	github.com/spf13/viper v1.7.0
-	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
-	github.com/swaggo/gin-swagger v1.2.0
-	github.com/swaggo/swag v1.6.9
-	go.uber.org/atomic v1.6.0
-	golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
-	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
-	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
-	gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect
-	gopkg.in/src-d/go-git.v4 v4.13.1
-	gopkg.in/yaml.v2 v2.3.0
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/spf13/cobra v1.1.3
+	github.com/spf13/viper v1.7.1
+	go.uber.org/dig v1.10.0
 )

backend/go.sum

@@ -4,6 +4,7 @@ cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSR
 cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3 h1:AVXDdKsrtX33oR9fbCMu/+c1o8Ofjq6Ku/MInaLVg5Y=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
@@ -14,35 +15,36 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc=
 github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY=
 github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
-github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
-github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd h1:+CYOsXi89xOqBkj7CuEJjA2It+j+R3ngUZEydr6mtkw=
+github.com/StackExchange/wmi v1.2.0 h1:noJEYkMQVlFCEAc+2ma5YyRhlfjcWfZqk5sBRYozdyM=
+github.com/StackExchange/wmi v1.2.0/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/Unknwon/goconfig v0.0.0-20191126170842-860a72fb44fd/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw=
+github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
+github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
+github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
 github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
-github.com/apex/log v1.1.4 h1:3Zk+boorIQAAGBrHn0JUtAau4ihMamT4WdnfdnXM1zQ=
-github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
-github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
+github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
+github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
+github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
 github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
 github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -52,23 +54,50 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk=
+github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
-github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
+github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
+github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20210715 h1:2ryY/+sdRR71Cd0dN7NvR6S9toVlm0LloaJC82NnZcg=
+github.com/crawlab-team/crawlab-core v0.6.0-beta.20210715/go.mod h1:OXO0hN3YwKN0hnJoa6bJ/DGWZjE4XzVAv+pfmCzOmds=
+github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78=
+github.com/crawlab-team/crawlab-db v0.1.0 h1:6dTVNb5+7cDkH8fkKOkFALk8laWNOuorYm3ZEKsvFLI=
+github.com/crawlab-team/crawlab-db v0.1.0/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04=
+github.com/crawlab-team/crawlab-fs v0.0.0/go.mod h1:k2VXprQspLAmbgO5sSpqMjg/xP4iKDkW4RyTWY8eTZM=
+github.com/crawlab-team/crawlab-fs v0.1.0 h1:iKSJJY4Wvea8Qss+zC/tLiZ371VeV75Z3cuqlsxydzY=
+github.com/crawlab-team/crawlab-fs v0.1.0/go.mod h1:dOE0TeWPDz9krwzt1H72rjj0Fn/aHe53yn7GoOZHD0s=
+github.com/crawlab-team/crawlab-grpc v0.1.0 h1:cCAvQMyXCuUumek2EmYBFRU4UwnC84454vwc+vg4PeE=
+github.com/crawlab-team/crawlab-grpc v0.1.0/go.mod h1:W9Yee6xfesxoaqS5K1sF1I1zlH+i6xqwy4lyoBTOdkc=
+github.com/crawlab-team/crawlab-log v0.1.0 h1:0t+lZEojs3Vqb/bMkk2qs3I+1+XdwKG3pMTfeK5PZWM=
+github.com/crawlab-team/crawlab-log v0.1.0/go.mod h1:N8nTTKEbr9ZQSlmw0+HNB4ZAMQF4yVMaJLx8YhXvhNo=
+github.com/crawlab-team/crawlab-vcs v0.1.0 h1:LjtKOOFzx1o7vvgGppC7jt/8lznyvFwwXBYggbSW9+4=
+github.com/crawlab-team/crawlab-vcs v0.1.0/go.mod h1:G6Hnt/3255QCGHO5Q0xJe1AbJE7m5t65E0v7flRJBJM=
+github.com/crawlab-team/go-trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg=
+github.com/crawlab-team/go-trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8=
+github.com/crawlab-team/goseaweedfs v0.1.6 h1:4Qmz75e65h4gbj0/sTM7pPVFwVJl8MihXf28igy2cFc=
+github.com/crawlab-team/goseaweedfs v0.1.6/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA=
+github.com/crawlab-team/goseaweedfs v0.2.0 h1:9acRgmJVss+ghIqm/XMCNU0G2RUBAvvDaOOASvWNInc=
+github.com/crawlab-team/goseaweedfs v0.2.0/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -77,7 +106,15 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fasthttp/websocket v1.4.2 h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM=
+github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
+github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -86,43 +123,47 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/gamexg/proxyclient v0.0.0-20210207161252-499908056324/go.mod h1:Y1lsrg1lNY/7qdJ6masxXhIOUHajrlcixeMMWU59cjg=
+github.com/gavv/httpexpect/v2 v2.2.0 h1:0VwaEBmQaNFHX9x591A8Up+8shCwdF/nF0qlRd/nI48=
+github.com/gavv/httpexpect/v2 v2.2.0/go.mod h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc=
 github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w=
 github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
-github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y=
-github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ=
 github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
+github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
+github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
+github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
+github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
+github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-openapi/jsonpointer v0.17.0 h1:nH6xp8XdXHx8dqveo0ZuJBluCO2qGrPbDNZ0dwoRHP0=
+github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
-github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
+github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
 github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
-github.com/go-openapi/jsonreference v0.19.4 h1:3Vw+rh13uq2JFNxgnMTGE1rnoieU9FmyE1gvnyylsYg=
-github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
-github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
+github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
-github.com/go-openapi/spec v0.19.9 h1:9z9cbFuZJ7AcvOHKIY+f6Aevb4vObNDkTEyoMfO7rAc=
-github.com/go-openapi/spec v0.19.9/go.mod h1:vqK/dIdLGCosfvYsQV3WfC7N3TiZSnGY2RZKoFK7X28=
-github.com/go-openapi/swag v0.17.0 h1:iqrgMg7Q7SvtbWLlltPrkMs0UBJI6oTSs79JFRUi880=
+github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
-github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -134,10 +175,37 @@ github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.3.0 h1:nZU+7q+yJoFmwvNgv/LnPUkwPal62+b2xXj0AU1Es7o=
 github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
+github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
+github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
+github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
+github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
+github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
+github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
+github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
+github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
+github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
+github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
+github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
+github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
+github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
+github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
+github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
+github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
+github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
+github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
+github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -151,6 +219,17 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -158,8 +237,14 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -167,12 +252,19 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/websocket v1.0.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
@@ -197,13 +289,19 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0=
 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
-github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
+github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E=
+github.com/imkira/go-interpol v1.0.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U=
 github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
 github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -211,6 +309,15 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
+github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
+github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -222,20 +329,43 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
+github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M=
+github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/linxGnu/goseaweedfs v0.1.5/go.mod h1:Zwe/7H7FJaPQyMTNKXgv6fhVDw6qi34MMJQp1K0VLNc=
+github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M=
+github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -243,10 +373,13 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
+github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU=
 github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -258,6 +391,8 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -278,19 +413,28 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE=
 github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI=
+github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -303,24 +447,44 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
+github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY=
+github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
+github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shadowsocks/shadowsocks-go v0.0.0-20200409064450-3e585ff90601/go.mod h1:mttDPaeLm87u74HMrP+n2tugXvIKWcwff/cqSX0lehY=
+github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto=
+github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
@@ -334,13 +498,18 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
-github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
+github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
 github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
@@ -358,35 +527,61 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14/go.mod h1:gxQT6pBGRuIGunNf/+tSOB5OHvguWi8Tbt82WOkf35E=
-github.com/swaggo/gin-swagger v1.2.0 h1:YskZXEiv51fjOMTsXrOetAjrMDfFaXD79PEoQBOe2W0=
 github.com/swaggo/gin-swagger v1.2.0/go.mod h1:qlH2+W7zXGZkczuL+r2nEBR2JTT+/lX05Nn6vPhc7OI=
-github.com/swaggo/swag v1.5.1 h1:2Agm8I4K5qb00620mHq0VJ05/KT4FtmALPIcQR9lEZM=
 github.com/swaggo/swag v1.5.1/go.mod h1:1Bl9F/ZBpVWh22nY0zmYyASPO1lI/zIwRDrpZU+tv8Y=
-github.com/swaggo/swag v1.6.9 h1:BukKRwZjnEcUxQt7Xgfrt9fpav0hiWw9YimdNO9wssw=
-github.com/swaggo/swag v1.6.9/go.mod h1:a0IpNeMfGidNOcm2TsqODUh9JHdHu3kxDA0UlGbBKjI=
+github.com/swaggo/swag v1.6.6/go.mod h1:xDhTyuFIujYiN3DKWC/H/83xcfHp+UE/IzWWampG7Zc=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
+github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
+github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
+github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
 github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go v1.1.5-pre h1:jyJKFOSEbdOc2HODrf2qcCkYOdq7zzXqA9bhW5oV4fM=
 github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v1.1.5-pre h1:5YV9PsFAN+ndcCtTM7s60no7nY7eTG3LPtxhSwuxzCs=
 github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
+github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
+github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
+github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
+github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
+github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
+github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y=
+github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.mongodb.org/mongo-driver v1.4.5 h1:TLtO+iD8krabXxvY1F1qpBOHgOxhLWR7XsT7kQeRmMY=
+go.mongodb.org/mongo-driver v1.4.5/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -394,21 +589,26 @@ go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
+go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
+golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -428,8 +628,6 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -446,24 +644,29 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -476,21 +679,32 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -508,26 +722,28 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606050223-4d9ae51c2468/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190611222205-d73e1c7e250b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200820010801-b793a1359eac h1:DugppSxw0LSF8lcjaODPJZoDzq0ElTGskTst3ZaBkHI=
-golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -538,6 +754,7 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -549,9 +766,27 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
@@ -560,11 +795,12 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
 gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
-gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
@@ -573,12 +809,10 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/russross/blackfriday.v2 v2.0.0 h1:+FlnIV8DSQnT7NZ43hcVKcdJdzZoeCmJj4Ql8gq5keA=
 gopkg.in/russross/blackfriday.v2 v2.0.0/go.mod h1:6sSBNz/GtOm/pJTuh5UmBK2ZHfmnxGbl2NZg1UliSOI=
-gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
 gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
-gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg=
 gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
-gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
 gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
@@ -591,10 +825,16 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
+gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo=
+moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

backend/lib/cron/.gitignore

@@ -1,22 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe

backend/lib/cron/.travis.yml

@@ -1 +0,0 @@
-language: go

backend/lib/cron/LICENSE

@@ -1,21 +0,0 @@
-Copyright (C) 2012 Rob Figueiredo
-All Rights Reserved.
-
-MIT LICENSE
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

backend/lib/cron/README.md

@@ -1,125 +0,0 @@
-[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron)
-[![Build Status](https://travis-ci.org/robfig/cron.svg?branch=master)](https://travis-ci.org/robfig/cron)
-
-# cron
-
-Cron V3 has been released!
-
-To download the specific tagged release, run:
-
-	go get github.com/robfig/cron/v3@v3.0.0
-
-Import it in your program as:
-
-	import "github.com/robfig/cron/v3"
-
-It requires Go 1.11 or later due to usage of Go Modules.
-
-Refer to the documentation here:
-http://godoc.org/github.com/robfig/cron
-
-The rest of this document describes the the advances in v3 and a list of
-breaking changes for users that wish to upgrade from an earlier version.
-
-## Upgrading to v3 (June 2019)
-
-cron v3 is a major upgrade to the library that addresses all outstanding bugs,
-feature requests, and rough edges. It is based on a merge of master which
-contains various fixes to issues found over the years and the v2 branch which
-contains some backwards-incompatible features like the ability to remove cron
-jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like
-the timezone support, and fixes a number of bugs.
-
-New features:
-
-- Support for Go modules. Callers must now import this library as
-  `github.com/robfig/cron/v3`, instead of `gopkg.in/...`
-
-- Fixed bugs:
-  - 0f01e6b parser: fix combining of Dow and Dom (#70)
-  - dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157)
-  - eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144)
-  - 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97)
-  - 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206)
-
-- Standard cron spec parsing by default (first field is "minute"), with an easy
-  way to opt into the seconds field (quartz-compatible). Although, note that the
-  year field (optional in Quartz) is not supported.
-
-- Extensible, key/value logging via an interface that complies with
-  the https://github.com/go-logr/logr project.
-
-- The new Chain & JobWrapper types allow you to install "interceptors" to add
-  cross-cutting behavior like the following:
-  - Recover any panics from jobs
-  - Delay a job's execution if the previous run hasn't completed yet
-  - Skip a job's execution if the previous run hasn't completed yet
-  - Log each job's invocations
-  - Notification when jobs are completed
-
-It is backwards incompatible with both v1 and v2. These updates are required:
-
-- The v1 branch accepted an optional seconds field at the beginning of the cron
-  spec. This is non-standard and has led to a lot of confusion. The new default
-  parser conforms to the standard as described by [the Cron wikipedia page].
-
-  UPDATING: To retain the old behavior, construct your Cron with a custom
-  parser:
-
-      // Seconds field, required
-      cron.New(cron.WithSeconds())
-
-      // Seconds field, optional
-      cron.New(
-          cron.WithParser(
-              cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
-
-- The Cron type now accepts functional options on construction rather than the
-  previous ad-hoc behavior modification mechanisms (setting a field, calling a setter).
-
-  UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be
-  updated to provide those values on construction.
-
-- CRON_TZ is now the recommended way to specify the timezone of a single
-  schedule, which is sanctioned by the specification. The legacy "TZ=" prefix
-  will continue to be supported since it is unambiguous and easy to do so.
-
-  UPDATING: No update is required.
-
-- By default, cron will no longer recover panics in jobs that it runs.
-  Recovering can be surprising (see issue #192) and seems to be at odds with
-  typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option
-  has been removed to accommodate the more general JobWrapper type.
-
-  UPDATING: To opt into panic recovery and configure the panic logger:
-
-      cron.New(cron.WithChain(
-          cron.Recover(logger),  // or use cron.DefaultLogger
-      ))
-
-- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was
-  removed, since it is duplicative with the leveled logging.
-
-  UPDATING: Callers should use `WithLogger` and specify a logger that does not
-  discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`:
-
-      cron.New(
-          cron.WithLogger(cron.VerbosePrintfLogger(logger)))
-
-
-### Background - Cron spec format
-
-There are two cron spec formats in common usage:
-
-- The "standard" cron format, described on [the Cron wikipedia page] and used by
-  the cron Linux system utility.
-
-- The cron format used by [the Quartz Scheduler], commonly used for scheduled
-  jobs in Java software
-
-[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron
-[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
-
-The original version of this package included an optional "seconds" field, which
-made it incompatible with both of these formats. Now, the "standard" format is
-the default format accepted, and the Quartz format is opt-in.

backend/lib/cron/chain.go

@@ -1,92 +0,0 @@
-package cron
-
-import (
-	"fmt"
-	"runtime"
-	"sync"
-	"time"
-)
-
-// JobWrapper decorates the given Job with some behavior.
-type JobWrapper func(Job) Job
-
-// Chain is a sequence of JobWrappers that decorates submitted jobs with
-// cross-cutting behaviors like logging or synchronization.
-type Chain struct {
-	wrappers []JobWrapper
-}
-
-// NewChain returns a Chain consisting of the given JobWrappers.
-func NewChain(c ...JobWrapper) Chain {
-	return Chain{c}
-}
-
-// Then decorates the given job with all JobWrappers in the chain.
-//
-// This:
-//     NewChain(m1, m2, m3).Then(job)
-// is equivalent to:
-//     m1(m2(m3(job)))
-func (c Chain) Then(j Job) Job {
-	for i := range c.wrappers {
-		j = c.wrappers[len(c.wrappers)-i-1](j)
-	}
-	return j
-}
-
-// Recover panics in wrapped jobs and log them with the provided logger.
-func Recover(logger Logger) JobWrapper {
-	return func(j Job) Job {
-		return FuncJob(func() {
-			defer func() {
-				if r := recover(); r != nil {
-					const size = 64 << 10
-					buf := make([]byte, size)
-					buf = buf[:runtime.Stack(buf, false)]
-					err, ok := r.(error)
-					if !ok {
-						err = fmt.Errorf("%v", r)
-					}
-					logger.Error(err, "panic", "stack", "...\n"+string(buf))
-				}
-			}()
-			j.Run()
-		})
-	}
-}
-
-// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
-// previous one is complete. Jobs running after a delay of more than a minute
-// have the delay logged at Info.
-func DelayIfStillRunning(logger Logger) JobWrapper {
-	return func(j Job) Job {
-		var mu sync.Mutex
-		return FuncJob(func() {
-			start := time.Now()
-			mu.Lock()
-			defer mu.Unlock()
-			if dur := time.Since(start); dur > time.Minute {
-				logger.Info("delay", "duration", dur)
-			}
-			j.Run()
-		})
-	}
-}
-
-// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
-// still running. It logs skips to the given logger at Info level.
-func SkipIfStillRunning(logger Logger) JobWrapper {
-	var ch = make(chan struct{}, 1)
-	ch <- struct{}{}
-	return func(j Job) Job {
-		return FuncJob(func() {
-			select {
-			case v := <-ch:
-				j.Run()
-				ch <- v
-			default:
-				logger.Info("skip")
-			}
-		})
-	}
-}

backend/lib/cron/chain_test.go

@@ -1,221 +0,0 @@
-package cron
-
-import (
-	"io/ioutil"
-	"log"
-	"reflect"
-	"sync"
-	"testing"
-	"time"
-)
-
-func appendingJob(slice *[]int, value int) Job {
-	var m sync.Mutex
-	return FuncJob(func() {
-		m.Lock()
-		*slice = append(*slice, value)
-		m.Unlock()
-	})
-}
-
-func appendingWrapper(slice *[]int, value int) JobWrapper {
-	return func(j Job) Job {
-		return FuncJob(func() {
-			appendingJob(slice, value).Run()
-			j.Run()
-		})
-	}
-}
-
-func TestChain(t *testing.T) {
-	var nums []int
-	var (
-		append1 = appendingWrapper(&nums, 1)
-		append2 = appendingWrapper(&nums, 2)
-		append3 = appendingWrapper(&nums, 3)
-		append4 = appendingJob(&nums, 4)
-	)
-	NewChain(append1, append2, append3).Then(append4).Run()
-	if !reflect.DeepEqual(nums, []int{1, 2, 3, 4}) {
-		t.Error("unexpected order of calls:", nums)
-	}
-}
-
-func TestChainRecover(t *testing.T) {
-	panickingJob := FuncJob(func() {
-		panic("panickingJob panics")
-	})
-
-	t.Run("panic exits job by default", func(t *testing.T) {
-		defer func() {
-			if err := recover(); err == nil {
-				t.Errorf("panic expected, but none received")
-			}
-		}()
-		NewChain().Then(panickingJob).
-			Run()
-	})
-
-	t.Run("Recovering JobWrapper recovers", func(t *testing.T) {
-		NewChain(Recover(PrintfLogger(log.New(ioutil.Discard, "", 0)))).
-			Then(panickingJob).
-			Run()
-	})
-
-	t.Run("composed with the *IfStillRunning wrappers", func(t *testing.T) {
-		NewChain(Recover(PrintfLogger(log.New(ioutil.Discard, "", 0)))).
-			Then(panickingJob).
-			Run()
-	})
-}
-
-type countJob struct {
-	m       sync.Mutex
-	started int
-	done    int
-	delay   time.Duration
-}
-
-func (j *countJob) Run() {
-	j.m.Lock()
-	j.started++
-	j.m.Unlock()
-	time.Sleep(j.delay)
-	j.m.Lock()
-	j.done++
-	j.m.Unlock()
-}
-
-func (j *countJob) Started() int {
-	defer j.m.Unlock()
-	j.m.Lock()
-	return j.started
-}
-
-func (j *countJob) Done() int {
-	defer j.m.Unlock()
-	j.m.Lock()
-	return j.done
-}
-
-func TestChainDelayIfStillRunning(t *testing.T) {
-
-	t.Run("runs immediately", func(t *testing.T) {
-		var j countJob
-		wrappedJob := NewChain(DelayIfStillRunning(DiscardLogger)).Then(&j)
-		go wrappedJob.Run()
-		time.Sleep(2 * time.Millisecond) // Give the job 2ms to complete.
-		if c := j.Done(); c != 1 {
-			t.Errorf("expected job run once, immediately, got %d", c)
-		}
-	})
-
-	t.Run("second run immediate if first done", func(t *testing.T) {
-		var j countJob
-		wrappedJob := NewChain(DelayIfStillRunning(DiscardLogger)).Then(&j)
-		go func() {
-			go wrappedJob.Run()
-			time.Sleep(time.Millisecond)
-			go wrappedJob.Run()
-		}()
-		time.Sleep(3 * time.Millisecond) // Give both jobs 3ms to complete.
-		if c := j.Done(); c != 2 {
-			t.Errorf("expected job run twice, immediately, got %d", c)
-		}
-	})
-
-	t.Run("second run delayed if first not done", func(t *testing.T) {
-		var j countJob
-		j.delay = 10 * time.Millisecond
-		wrappedJob := NewChain(DelayIfStillRunning(DiscardLogger)).Then(&j)
-		go func() {
-			go wrappedJob.Run()
-			time.Sleep(time.Millisecond)
-			go wrappedJob.Run()
-		}()
-
-		// After 5ms, the first job is still in progress, and the second job was
-		// run but should be waiting for it to finish.
-		time.Sleep(5 * time.Millisecond)
-		started, done := j.Started(), j.Done()
-		if started != 1 || done != 0 {
-			t.Error("expected first job started, but not finished, got", started, done)
-		}
-
-		// Verify that the second job completes.
-		time.Sleep(25 * time.Millisecond)
-		started, done = j.Started(), j.Done()
-		if started != 2 || done != 2 {
-			t.Error("expected both jobs done, got", started, done)
-		}
-	})
-
-}
-
-func TestChainSkipIfStillRunning(t *testing.T) {
-
-	t.Run("runs immediately", func(t *testing.T) {
-		var j countJob
-		wrappedJob := NewChain(SkipIfStillRunning(DiscardLogger)).Then(&j)
-		go wrappedJob.Run()
-		time.Sleep(2 * time.Millisecond) // Give the job 2ms to complete.
-		if c := j.Done(); c != 1 {
-			t.Errorf("expected job run once, immediately, got %d", c)
-		}
-	})
-
-	t.Run("second run immediate if first done", func(t *testing.T) {
-		var j countJob
-		wrappedJob := NewChain(SkipIfStillRunning(DiscardLogger)).Then(&j)
-		go func() {
-			go wrappedJob.Run()
-			time.Sleep(time.Millisecond)
-			go wrappedJob.Run()
-		}()
-		time.Sleep(3 * time.Millisecond) // Give both jobs 3ms to complete.
-		if c := j.Done(); c != 2 {
-			t.Errorf("expected job run twice, immediately, got %d", c)
-		}
-	})
-
-	t.Run("second run skipped if first not done", func(t *testing.T) {
-		var j countJob
-		j.delay = 10 * time.Millisecond
-		wrappedJob := NewChain(SkipIfStillRunning(DiscardLogger)).Then(&j)
-		go func() {
-			go wrappedJob.Run()
-			time.Sleep(time.Millisecond)
-			go wrappedJob.Run()
-		}()
-
-		// After 5ms, the first job is still in progress, and the second job was
-		// aleady skipped.
-		time.Sleep(5 * time.Millisecond)
-		started, done := j.Started(), j.Done()
-		if started != 1 || done != 0 {
-			t.Error("expected first job started, but not finished, got", started, done)
-		}
-
-		// Verify that the first job completes and second does not run.
-		time.Sleep(25 * time.Millisecond)
-		started, done = j.Started(), j.Done()
-		if started != 1 || done != 1 {
-			t.Error("expected second job skipped, got", started, done)
-		}
-	})
-
-	t.Run("skip 10 jobs on rapid fire", func(t *testing.T) {
-		var j countJob
-		j.delay = 10 * time.Millisecond
-		wrappedJob := NewChain(SkipIfStillRunning(DiscardLogger)).Then(&j)
-		for i := 0; i < 11; i++ {
-			go wrappedJob.Run()
-		}
-		time.Sleep(200 * time.Millisecond)
-		done := j.Done()
-		if done != 1 {
-			t.Error("expected 1 jobs executed, 10 jobs dropped, got", done)
-		}
-	})
-
-}

backend/lib/cron/constantdelay.go

@@ -1,27 +0,0 @@
-package cron
-
-import "time"
-
-// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes".
-// It does not support jobs more frequent than once a second.
-type ConstantDelaySchedule struct {
-	Delay time.Duration
-}
-
-// Every returns a crontab Schedule that activates once every duration.
-// Delays of less than a second are not supported (will round up to 1 second).
-// Any fields less than a Second are truncated.
-func Every(duration time.Duration) ConstantDelaySchedule {
-	if duration < time.Second {
-		duration = time.Second
-	}
-	return ConstantDelaySchedule{
-		Delay: duration - time.Duration(duration.Nanoseconds())%time.Second,
-	}
-}
-
-// Next returns the next time this should be run.
-// This rounds so that the next activation time will be on the second.
-func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time {
-	return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond)
-}

backend/lib/cron/constantdelay_test.go

@@ -1,54 +0,0 @@
-package cron
-
-import (
-	"testing"
-	"time"
-)
-
-func TestConstantDelayNext(t *testing.T) {
-	tests := []struct {
-		time     string
-		delay    time.Duration
-		expected string
-	}{
-		// Simple cases
-		{"Mon Jul 9 14:45 2012", 15*time.Minute + 50*time.Nanosecond, "Mon Jul 9 15:00 2012"},
-		{"Mon Jul 9 14:59 2012", 15 * time.Minute, "Mon Jul 9 15:14 2012"},
-		{"Mon Jul 9 14:59:59 2012", 15 * time.Minute, "Mon Jul 9 15:14:59 2012"},
-
-		// Wrap around hours
-		{"Mon Jul 9 15:45 2012", 35 * time.Minute, "Mon Jul 9 16:20 2012"},
-
-		// Wrap around days
-		{"Mon Jul 9 23:46 2012", 14 * time.Minute, "Tue Jul 10 00:00 2012"},
-		{"Mon Jul 9 23:45 2012", 35 * time.Minute, "Tue Jul 10 00:20 2012"},
-		{"Mon Jul 9 23:35:51 2012", 44*time.Minute + 24*time.Second, "Tue Jul 10 00:20:15 2012"},
-		{"Mon Jul 9 23:35:51 2012", 25*time.Hour + 44*time.Minute + 24*time.Second, "Thu Jul 11 01:20:15 2012"},
-
-		// Wrap around months
-		{"Mon Jul 9 23:35 2012", 91*24*time.Hour + 25*time.Minute, "Thu Oct 9 00:00 2012"},
-
-		// Wrap around minute, hour, day, month, and year
-		{"Mon Dec 31 23:59:45 2012", 15 * time.Second, "Tue Jan 1 00:00:00 2013"},
-
-		// Round to nearest second on the delay
-		{"Mon Jul 9 14:45 2012", 15*time.Minute + 50*time.Nanosecond, "Mon Jul 9 15:00 2012"},
-
-		// Round up to 1 second if the duration is less.
-		{"Mon Jul 9 14:45:00 2012", 15 * time.Millisecond, "Mon Jul 9 14:45:01 2012"},
-
-		// Round to nearest second when calculating the next time.
-		{"Mon Jul 9 14:45:00.005 2012", 15 * time.Minute, "Mon Jul 9 15:00 2012"},
-
-		// Round to nearest second for both.
-		{"Mon Jul 9 14:45:00.005 2012", 15*time.Minute + 50*time.Nanosecond, "Mon Jul 9 15:00 2012"},
-	}
-
-	for _, c := range tests {
-		actual := Every(c.delay).Next(getTime(c.time))
-		expected := getTime(c.expected)
-		if actual != expected {
-			t.Errorf("%s, \"%s\": (expected) %v != %v (actual)", c.time, c.delay, expected, actual)
-		}
-	}
-}

backend/lib/cron/cron.go

@@ -1,350 +0,0 @@
-package cron
-
-import (
-	"context"
-	"sort"
-	"sync"
-	"time"
-)
-
-// Cron keeps track of any number of entries, invoking the associated func as
-// specified by the schedule. It may be started, stopped, and the entries may
-// be inspected while running.
-type Cron struct {
-	entries   []*Entry
-	chain     Chain
-	stop      chan struct{}
-	add       chan *Entry
-	remove    chan EntryID
-	snapshot  chan chan []Entry
-	running   bool
-	logger    Logger
-	runningMu sync.Mutex
-	location  *time.Location
-	parser    Parser
-	nextID    EntryID
-	jobWaiter sync.WaitGroup
-}
-
-// Job is an interface for submitted cron jobs.
-type Job interface {
-	Run()
-}
-
-// Schedule describes a job's duty cycle.
-type Schedule interface {
-	// Next returns the next activation time, later than the given time.
-	// Next is invoked initially, and then each time the job is run.
-	Next(time.Time) time.Time
-}
-
-// EntryID identifies an entry within a Cron instance
-type EntryID int
-
-// Entry consists of a schedule and the func to execute on that schedule.
-type Entry struct {
-	// ID is the cron-assigned ID of this entry, which may be used to look up a
-	// snapshot or remove it.
-	ID EntryID
-
-	// Schedule on which this job should be run.
-	Schedule Schedule
-
-	// Next time the job will run, or the zero time if Cron has not been
-	// started or this entry's schedule is unsatisfiable
-	Next time.Time
-
-	// Prev is the last time this job was run, or the zero time if never.
-	Prev time.Time
-
-	// WrappedJob is the thing to run when the Schedule is activated.
-	WrappedJob Job
-
-	// Job is the thing that was submitted to cron.
-	// It is kept around so that user code that needs to get at the job later,
-	// e.g. via Entries() can do so.
-	Job Job
-}
-
-// Valid returns true if this is not the zero entry.
-func (e Entry) Valid() bool { return e.ID != 0 }
-
-// byTime is a wrapper for sorting the entry array by time
-// (with zero time at the end).
-type byTime []*Entry
-
-func (s byTime) Len() int      { return len(s) }
-func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-func (s byTime) Less(i, j int) bool {
-	// Two zero times should return false.
-	// Otherwise, zero is "greater" than any other time.
-	// (To sort it at the end of the list.)
-	if s[i].Next.IsZero() {
-		return false
-	}
-	if s[j].Next.IsZero() {
-		return true
-	}
-	return s[i].Next.Before(s[j].Next)
-}
-
-// New returns a new Cron job runner, modified by the given options.
-//
-// Available Settings
-//
-//   Time Zone
-//     Description: The time zone in which schedules are interpreted
-//     Default:     time.Local
-//
-//   Parser
-//     Description: Parser converts cron spec strings into cron.Schedules.
-//     Default:     Accepts this spec: https://en.wikipedia.org/wiki/Cron
-//
-//   Chain
-//     Description: Wrap submitted jobs to customize behavior.
-//     Default:     A chain that recovers panics and logs them to stderr.
-//
-// See "cron.With*" to modify the default behavior.
-func New(opts ...Option) *Cron {
-	c := &Cron{
-		entries:   nil,
-		chain:     NewChain(),
-		add:       make(chan *Entry),
-		stop:      make(chan struct{}),
-		snapshot:  make(chan chan []Entry),
-		remove:    make(chan EntryID),
-		running:   false,
-		runningMu: sync.Mutex{},
-		logger:    DefaultLogger,
-		location:  time.Local,
-		parser:    standardParser,
-	}
-	for _, opt := range opts {
-		opt(c)
-	}
-	return c
-}
-
-// FuncJob is a wrapper that turns a func() into a cron.Job
-type FuncJob func()
-
-func (f FuncJob) Run() { f() }
-
-// AddFunc adds a func to the Cron to be run on the given schedule.
-// The spec is parsed using the time zone of this Cron instance as the default.
-// An opaque ID is returned that can be used to later remove it.
-func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) {
-	return c.AddJob(spec, FuncJob(cmd))
-}
-
-// AddJob adds a Job to the Cron to be run on the given schedule.
-// The spec is parsed using the time zone of this Cron instance as the default.
-// An opaque ID is returned that can be used to later remove it.
-func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) {
-	schedule, err := c.parser.Parse(spec)
-	if err != nil {
-		return 0, err
-	}
-	return c.Schedule(schedule, cmd), nil
-}
-
-// Schedule adds a Job to the Cron to be run on the given schedule.
-// The job is wrapped with the configured Chain.
-func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID {
-	c.runningMu.Lock()
-	defer c.runningMu.Unlock()
-	c.nextID++
-	entry := &Entry{
-		ID:         c.nextID,
-		Schedule:   schedule,
-		WrappedJob: c.chain.Then(cmd),
-		Job:        cmd,
-	}
-	if !c.running {
-		c.entries = append(c.entries, entry)
-	} else {
-		c.add <- entry
-	}
-	return entry.ID
-}
-
-// Entries returns a snapshot of the cron entries.
-func (c *Cron) Entries() []Entry {
-	c.runningMu.Lock()
-	defer c.runningMu.Unlock()
-	if c.running {
-		replyChan := make(chan []Entry, 1)
-		c.snapshot <- replyChan
-		return <-replyChan
-	}
-	return c.entrySnapshot()
-}
-
-// Location gets the time zone location
-func (c *Cron) Location() *time.Location {
-	return c.location
-}
-
-// Entry returns a snapshot of the given entry, or nil if it couldn't be found.
-func (c *Cron) Entry(id EntryID) Entry {
-	for _, entry := range c.Entries() {
-		if id == entry.ID {
-			return entry
-		}
-	}
-	return Entry{}
-}
-
-// Remove an entry from being run in the future.
-func (c *Cron) Remove(id EntryID) {
-	c.runningMu.Lock()
-	defer c.runningMu.Unlock()
-	if c.running {
-		c.remove <- id
-	} else {
-		c.removeEntry(id)
-	}
-}
-
-// Start the cron scheduler in its own goroutine, or no-op if already started.
-func (c *Cron) Start() {
-	c.runningMu.Lock()
-	defer c.runningMu.Unlock()
-	if c.running {
-		return
-	}
-	c.running = true
-	go c.run()
-}
-
-// Run the cron scheduler, or no-op if already running.
-func (c *Cron) Run() {
-	c.runningMu.Lock()
-	if c.running {
-		c.runningMu.Unlock()
-		return
-	}
-	c.running = true
-	c.runningMu.Unlock()
-	c.run()
-}
-
-// run the scheduler.. this is private just due to the need to synchronize
-// access to the 'running' state variable.
-func (c *Cron) run() {
-	c.logger.Info("start")
-
-	// Figure out the next activation times for each entry.
-	now := c.now()
-	for _, entry := range c.entries {
-		entry.Next = entry.Schedule.Next(now)
-		c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next)
-	}
-
-	for {
-		// Determine the next entry to run.
-		sort.Sort(byTime(c.entries))
-
-		var timer *time.Timer
-		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
-			// If there are no entries yet, just sleep - it still handles new entries
-			// and stop requests.
-			timer = time.NewTimer(100000 * time.Hour)
-		} else {
-			timer = time.NewTimer(c.entries[0].Next.Sub(now))
-		}
-
-		for {
-			select {
-			case now = <-timer.C:
-				now = now.In(c.location)
-				c.logger.Info("wake", "now", now)
-
-				// Run every entry whose next time was less than now
-				for _, e := range c.entries {
-					if e.Next.After(now) || e.Next.IsZero() {
-						break
-					}
-					c.startJob(e.WrappedJob)
-					e.Prev = e.Next
-					e.Next = e.Schedule.Next(now)
-					c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next)
-				}
-
-			case newEntry := <-c.add:
-				timer.Stop()
-				now = c.now()
-				newEntry.Next = newEntry.Schedule.Next(now)
-				c.entries = append(c.entries, newEntry)
-				c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next)
-
-			case replyChan := <-c.snapshot:
-				replyChan <- c.entrySnapshot()
-				continue
-
-			case <-c.stop:
-				timer.Stop()
-				c.logger.Info("stop")
-				return
-
-			case id := <-c.remove:
-				timer.Stop()
-				now = c.now()
-				c.removeEntry(id)
-				c.logger.Info("removed", "entry", id)
-			}
-
-			break
-		}
-	}
-}
-
-// startJob runs the given job in a new goroutine.
-func (c *Cron) startJob(j Job) {
-	c.jobWaiter.Add(1)
-	go func() {
-		defer c.jobWaiter.Done()
-		j.Run()
-	}()
-}
-
-// now returns current time in c location
-func (c *Cron) now() time.Time {
-	return time.Now().In(c.location)
-}
-
-// Stop stops the cron scheduler if it is running; otherwise it does nothing.
-// A context is returned so the caller can wait for running jobs to complete.
-func (c *Cron) Stop() context.Context {
-	c.runningMu.Lock()
-	defer c.runningMu.Unlock()
-	if c.running {
-		c.stop <- struct{}{}
-		c.running = false
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		c.jobWaiter.Wait()
-		cancel()
-	}()
-	return ctx
-}
-
-// entrySnapshot returns a copy of the current cron entry list.
-func (c *Cron) entrySnapshot() []Entry {
-	var entries = make([]Entry, len(c.entries))
-	for i, e := range c.entries {
-		entries[i] = *e
-	}
-	return entries
-}
-
-func (c *Cron) removeEntry(id EntryID) {
-	var entries []*Entry
-	for _, e := range c.entries {
-		if e.ID != id {
-			entries = append(entries, e)
-		}
-	}
-	c.entries = entries
-}

backend/lib/cron/cron_test.go

@@ -1,699 +0,0 @@
-package cron
-
-import (
-	"bytes"
-	"fmt"
-	"log"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-// Many tests schedule a job for every second, and then wait at most a second
-// for it to run.  This amount is just slightly larger than 1 second to
-// compensate for a few milliseconds of runtime.
-const OneSecond = 1*time.Second + 50*time.Millisecond
-
-type syncWriter struct {
-	wr bytes.Buffer
-	m  sync.Mutex
-}
-
-func (sw *syncWriter) Write(data []byte) (n int, err error) {
-	sw.m.Lock()
-	n, err = sw.wr.Write(data)
-	sw.m.Unlock()
-	return
-}
-
-func (sw *syncWriter) String() string {
-	sw.m.Lock()
-	defer sw.m.Unlock()
-	return sw.wr.String()
-}
-
-func newBufLogger(sw *syncWriter) Logger {
-	return PrintfLogger(log.New(sw, "", log.LstdFlags))
-}
-
-func TestFuncPanicRecovery(t *testing.T) {
-	var buf syncWriter
-	cron := New(WithParser(secondParser),
-		WithChain(Recover(newBufLogger(&buf))))
-	cron.Start()
-	defer cron.Stop()
-	_, _ = cron.AddFunc("* * * * * ?", func() {
-		panic("YOLO")
-	})
-	<-time.After(OneSecond)
-	if !strings.Contains(buf.String(), "YOLO") {
-		t.Error("expected a panic to be logged, got none")
-	}
-
-}
-
-type DummyJob struct{}
-
-func (d DummyJob) Run() {
-	panic("YOLO")
-}
-
-func TestJobPanicRecovery(t *testing.T) {
-	var job DummyJob
-
-	var buf syncWriter
-	cron := New(WithParser(secondParser),
-		WithChain(Recover(newBufLogger(&buf))))
-	cron.Start()
-	defer cron.Stop()
-	_, _ = cron.AddJob("* * * * * ?", job)
-
-	select {
-	case <-time.After(OneSecond):
-		if !strings.Contains(buf.String(), "YOLO") {
-			t.Error("expected a panic to be logged, got none")
-		}
-		return
-	}
-}
-
-// Start and stop cron with no entries.
-func TestNoEntries(t *testing.T) {
-	cron := newWithSeconds()
-	cron.Start()
-
-	select {
-	case <-time.After(OneSecond):
-		t.Fatal("expected cron will be stopped immediately")
-	case <-stop(cron):
-	}
-}
-
-// Start, stop, then add an entry. Verify entry doesn't run.
-func TestStopCausesJobsToNotRun(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	cron.Start()
-	cron.Stop()
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-
-	select {
-	case <-time.After(OneSecond):
-		// No job ran!
-	case <-wait(wg):
-		t.Fatal("expected stopped cron does not run any job")
-	}
-}
-
-// Add a job, start cron, expect it runs.
-func TestAddBeforeRunning(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-	cron.Start()
-	defer cron.Stop()
-
-	// Give cron 2 seconds to run our job (which is always activated).
-	select {
-	case <-time.After(OneSecond):
-		t.Fatal("expected job runs")
-	case <-wait(wg):
-	}
-}
-
-// Start cron, add a job, expect it runs.
-func TestAddWhileRunning(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	cron.Start()
-	defer cron.Stop()
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-
-	select {
-	case <-time.After(OneSecond):
-		t.Fatal("expected job runs")
-	case <-wait(wg):
-	}
-}
-
-// Test for #34. Adding a job after calling start results in multiple job invocations
-func TestAddWhileRunningWithDelay(t *testing.T) {
-	cron := newWithSeconds()
-	cron.Start()
-	defer cron.Stop()
-	time.Sleep(5 * time.Second)
-	var calls int64
-	_, _ = cron.AddFunc("* * * * * *", func() { atomic.AddInt64(&calls, 1) })
-
-	<-time.After(OneSecond)
-	if atomic.LoadInt64(&calls) != 1 {
-		t.Errorf("called %d times, expected 1\n", calls)
-	}
-}
-
-// Add a job, remove a job, start cron, expect nothing runs.
-func TestRemoveBeforeRunning(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	id, _ := cron.AddFunc("* * * * * ?", func() { wg.Done() })
-	cron.Remove(id)
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond):
-		// Success, shouldn't run
-	case <-wait(wg):
-		t.FailNow()
-	}
-}
-
-// Start cron, add a job, remove it, expect it doesn't run.
-func TestRemoveWhileRunning(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	cron.Start()
-	defer cron.Stop()
-	id, _ := cron.AddFunc("* * * * * ?", func() { wg.Done() })
-	cron.Remove(id)
-
-	select {
-	case <-time.After(OneSecond):
-	case <-wait(wg):
-		t.FailNow()
-	}
-}
-
-// Test timing with Entries.
-func TestSnapshotEntries(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := New()
-	_, _ = cron.AddFunc("@every 2s", func() { wg.Done() })
-	cron.Start()
-	defer cron.Stop()
-
-	// Cron should fire in 2 seconds. After 1 second, call Entries.
-	select {
-	case <-time.After(OneSecond):
-		cron.Entries()
-	}
-
-	// Even though Entries was called, the cron should fire at the 2 second mark.
-	select {
-	case <-time.After(OneSecond):
-		t.Error("expected job runs at 2 second mark")
-	case <-wait(wg):
-	}
-}
-
-// Test that the entries are correctly sorted.
-// Add a bunch of long-in-the-future entries, and an immediate entry, and ensure
-// that the immediate entry runs immediately.
-// Also: Test that multiple jobs run in the same instant.
-func TestMultipleEntries(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("0 0 0 1 1 ?", func() {})
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-	id1, _ := cron.AddFunc("* * * * * ?", func() { t.Fatal() })
-	id2, _ := cron.AddFunc("* * * * * ?", func() { t.Fatal() })
-	_, _ = cron.AddFunc("0 0 0 31 12 ?", func() {})
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-
-	cron.Remove(id1)
-	cron.Start()
-	cron.Remove(id2)
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond):
-		t.Error("expected job run in proper order")
-	case <-wait(wg):
-	}
-}
-
-// Test running the same job twice.
-func TestRunningJobTwice(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("0 0 0 1 1 ?", func() {})
-	_, _ = cron.AddFunc("0 0 0 31 12 ?", func() {})
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(2 * OneSecond):
-		t.Error("expected job fires 2 times")
-	case <-wait(wg):
-	}
-}
-
-func TestRunningMultipleSchedules(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("0 0 0 1 1 ?", func() {})
-	_, _ = cron.AddFunc("0 0 0 31 12 ?", func() {})
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-	cron.Schedule(Every(time.Minute), FuncJob(func() {}))
-	cron.Schedule(Every(time.Second), FuncJob(func() { wg.Done() }))
-	cron.Schedule(Every(time.Hour), FuncJob(func() {}))
-
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(2 * OneSecond):
-		t.Error("expected job fires 2 times")
-	case <-wait(wg):
-	}
-}
-
-// Test that the cron is run in the local time zone (as opposed to UTC).
-func TestLocalTimezone(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-
-	now := time.Now()
-	// FIX: Issue #205
-	// This calculation doesn't work in seconds 58 or 59.
-	// Take the easy way out and sleep.
-	if now.Second() >= 58 {
-		time.Sleep(2 * time.Second)
-		now = time.Now()
-	}
-	spec := fmt.Sprintf("%d,%d %d %d %d %d ?",
-		now.Second()+1, now.Second()+2, now.Minute(), now.Hour(), now.Day(), now.Month())
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc(spec, func() { wg.Done() })
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond * 2):
-		t.Error("expected job fires 2 times")
-	case <-wait(wg):
-	}
-}
-
-// Test that the cron is run in the given time zone (as opposed to local).
-func TestNonLocalTimezone(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-
-	loc, err := time.LoadLocation("Atlantic/Cape_Verde")
-	if err != nil {
-		fmt.Printf("Failed to load time zone Atlantic/Cape_Verde: %+v", err)
-		t.Fail()
-	}
-
-	now := time.Now().In(loc)
-	// FIX: Issue #205
-	// This calculation doesn't work in seconds 58 or 59.
-	// Take the easy way out and sleep.
-	if now.Second() >= 58 {
-		time.Sleep(2 * time.Second)
-		now = time.Now().In(loc)
-	}
-	spec := fmt.Sprintf("%d,%d %d %d %d %d ?",
-		now.Second()+1, now.Second()+2, now.Minute(), now.Hour(), now.Day(), now.Month())
-
-	cron := New(WithLocation(loc), WithParser(secondParser))
-	_, _ = cron.AddFunc(spec, func() { wg.Done() })
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond * 2):
-		t.Error("expected job fires 2 times")
-	case <-wait(wg):
-	}
-}
-
-// Test that calling stop before start silently returns without
-// blocking the stop channel.
-func TestStopWithoutStart(t *testing.T) {
-	cron := New()
-	cron.Stop()
-}
-
-type testJob struct {
-	wg   *sync.WaitGroup
-	name string
-}
-
-func (t testJob) Run() {
-	t.wg.Done()
-}
-
-// Test that adding an invalid job spec returns an error
-func TestInvalidJobSpec(t *testing.T) {
-	cron := New()
-	_, err := cron.AddJob("this will not parse", nil)
-	if err == nil {
-		t.Errorf("expected an error with invalid spec, got nil")
-	}
-}
-
-// Test blocking run method behaves as Start()
-func TestBlockingRun(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("* * * * * ?", func() { wg.Done() })
-
-	var unblockChan = make(chan struct{})
-
-	go func() {
-		cron.Run()
-		close(unblockChan)
-	}()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond):
-		t.Error("expected job fires")
-	case <-unblockChan:
-		t.Error("expected that Run() blocks")
-	case <-wait(wg):
-	}
-}
-
-// Test that double-running is a no-op
-func TestStartNoop(t *testing.T) {
-	var tickChan = make(chan struct{}, 2)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddFunc("* * * * * ?", func() {
-		tickChan <- struct{}{}
-	})
-
-	cron.Start()
-	defer cron.Stop()
-
-	// Wait for the first firing to ensure the runner is going
-	<-tickChan
-
-	cron.Start()
-
-	<-tickChan
-
-	// Fail if this job fires again in a short period, indicating a double-run
-	select {
-	case <-time.After(time.Millisecond):
-	case <-tickChan:
-		t.Error("expected job fires exactly twice")
-	}
-}
-
-// Simple test using Runnables.
-func TestJob(t *testing.T) {
-	wg := &sync.WaitGroup{}
-	wg.Add(1)
-
-	cron := newWithSeconds()
-	_, _ = cron.AddJob("0 0 0 30 Feb ?", testJob{wg, "job0"})
-	_, _ = cron.AddJob("0 0 0 1 1 ?", testJob{wg, "job1"})
-	job2, _ := cron.AddJob("* * * * * ?", testJob{wg, "job2"})
-	_, _ = cron.AddJob("1 0 0 1 1 ?", testJob{wg, "job3"})
-	cron.Schedule(Every(5*time.Second+5*time.Nanosecond), testJob{wg, "job4"})
-	job5 := cron.Schedule(Every(5*time.Minute), testJob{wg, "job5"})
-
-	// Test getting an Entry pre-Start.
-	if actualName := cron.Entry(job2).Job.(testJob).name; actualName != "job2" {
-		t.Error("wrong job retrieved:", actualName)
-	}
-	if actualName := cron.Entry(job5).Job.(testJob).name; actualName != "job5" {
-		t.Error("wrong job retrieved:", actualName)
-	}
-
-	cron.Start()
-	defer cron.Stop()
-
-	select {
-	case <-time.After(OneSecond):
-		t.FailNow()
-	case <-wait(wg):
-	}
-
-	// Ensure the entries are in the right order.
-	expecteds := []string{"job2", "job4", "job5", "job1", "job3", "job0"}
-
-	var actuals = make([]string, 0, len(cron.Entries()))
-	for _, entry := range cron.Entries() {
-		actuals = append(actuals, entry.Job.(testJob).name)
-	}
-
-	for i, expected := range expecteds {
-		if actuals[i] != expected {
-			t.Fatalf("Jobs not in the right order.  (expected) %s != %s (actual)", expecteds, actuals)
-		}
-	}
-
-	// Test getting Entries.
-	if actualName := cron.Entry(job2).Job.(testJob).name; actualName != "job2" {
-		t.Error("wrong job retrieved:", actualName)
-	}
-	if actualName := cron.Entry(job5).Job.(testJob).name; actualName != "job5" {
-		t.Error("wrong job retrieved:", actualName)
-	}
-}
-
-// Issue #206
-// Ensure that the next run of a job after removing an entry is accurate.
-func TestScheduleAfterRemoval(t *testing.T) {
-	var wg1 sync.WaitGroup
-	var wg2 sync.WaitGroup
-	wg1.Add(1)
-	wg2.Add(1)
-
-	// The first time this job is run, set a timer and remove the other job
-	// 750ms later. Correct behavior would be to still run the job again in
-	// 250ms, but the bug would cause it to run instead 1s later.
-
-	var calls int
-	var mu sync.Mutex
-
-	cron := newWithSeconds()
-	hourJob := cron.Schedule(Every(time.Hour), FuncJob(func() {}))
-	cron.Schedule(Every(time.Second), FuncJob(func() {
-		mu.Lock()
-		defer mu.Unlock()
-		switch calls {
-		case 0:
-			wg1.Done()
-			calls++
-		case 1:
-			time.Sleep(750 * time.Millisecond)
-			cron.Remove(hourJob)
-			calls++
-		case 2:
-			calls++
-			wg2.Done()
-		case 3:
-			panic("unexpected 3rd call")
-		}
-	}))
-
-	cron.Start()
-	defer cron.Stop()
-
-	// the first run might be any length of time 0 - 1s, since the schedule
-	// rounds to the second. wait for the first run to true up.
-	wg1.Wait()
-
-	select {
-	case <-time.After(2 * OneSecond):
-		t.Error("expected job fires 2 times")
-	case <-wait(&wg2):
-	}
-}
-
-type ZeroSchedule struct{}
-
-func (*ZeroSchedule) Next(time.Time) time.Time {
-	return time.Time{}
-}
-
-// Tests that job without time does not run
-func TestJobWithZeroTimeDoesNotRun(t *testing.T) {
-	cron := newWithSeconds()
-	var calls int64
-	_, _ = cron.AddFunc("* * * * * *", func() { atomic.AddInt64(&calls, 1) })
-	cron.Schedule(new(ZeroSchedule), FuncJob(func() { t.Error("expected zero task will not run") }))
-	cron.Start()
-	defer cron.Stop()
-	<-time.After(OneSecond)
-	if atomic.LoadInt64(&calls) != 1 {
-		t.Errorf("called %d times, expected 1\n", calls)
-	}
-}
-
-func TestStopAndWait(t *testing.T) {
-	t.Run("nothing running, returns immediately", func(t *testing.T) {
-		cron := newWithSeconds()
-		cron.Start()
-		ctx := cron.Stop()
-		select {
-		case <-ctx.Done():
-		case <-time.After(time.Millisecond):
-			t.Error("context was not done immediately")
-		}
-	})
-
-	t.Run("repeated calls to Stop", func(t *testing.T) {
-		cron := newWithSeconds()
-		cron.Start()
-		_ = cron.Stop()
-		time.Sleep(time.Millisecond)
-		ctx := cron.Stop()
-		select {
-		case <-ctx.Done():
-		case <-time.After(time.Millisecond):
-			t.Error("context was not done immediately")
-		}
-	})
-
-	t.Run("a couple fast jobs added, still returns immediately", func(t *testing.T) {
-		cron := newWithSeconds()
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		cron.Start()
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		time.Sleep(time.Second)
-		ctx := cron.Stop()
-		select {
-		case <-ctx.Done():
-		case <-time.After(time.Millisecond):
-			t.Error("context was not done immediately")
-		}
-	})
-
-	t.Run("a couple fast jobs and a slow job added, waits for slow job", func(t *testing.T) {
-		cron := newWithSeconds()
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		cron.Start()
-		_, _ = cron.AddFunc("* * * * * *", func() { time.Sleep(2 * time.Second) })
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		time.Sleep(time.Second)
-
-		ctx := cron.Stop()
-
-		// Verify that it is not done for at least 750ms
-		select {
-		case <-ctx.Done():
-			t.Error("context was done too quickly immediately")
-		case <-time.After(750 * time.Millisecond):
-			// expected, because the job sleeping for 1 second is still running
-		}
-
-		// Verify that it IS done in the next 500ms (giving 250ms buffer)
-		select {
-		case <-ctx.Done():
-			// expected
-		case <-time.After(1500 * time.Millisecond):
-			t.Error("context not done after job should have completed")
-		}
-	})
-
-	t.Run("repeated calls to stop, waiting for completion and after", func(t *testing.T) {
-		cron := newWithSeconds()
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		_, _ = cron.AddFunc("* * * * * *", func() { time.Sleep(2 * time.Second) })
-		cron.Start()
-		_, _ = cron.AddFunc("* * * * * *", func() {})
-		time.Sleep(time.Second)
-		ctx := cron.Stop()
-		ctx2 := cron.Stop()
-
-		// Verify that it is not done for at least 1500ms
-		select {
-		case <-ctx.Done():
-			t.Error("context was done too quickly immediately")
-		case <-ctx2.Done():
-			t.Error("context2 was done too quickly immediately")
-		case <-time.After(1500 * time.Millisecond):
-			// expected, because the job sleeping for 2 seconds is still running
-		}
-
-		// Verify that it IS done in the next 1s (giving 500ms buffer)
-		select {
-		case <-ctx.Done():
-			// expected
-		case <-time.After(time.Second):
-			t.Error("context not done after job should have completed")
-		}
-
-		// Verify that ctx2 is also done.
-		select {
-		case <-ctx2.Done():
-			// expected
-		case <-time.After(time.Millisecond):
-			t.Error("context2 not done even though context1 is")
-		}
-
-		// Verify that a new context retrieved from stop is immediately done.
-		ctx3 := cron.Stop()
-		select {
-		case <-ctx3.Done():
-			// expected
-		case <-time.After(time.Millisecond):
-			t.Error("context not done even when cron Stop is completed")
-		}
-
-	})
-}
-
-func TestMultiThreadedStartAndStop(t *testing.T) {
-	cron := New()
-	go cron.Run()
-	time.Sleep(2 * time.Millisecond)
-	cron.Stop()
-}
-
-func wait(wg *sync.WaitGroup) chan bool {
-	ch := make(chan bool)
-	go func() {
-		wg.Wait()
-		ch <- true
-	}()
-	return ch
-}
-
-func stop(cron *Cron) chan bool {
-	ch := make(chan bool)
-	go func() {
-		cron.Stop()
-		ch <- true
-	}()
-	return ch
-}
-
-// newWithSeconds returns a Cron with the seconds field enabled.
-func newWithSeconds() *Cron {
-	return New(WithParser(secondParser), WithChain())
-}

backend/lib/cron/doc.go

@@ -1,212 +0,0 @@
-/*
-Package cron implements a cron spec parser and job runner.
-
-Usage
-
-Callers may register Funcs to be invoked on a given schedule.  Cron will run
-them in their own goroutines.
-
-	c := cron.New()
-	c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") })
-	c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") })
-	c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") })
-	c.AddFunc("@hourly",      func() { fmt.Println("Every hour, starting an hour from now") })
-	c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") })
-	c.Start()
-	..
-	// Funcs are invoked in their own goroutine, asynchronously.
-	...
-	// Funcs may also be added to a running Cron
-	c.AddFunc("@daily", func() { fmt.Println("Every day") })
-	..
-	// Inspect the cron job entries' next and previous run times.
-	inspect(c.Entries())
-	..
-	c.Stop()  // Stop the scheduler (does not stop any jobs already running).
-
-CRON Expression Format
-
-A cron expression represents a set of times, using 5 space-separated fields.
-
-	Field name   | Mandatory? | Allowed values  | Allowed special characters
-	----------   | ---------- | --------------  | --------------------------
-	Minutes      | Yes        | 0-59            | * / , -
-	Hours        | Yes        | 0-23            | * / , -
-	Day of month | Yes        | 1-31            | * / , - ?
-	Month        | Yes        | 1-12 or JAN-DEC | * / , -
-	Day of week  | Yes        | 0-6 or SUN-SAT  | * / , - ?
-
-Month and Day-of-week field values are case insensitive.  "SUN", "Sun", and
-"sun" are equally accepted.
-
-The specific interpretation of the format is based on the Cron Wikipedia page:
-https://en.wikipedia.org/wiki/Cron
-
-Alternative Formats
-
-Alternative Cron expression formats support other fields like seconds. You can
-implement that by creating a custom Parser as follows.
-
-      cron.New(
-          cron.WithParser(
-              cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
-
-The most popular alternative Cron expression format is Quartz:
-http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
-
-Special Characters
-
-Asterisk ( * )
-
-The asterisk indicates that the cron expression will match for all values of the
-field; e.g., using an asterisk in the 5th field (month) would indicate every
-month.
-
-Slash ( / )
-
-Slashes are used to describe increments of ranges. For example 3-59/15 in the
-1st field (minutes) would indicate the 3rd minute of the hour and every 15
-minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...",
-that is, an increment over the largest possible range of the field.  The form
-"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the
-increment until the end of that specific range.  It does not wrap around.
-
-Comma ( , )
-
-Commas are used to separate items of a list. For example, using "MON,WED,FRI" in
-the 5th field (day of week) would mean Mondays, Wednesdays and Fridays.
-
-Hyphen ( - )
-
-Hyphens are used to define ranges. For example, 9-17 would indicate every
-hour between 9am and 5pm inclusive.
-
-Question mark ( ? )
-
-Question mark may be used instead of '*' for leaving either day-of-month or
-day-of-week blank.
-
-Predefined schedules
-
-You may use one of several pre-defined schedules in place of a cron expression.
-
-	Entry                  | Description                                | Equivalent To
-	-----                  | -----------                                | -------------
-	@yearly (or @annually) | Run once a year, midnight, Jan. 1st        | 0 0 1 1 *
-	@monthly               | Run once a month, midnight, first of month | 0 0 1 * *
-	@weekly                | Run once a week, midnight between Sat/Sun  | 0 0 * * 0
-	@daily (or @midnight)  | Run once a day, midnight                   | 0 0 * * *
-	@hourly                | Run once an hour, beginning of hour        | 0 * * * *
-
-Intervals
-
-You may also schedule a job to execute at fixed intervals, starting at the time it's added
-or cron is run. This is supported by formatting the cron spec like this:
-
-    @every <duration>
-
-where "duration" is a string accepted by time.ParseDuration
-(http://golang.org/pkg/time/#ParseDuration).
-
-For example, "@every 1h30m10s" would indicate a schedule that activates after
-1 hour, 30 minutes, 10 seconds, and then every interval after that.
-
-Note: The interval does not take the job runtime into account.  For example,
-if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes,
-it will have only 2 minutes of idle time between each run.
-
-Time zones
-
-By default, all interpretation and scheduling is done in the machine's local
-time zone (time.Local). You can specify a different time zone on construction:
-
-      cron.New(
-          cron.WithLocation(time.UTC))
-
-Individual cron schedules may also override the time zone they are to be
-interpreted in by providing an additional space-separated field at the beginning
-of the cron spec, of the form "CRON_TZ=Asia/Tokyo".
-
-For example:
-
-	# Runs at 6am in time.Local
-	cron.New().AddFunc("0 6 * * ?", ...)
-
-	# Runs at 6am in America/New_York
-	nyc, _ := time.LoadLocation("America/New_York")
-	c := cron.New(cron.WithLocation(nyc))
-	c.AddFunc("0 6 * * ?", ...)
-
-	# Runs at 6am in Asia/Tokyo
-	cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
-
-	# Runs at 6am in Asia/Tokyo
-	c := cron.New(cron.WithLocation(nyc))
-	c.SetLocation("America/New_York")
-	c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
-
-The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility.
-
-Be aware that jobs scheduled during daylight-savings leap-ahead transitions will
-not be run!
-
-Job Wrappers / Chain
-
-A Cron runner may be configured with a chain of job wrappers to add
-cross-cutting functionality to all submitted jobs. For example, they may be used
-to achieve the following effects:
-
-  - Recover any panics from jobs (activated by default)
-  - Delay a job's execution if the previous run hasn't completed yet
-  - Skip a job's execution if the previous run hasn't completed yet
-  - Log each job's invocations
-
-Install wrappers for all jobs added to a cron using the `cron.WithChain` option:
-
-	cron.New(cron.WithChain(
-		cron.SkipIfStillRunning(logger),
-	))
-
-Install wrappers for individual jobs by explicitly wrapping them:
-
-	job = cron.NewChain(
-		cron.SkipIfStillRunning(logger),
-	).Then(job)
-
-Thread safety
-
-Since the Cron service runs concurrently with the calling code, some amount of
-care must be taken to ensure proper synchronization.
-
-All cron methods are designed to be correctly synchronized as long as the caller
-ensures that invocations have a clear happens-before ordering between them.
-
-Logging
-
-Cron defines a Logger interface that is a subset of the one defined in
-github.com/go-logr/logr. It has two logging levels (Info and Error), and
-parameters are key/value pairs. This makes it possible for cron logging to plug
-into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided
-to wrap the standard library *log.Logger.
-
-For additional insight into Cron operations, verbose logging may be activated
-which will record job runs, scheduling decisions, and added or removed jobs.
-Activate it with a one-off logger as follows:
-
-	cron.New(
-		cron.WithLogger(
-			cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))))
-
-
-Implementation
-
-Cron entries are stored in an array, sorted by their next activation time.  Cron
-sleeps until the next job is due to be run.
-
-Upon waking:
- - it runs each entry that is active on that second
- - it calculates the next run times for the jobs that were run
- - it re-sorts the array of entries by next activation time.
- - it goes to sleep until the soonest job.
-*/
-package cron

backend/lib/cron/logger.go

@@ -1,86 +0,0 @@
-package cron
-
-import (
-	"io/ioutil"
-	"log"
-	"os"
-	"strings"
-	"time"
-)
-
-// DefaultLogger is used by Cron if none is specified.
-var DefaultLogger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))
-
-// DiscardLogger can be used by callers to discard all log messages.
-var DiscardLogger = PrintfLogger(log.New(ioutil.Discard, "", 0))
-
-// Logger is the interface used in this package for logging, so that any backend
-// can be plugged in. It is a subset of the github.com/go-logr/logr interface.
-type Logger interface {
-	// Info logs routine messages about cron's operation.
-	Info(msg string, keysAndValues ...interface{})
-	// Error logs an error condition.
-	Error(err error, msg string, keysAndValues ...interface{})
-}
-
-// PrintfLogger wraps a Printf-based logger (such as the standard library "log")
-// into an implementation of the Logger interface which logs errors only.
-func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
-	return printfLogger{l, false}
-}
-
-// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library
-// "log") into an implementation of the Logger interface which logs everything.
-func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
-	return printfLogger{l, true}
-}
-
-type printfLogger struct {
-	logger  interface{ Printf(string, ...interface{}) }
-	logInfo bool
-}
-
-func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) {
-	if pl.logInfo {
-		keysAndValues = formatTimes(keysAndValues)
-		pl.logger.Printf(
-			formatString(len(keysAndValues)),
-			append([]interface{}{msg}, keysAndValues...)...)
-	}
-}
-
-func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) {
-	keysAndValues = formatTimes(keysAndValues)
-	pl.logger.Printf(
-		formatString(len(keysAndValues)+2),
-		append([]interface{}{msg, "error", err}, keysAndValues...)...)
-}
-
-// formatString returns a logfmt-like format string for the number of
-// key/values.
-func formatString(numKeysAndValues int) string {
-	var sb strings.Builder
-	sb.WriteString("%s")
-	if numKeysAndValues > 0 {
-		sb.WriteString(", ")
-	}
-	for i := 0; i < numKeysAndValues/2; i++ {
-		if i > 0 {
-			sb.WriteString(", ")
-		}
-		sb.WriteString("%v=%v")
-	}
-	return sb.String()
-}
-
-// formatTimes formats any time.Time values as RFC3339.
-func formatTimes(keysAndValues []interface{}) []interface{} {
-	var formattedArgs []interface{}
-	for _, arg := range keysAndValues {
-		if t, ok := arg.(time.Time); ok {
-			arg = t.Format(time.RFC3339)
-		}
-		formattedArgs = append(formattedArgs, arg)
-	}
-	return formattedArgs
-}

backend/lib/cron/option.go

@@ -1,45 +0,0 @@
-package cron
-
-import (
-	"time"
-)
-
-// Option represents a modification to the default behavior of a Cron.
-type Option func(*Cron)
-
-// WithLocation overrides the timezone of the cron instance.
-func WithLocation(loc *time.Location) Option {
-	return func(c *Cron) {
-		c.location = loc
-	}
-}
-
-// WithSeconds overrides the parser used for interpreting job schedules to
-// include a seconds field as the first one.
-func WithSeconds() Option {
-	return WithParser(NewParser(
-		Second | Minute | Hour | Dom | Month | Dow | Descriptor,
-	))
-}
-
-// WithParser overrides the parser used for interpreting job schedules.
-func WithParser(p Parser) Option {
-	return func(c *Cron) {
-		c.parser = p
-	}
-}
-
-// WithChain specifies Job wrappers to apply to all jobs added to this cron.
-// Refer to the Chain* functions in this package for provided wrappers.
-func WithChain(wrappers ...JobWrapper) Option {
-	return func(c *Cron) {
-		c.chain = NewChain(wrappers...)
-	}
-}
-
-// WithLogger uses the provided logger.
-func WithLogger(logger Logger) Option {
-	return func(c *Cron) {
-		c.logger = logger
-	}
-}

backend/lib/cron/option_test.go

@@ -1,42 +0,0 @@
-package cron
-
-import (
-	"log"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestWithLocation(t *testing.T) {
-	c := New(WithLocation(time.UTC))
-	if c.location != time.UTC {
-		t.Errorf("expected UTC, got %v", c.location)
-	}
-}
-
-func TestWithParser(t *testing.T) {
-	var parser = NewParser(Dow)
-	c := New(WithParser(parser))
-	if c.parser != parser {
-		t.Error("expected provided parser")
-	}
-}
-
-func TestWithVerboseLogger(t *testing.T) {
-	var buf syncWriter
-	var logger = log.New(&buf, "", log.LstdFlags)
-	c := New(WithLogger(VerbosePrintfLogger(logger)))
-	if c.logger.(printfLogger).logger != logger {
-		t.Error("expected provided logger")
-	}
-
-	_, _ = c.AddFunc("@every 1s", func() {})
-	c.Start()
-	time.Sleep(OneSecond)
-	c.Stop()
-	out := buf.String()
-	if !strings.Contains(out, "schedule,") ||
-		!strings.Contains(out, "run,") {
-		t.Error("expected to see some actions, got:", out)
-	}
-}

backend/lib/cron/parser.go

@@ -1,434 +0,0 @@
-package cron
-
-import (
-	"fmt"
-	"math"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// Configuration options for creating a parser. Most options specify which
-// fields should be included, while others enable features. If a field is not
-// included the parser will assume a default value. These options do not change
-// the order fields are parse in.
-type ParseOption int
-
-const (
-	Second         ParseOption = 1 << iota // Seconds field, default 0
-	SecondOptional                         // Optional seconds field, default 0
-	Minute                                 // Minutes field, default 0
-	Hour                                   // Hours field, default 0
-	Dom                                    // Day of month field, default *
-	Month                                  // Month field, default *
-	Dow                                    // Day of week field, default *
-	DowOptional                            // Optional day of week field, default *
-	Descriptor                             // Allow descriptors such as @monthly, @weekly, etc.
-)
-
-var places = []ParseOption{
-	Second,
-	Minute,
-	Hour,
-	Dom,
-	Month,
-	Dow,
-}
-
-var defaults = []string{
-	"0",
-	"0",
-	"0",
-	"*",
-	"*",
-	"*",
-}
-
-// A custom Parser that can be configured.
-type Parser struct {
-	options ParseOption
-}
-
-// NewParser creates a Parser with custom options.
-//
-// It panics if more than one Optional is given, since it would be impossible to
-// correctly infer which optional is provided or missing in general.
-//
-// Examples
-//
-//  // Standard parser without descriptors
-//  specParser := NewParser(Minute | Hour | Dom | Month | Dow)
-//  sched, err := specParser.Parse("0 0 15 */3 *")
-//
-//  // Same as above, just excludes time fields
-//  subsParser := NewParser(Dom | Month | Dow)
-//  sched, err := specParser.Parse("15 */3 *")
-//
-//  // Same as above, just makes Dow optional
-//  subsParser := NewParser(Dom | Month | DowOptional)
-//  sched, err := specParser.Parse("15 */3")
-//
-func NewParser(options ParseOption) Parser {
-	optionals := 0
-	if options&DowOptional > 0 {
-		optionals++
-	}
-	if options&SecondOptional > 0 {
-		optionals++
-	}
-	if optionals > 1 {
-		panic("multiple optionals may not be configured")
-	}
-	return Parser{options}
-}
-
-// Parse returns a new crontab schedule representing the given spec.
-// It returns a descriptive error if the spec is not valid.
-// It accepts crontab specs and features configured by NewParser.
-func (p Parser) Parse(spec string) (Schedule, error) {
-	if len(spec) == 0 {
-		return nil, fmt.Errorf("empty spec string")
-	}
-
-	// Extract timezone if present
-	var loc = time.Local
-	if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") {
-		var err error
-		i := strings.Index(spec, " ")
-		eq := strings.Index(spec, "=")
-		if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil {
-			return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err)
-		}
-		spec = strings.TrimSpace(spec[i:])
-	}
-
-	// Handle named schedules (descriptors), if configured
-	if strings.HasPrefix(spec, "@") {
-		if p.options&Descriptor == 0 {
-			return nil, fmt.Errorf("parser does not accept descriptors: %v", spec)
-		}
-		return parseDescriptor(spec, loc)
-	}
-
-	// Split on whitespace.
-	fields := strings.Fields(spec)
-
-	// Validate & fill in any omitted or optional fields
-	var err error
-	fields, err = normalizeFields(fields, p.options)
-	if err != nil {
-		return nil, err
-	}
-
-	field := func(field string, r bounds) uint64 {
-		if err != nil {
-			return 0
-		}
-		var bits uint64
-		bits, err = getField(field, r)
-		return bits
-	}
-
-	var (
-		second     = field(fields[0], seconds)
-		minute     = field(fields[1], minutes)
-		hour       = field(fields[2], hours)
-		dayofmonth = field(fields[3], dom)
-		month      = field(fields[4], months)
-		dayofweek  = field(fields[5], dow)
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &SpecSchedule{
-		Second:   second,
-		Minute:   minute,
-		Hour:     hour,
-		Dom:      dayofmonth,
-		Month:    month,
-		Dow:      dayofweek,
-		Location: loc,
-	}, nil
-}
-
-// normalizeFields takes a subset set of the time fields and returns the full set
-// with defaults (zeroes) populated for unset fields.
-//
-// As part of performing this function, it also validates that the provided
-// fields are compatible with the configured options.
-func normalizeFields(fields []string, options ParseOption) ([]string, error) {
-	// Validate optionals & add their field to options
-	optionals := 0
-	if options&SecondOptional > 0 {
-		options |= Second
-		optionals++
-	}
-	if options&DowOptional > 0 {
-		options |= Dow
-		optionals++
-	}
-	if optionals > 1 {
-		return nil, fmt.Errorf("multiple optionals may not be configured")
-	}
-
-	// Figure out how many fields we need
-	max := 0
-	for _, place := range places {
-		if options&place > 0 {
-			max++
-		}
-	}
-	min := max - optionals
-
-	// Validate number of fields
-	if count := len(fields); count < min || count > max {
-		if min == max {
-			return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields)
-		}
-		return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields)
-	}
-
-	// Populate the optional field if not provided
-	if min < max && len(fields) == min {
-		switch {
-		case options&DowOptional > 0:
-			fields = append(fields, defaults[5]) // TODO: improve access to default
-		case options&SecondOptional > 0:
-			fields = append([]string{defaults[0]}, fields...)
-		default:
-			return nil, fmt.Errorf("unknown optional field")
-		}
-	}
-
-	// Populate all fields not part of options with their defaults
-	n := 0
-	expandedFields := make([]string, len(places))
-	copy(expandedFields, defaults)
-	for i, place := range places {
-		if options&place > 0 {
-			expandedFields[i] = fields[n]
-			n++
-		}
-	}
-	return expandedFields, nil
-}
-
-var standardParser = NewParser(
-	Minute | Hour | Dom | Month | Dow | Descriptor,
-)
-
-// ParseStandard returns a new crontab schedule representing the given
-// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries
-// representing: minute, hour, day of month, month and day of week, in that
-// order. It returns a descriptive error if the spec is not valid.
-//
-// It accepts
-//   - Standard crontab specs, e.g. "* * * * ?"
-//   - Descriptors, e.g. "@midnight", "@every 1h30m"
-func ParseStandard(standardSpec string) (Schedule, error) {
-	return standardParser.Parse(standardSpec)
-}
-
-// getField returns an Int with the bits set representing all of the times that
-// the field represents or error parsing field value.  A "field" is a comma-separated
-// list of "ranges".
-func getField(field string, r bounds) (uint64, error) {
-	var bits uint64
-	ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
-	for _, expr := range ranges {
-		bit, err := getRange(expr, r)
-		if err != nil {
-			return bits, err
-		}
-		bits |= bit
-	}
-	return bits, nil
-}
-
-// getRange returns the bits indicated by the given expression:
-//   number | number "-" number [ "/" number ]
-// or error parsing range.
-func getRange(expr string, r bounds) (uint64, error) {
-	var (
-		start, end, step uint
-		rangeAndStep     = strings.Split(expr, "/")
-		lowAndHigh       = strings.Split(rangeAndStep[0], "-")
-		singleDigit      = len(lowAndHigh) == 1
-		err              error
-	)
-
-	var extra uint64
-	if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
-		start = r.min
-		end = r.max
-		extra = starBit
-	} else {
-		start, err = parseIntOrName(lowAndHigh[0], r.names)
-		if err != nil {
-			return 0, err
-		}
-		switch len(lowAndHigh) {
-		case 1:
-			end = start
-		case 2:
-			end, err = parseIntOrName(lowAndHigh[1], r.names)
-			if err != nil {
-				return 0, err
-			}
-		default:
-			return 0, fmt.Errorf("too many hyphens: %s", expr)
-		}
-	}
-
-	switch len(rangeAndStep) {
-	case 1:
-		step = 1
-	case 2:
-		step, err = mustParseInt(rangeAndStep[1])
-		if err != nil {
-			return 0, err
-		}
-
-		// Special handling: "N/step" means "N-max/step".
-		if singleDigit {
-			end = r.max
-		}
-		if step > 1 {
-			extra = 0
-		}
-	default:
-		return 0, fmt.Errorf("too many slashes: %s", expr)
-	}
-
-	if start < r.min {
-		return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
-	}
-	if end > r.max {
-		return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
-	}
-	if start > end {
-		return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
-	}
-	if step == 0 {
-		return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
-	}
-
-	return getBits(start, end, step) | extra, nil
-}
-
-// parseIntOrName returns the (possibly-named) integer contained in expr.
-func parseIntOrName(expr string, names map[string]uint) (uint, error) {
-	if names != nil {
-		if namedInt, ok := names[strings.ToLower(expr)]; ok {
-			return namedInt, nil
-		}
-	}
-	return mustParseInt(expr)
-}
-
-// mustParseInt parses the given expression as an int or returns an error.
-func mustParseInt(expr string) (uint, error) {
-	num, err := strconv.Atoi(expr)
-	if err != nil {
-		return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
-	}
-	if num < 0 {
-		return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
-	}
-
-	return uint(num), nil
-}
-
-// getBits sets all bits in the range [min, max], modulo the given step size.
-func getBits(min, max, step uint) uint64 {
-	var bits uint64
-
-	// If step is 1, use shifts.
-	if step == 1 {
-		return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
-	}
-
-	// Else, use a simple loop.
-	for i := min; i <= max; i += step {
-		bits |= 1 << i
-	}
-	return bits
-}
-
-// all returns all bits within the given bounds.  (plus the star bit)
-func all(r bounds) uint64 {
-	return getBits(r.min, r.max, 1) | starBit
-}
-
-// parseDescriptor returns a predefined schedule for the expression, or error if none matches.
-func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) {
-	switch descriptor {
-	case "@yearly", "@annually":
-		return &SpecSchedule{
-			Second:   1 << seconds.min,
-			Minute:   1 << minutes.min,
-			Hour:     1 << hours.min,
-			Dom:      1 << dom.min,
-			Month:    1 << months.min,
-			Dow:      all(dow),
-			Location: loc,
-		}, nil
-
-	case "@monthly":
-		return &SpecSchedule{
-			Second:   1 << seconds.min,
-			Minute:   1 << minutes.min,
-			Hour:     1 << hours.min,
-			Dom:      1 << dom.min,
-			Month:    all(months),
-			Dow:      all(dow),
-			Location: loc,
-		}, nil
-
-	case "@weekly":
-		return &SpecSchedule{
-			Second:   1 << seconds.min,
-			Minute:   1 << minutes.min,
-			Hour:     1 << hours.min,
-			Dom:      all(dom),
-			Month:    all(months),
-			Dow:      1 << dow.min,
-			Location: loc,
-		}, nil
-
-	case "@daily", "@midnight":
-		return &SpecSchedule{
-			Second:   1 << seconds.min,
-			Minute:   1 << minutes.min,
-			Hour:     1 << hours.min,
-			Dom:      all(dom),
-			Month:    all(months),
-			Dow:      all(dow),
-			Location: loc,
-		}, nil
-
-	case "@hourly":
-		return &SpecSchedule{
-			Second:   1 << seconds.min,
-			Minute:   1 << minutes.min,
-			Hour:     all(hours),
-			Dom:      all(dom),
-			Month:    all(months),
-			Dow:      all(dow),
-			Location: loc,
-		}, nil
-
-	}
-
-	const every = "@every "
-	if strings.HasPrefix(descriptor, every) {
-		duration, err := time.ParseDuration(descriptor[len(every):])
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err)
-		}
-		return Every(duration), nil
-	}
-
-	return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor)
-}

backend/lib/cron/parser_test.go

@@ -1,384 +0,0 @@
-package cron
-
-import (
-	"reflect"
-	"strings"
-	"testing"
-	"time"
-)
-
-var secondParser = NewParser(Second | Minute | Hour | Dom | Month | DowOptional | Descriptor)
-
-func TestRange(t *testing.T) {
-	zero := uint64(0)
-	ranges := []struct {
-		expr     string
-		min, max uint
-		expected uint64
-		err      string
-	}{
-		{"5", 0, 7, 1 << 5, ""},
-		{"0", 0, 7, 1 << 0, ""},
-		{"7", 0, 7, 1 << 7, ""},
-
-		{"5-5", 0, 7, 1 << 5, ""},
-		{"5-6", 0, 7, 1<<5 | 1<<6, ""},
-		{"5-7", 0, 7, 1<<5 | 1<<6 | 1<<7, ""},
-
-		{"5-6/2", 0, 7, 1 << 5, ""},
-		{"5-7/2", 0, 7, 1<<5 | 1<<7, ""},
-		{"5-7/1", 0, 7, 1<<5 | 1<<6 | 1<<7, ""},
-
-		{"*", 1, 3, 1<<1 | 1<<2 | 1<<3 | starBit, ""},
-		{"*/2", 1, 3, 1<<1 | 1<<3, ""},
-
-		{"5--5", 0, 0, zero, "too many hyphens"},
-		{"jan-x", 0, 0, zero, "failed to parse int from"},
-		{"2-x", 1, 5, zero, "failed to parse int from"},
-		{"*/-12", 0, 0, zero, "negative number"},
-		{"*//2", 0, 0, zero, "too many slashes"},
-		{"1", 3, 5, zero, "below minimum"},
-		{"6", 3, 5, zero, "above maximum"},
-		{"5-3", 3, 5, zero, "beyond end of range"},
-		{"*/0", 0, 0, zero, "should be a positive number"},
-	}
-
-	for _, c := range ranges {
-		actual, err := getRange(c.expr, bounds{c.min, c.max, nil})
-		if len(c.err) != 0 && (err == nil || !strings.Contains(err.Error(), c.err)) {
-			t.Errorf("%s => expected %v, got %v", c.expr, c.err, err)
-		}
-		if len(c.err) == 0 && err != nil {
-			t.Errorf("%s => unexpected error %v", c.expr, err)
-		}
-		if actual != c.expected {
-			t.Errorf("%s => expected %d, got %d", c.expr, c.expected, actual)
-		}
-	}
-}
-
-func TestField(t *testing.T) {
-	fields := []struct {
-		expr     string
-		min, max uint
-		expected uint64
-	}{
-		{"5", 1, 7, 1 << 5},
-		{"5,6", 1, 7, 1<<5 | 1<<6},
-		{"5,6,7", 1, 7, 1<<5 | 1<<6 | 1<<7},
-		{"1,5-7/2,3", 1, 7, 1<<1 | 1<<5 | 1<<7 | 1<<3},
-	}
-
-	for _, c := range fields {
-		actual, _ := getField(c.expr, bounds{c.min, c.max, nil})
-		if actual != c.expected {
-			t.Errorf("%s => expected %d, got %d", c.expr, c.expected, actual)
-		}
-	}
-}
-
-func TestAll(t *testing.T) {
-	allBits := []struct {
-		r        bounds
-		expected uint64
-	}{
-		{minutes, 0xfffffffffffffff}, // 0-59: 60 ones
-		{hours, 0xffffff},            // 0-23: 24 ones
-		{dom, 0xfffffffe},            // 1-31: 31 ones, 1 zero
-		{months, 0x1ffe},             // 1-12: 12 ones, 1 zero
-		{dow, 0x7f},                  // 0-6: 7 ones
-	}
-
-	for _, c := range allBits {
-		actual := all(c.r) // all() adds the starBit, so compensate for that..
-		if c.expected|starBit != actual {
-			t.Errorf("%d-%d/%d => expected %b, got %b",
-				c.r.min, c.r.max, 1, c.expected|starBit, actual)
-		}
-	}
-}
-
-func TestBits(t *testing.T) {
-	bits := []struct {
-		min, max, step uint
-		expected       uint64
-	}{
-		{0, 0, 1, 0x1},
-		{1, 1, 1, 0x2},
-		{1, 5, 2, 0x2a}, // 101010
-		{1, 4, 2, 0xa},  // 1010
-	}
-
-	for _, c := range bits {
-		actual := getBits(c.min, c.max, c.step)
-		if c.expected != actual {
-			t.Errorf("%d-%d/%d => expected %b, got %b",
-				c.min, c.max, c.step, c.expected, actual)
-		}
-	}
-}
-
-func TestParseScheduleErrors(t *testing.T) {
-	var tests = []struct{ expr, err string }{
-		{"* 5 j * * *", "failed to parse int from"},
-		{"@every Xm", "failed to parse duration"},
-		{"@unrecognized", "unrecognized descriptor"},
-		{"* * * *", "expected 5 to 6 fields"},
-		{"", "empty spec string"},
-	}
-	for _, c := range tests {
-		actual, err := secondParser.Parse(c.expr)
-		if err == nil || !strings.Contains(err.Error(), c.err) {
-			t.Errorf("%s => expected %v, got %v", c.expr, c.err, err)
-		}
-		if actual != nil {
-			t.Errorf("expected nil schedule on error, got %v", actual)
-		}
-	}
-}
-
-func TestParseSchedule(t *testing.T) {
-	tokyo, _ := time.LoadLocation("Asia/Tokyo")
-	entries := []struct {
-		parser   Parser
-		expr     string
-		expected Schedule
-	}{
-		{secondParser, "0 5 * * * *", every5min(time.Local)},
-		{standardParser, "5 * * * *", every5min(time.Local)},
-		{secondParser, "CRON_TZ=UTC  0 5 * * * *", every5min(time.UTC)},
-		{standardParser, "CRON_TZ=UTC  5 * * * *", every5min(time.UTC)},
-		{secondParser, "CRON_TZ=Asia/Tokyo 0 5 * * * *", every5min(tokyo)},
-		{secondParser, "@every 5m", ConstantDelaySchedule{5 * time.Minute}},
-		{secondParser, "@midnight", midnight(time.Local)},
-		{secondParser, "TZ=UTC  @midnight", midnight(time.UTC)},
-		{secondParser, "TZ=Asia/Tokyo @midnight", midnight(tokyo)},
-		{secondParser, "@yearly", annual(time.Local)},
-		{secondParser, "@annually", annual(time.Local)},
-		{
-			parser: secondParser,
-			expr:   "* 5 * * * *",
-			expected: &SpecSchedule{
-				Second:   all(seconds),
-				Minute:   1 << 5,
-				Hour:     all(hours),
-				Dom:      all(dom),
-				Month:    all(months),
-				Dow:      all(dow),
-				Location: time.Local,
-			},
-		},
-	}
-
-	for _, c := range entries {
-		actual, err := c.parser.Parse(c.expr)
-		if err != nil {
-			t.Errorf("%s => unexpected error %v", c.expr, err)
-		}
-		if !reflect.DeepEqual(actual, c.expected) {
-			t.Errorf("%s => expected %b, got %b", c.expr, c.expected, actual)
-		}
-	}
-}
-
-func TestOptionalSecondSchedule(t *testing.T) {
-	parser := NewParser(SecondOptional | Minute | Hour | Dom | Month | Dow | Descriptor)
-	entries := []struct {
-		expr     string
-		expected Schedule
-	}{
-		{"0 5 * * * *", every5min(time.Local)},
-		{"5 5 * * * *", every5min5s(time.Local)},
-		{"5 * * * *", every5min(time.Local)},
-	}
-
-	for _, c := range entries {
-		actual, err := parser.Parse(c.expr)
-		if err != nil {
-			t.Errorf("%s => unexpected error %v", c.expr, err)
-		}
-		if !reflect.DeepEqual(actual, c.expected) {
-			t.Errorf("%s => expected %b, got %b", c.expr, c.expected, actual)
-		}
-	}
-}
-
-func TestNormalizeFields(t *testing.T) {
-	tests := []struct {
-		name     string
-		input    []string
-		options  ParseOption
-		expected []string
-	}{
-		{
-			"AllFields_NoOptional",
-			[]string{"0", "5", "*", "*", "*", "*"},
-			Second | Minute | Hour | Dom | Month | Dow | Descriptor,
-			[]string{"0", "5", "*", "*", "*", "*"},
-		},
-		{
-			"AllFields_SecondOptional_Provided",
-			[]string{"0", "5", "*", "*", "*", "*"},
-			SecondOptional | Minute | Hour | Dom | Month | Dow | Descriptor,
-			[]string{"0", "5", "*", "*", "*", "*"},
-		},
-		{
-			"AllFields_SecondOptional_NotProvided",
-			[]string{"5", "*", "*", "*", "*"},
-			SecondOptional | Minute | Hour | Dom | Month | Dow | Descriptor,
-			[]string{"0", "5", "*", "*", "*", "*"},
-		},
-		{
-			"SubsetFields_NoOptional",
-			[]string{"5", "15", "*"},
-			Hour | Dom | Month,
-			[]string{"0", "0", "5", "15", "*", "*"},
-		},
-		{
-			"SubsetFields_DowOptional_Provided",
-			[]string{"5", "15", "*", "4"},
-			Hour | Dom | Month | DowOptional,
-			[]string{"0", "0", "5", "15", "*", "4"},
-		},
-		{
-			"SubsetFields_DowOptional_NotProvided",
-			[]string{"5", "15", "*"},
-			Hour | Dom | Month | DowOptional,
-			[]string{"0", "0", "5", "15", "*", "*"},
-		},
-		{
-			"SubsetFields_SecondOptional_NotProvided",
-			[]string{"5", "15", "*"},
-			SecondOptional | Hour | Dom | Month,
-			[]string{"0", "0", "5", "15", "*", "*"},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			actual, err := normalizeFields(test.input, test.options)
-			if err != nil {
-				t.Errorf("unexpected error: %v", err)
-			}
-			if !reflect.DeepEqual(actual, test.expected) {
-				t.Errorf("expected %v, got %v", test.expected, actual)
-			}
-		})
-	}
-}
-
-func TestNormalizeFields_Errors(t *testing.T) {
-	tests := []struct {
-		name    string
-		input   []string
-		options ParseOption
-		err     string
-	}{
-		{
-			"TwoOptionals",
-			[]string{"0", "5", "*", "*", "*", "*"},
-			SecondOptional | Minute | Hour | Dom | Month | DowOptional,
-			"",
-		},
-		{
-			"TooManyFields",
-			[]string{"0", "5", "*", "*"},
-			SecondOptional | Minute | Hour,
-			"",
-		},
-		{
-			"NoFields",
-			[]string{},
-			SecondOptional | Minute | Hour,
-			"",
-		},
-		{
-			"TooFewFields",
-			[]string{"*"},
-			SecondOptional | Minute | Hour,
-			"",
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			actual, err := normalizeFields(test.input, test.options)
-			if err == nil {
-				t.Errorf("expected an error, got none. results: %v", actual)
-				return
-			}
-			if !strings.Contains(err.Error(), test.err) {
-				t.Errorf("expected error %q, got %q", test.err, err.Error())
-			}
-		})
-	}
-}
-
-func TestStandardSpecSchedule(t *testing.T) {
-	entries := []struct {
-		expr     string
-		expected Schedule
-		err      string
-	}{
-		{
-			expr:     "5 * * * *",
-			expected: &SpecSchedule{1 << seconds.min, 1 << 5, all(hours), all(dom), all(months), all(dow), time.Local},
-		},
-		{
-			expr:     "@every 5m",
-			expected: ConstantDelaySchedule{time.Duration(5) * time.Minute},
-		},
-		{
-			expr: "5 j * * *",
-			err:  "failed to parse int from",
-		},
-		{
-			expr: "* * * *",
-			err:  "expected exactly 5 fields",
-		},
-	}
-
-	for _, c := range entries {
-		actual, err := ParseStandard(c.expr)
-		if len(c.err) != 0 && (err == nil || !strings.Contains(err.Error(), c.err)) {
-			t.Errorf("%s => expected %v, got %v", c.expr, c.err, err)
-		}
-		if len(c.err) == 0 && err != nil {
-			t.Errorf("%s => unexpected error %v", c.expr, err)
-		}
-		if !reflect.DeepEqual(actual, c.expected) {
-			t.Errorf("%s => expected %b, got %b", c.expr, c.expected, actual)
-		}
-	}
-}
-
-func TestNoDescriptorParser(t *testing.T) {
-	parser := NewParser(Minute | Hour)
-	_, err := parser.Parse("@every 1m")
-	if err == nil {
-		t.Error("expected an error, got none")
-	}
-}
-
-func every5min(loc *time.Location) *SpecSchedule {
-	return &SpecSchedule{1 << 0, 1 << 5, all(hours), all(dom), all(months), all(dow), loc}
-}
-
-func every5min5s(loc *time.Location) *SpecSchedule {
-	return &SpecSchedule{1 << 5, 1 << 5, all(hours), all(dom), all(months), all(dow), loc}
-}
-
-func midnight(loc *time.Location) *SpecSchedule {
-	return &SpecSchedule{1, 1, 1, all(dom), all(months), all(dow), loc}
-}
-
-func annual(loc *time.Location) *SpecSchedule {
-	return &SpecSchedule{
-		Second:   1 << seconds.min,
-		Minute:   1 << minutes.min,
-		Hour:     1 << hours.min,
-		Dom:      1 << dom.min,
-		Month:    1 << months.min,
-		Dow:      all(dow),
-		Location: loc,
-	}
-}

backend/lib/cron/spec.go

@@ -1,188 +0,0 @@
-package cron
-
-import "time"
-
-// SpecSchedule specifies a duty cycle (to the second granularity), based on a
-// traditional crontab specification. It is computed initially and stored as bit sets.
-type SpecSchedule struct {
-	Second, Minute, Hour, Dom, Month, Dow uint64
-
-	// Override location for this schedule.
-	Location *time.Location
-}
-
-// bounds provides a range of acceptable values (plus a map of name to value).
-type bounds struct {
-	min, max uint
-	names    map[string]uint
-}
-
-// The bounds for each field.
-var (
-	seconds = bounds{0, 59, nil}
-	minutes = bounds{0, 59, nil}
-	hours   = bounds{0, 23, nil}
-	dom     = bounds{1, 31, nil}
-	months  = bounds{1, 12, map[string]uint{
-		"jan": 1,
-		"feb": 2,
-		"mar": 3,
-		"apr": 4,
-		"may": 5,
-		"jun": 6,
-		"jul": 7,
-		"aug": 8,
-		"sep": 9,
-		"oct": 10,
-		"nov": 11,
-		"dec": 12,
-	}}
-	dow = bounds{0, 6, map[string]uint{
-		"sun": 0,
-		"mon": 1,
-		"tue": 2,
-		"wed": 3,
-		"thu": 4,
-		"fri": 5,
-		"sat": 6,
-	}}
-)
-
-const (
-	// Set the top bit if a star was included in the expression.
-	starBit = 1 << 63
-)
-
-// Next returns the next time this schedule is activated, greater than the given
-// time.  If no time can be found to satisfy the schedule, return the zero time.
-func (s *SpecSchedule) Next(t time.Time) time.Time {
-	// General approach
-	//
-	// For Month, Day, Hour, Minute, Second:
-	// Check if the time value matches.  If yes, continue to the next field.
-	// If the field doesn't match the schedule, then increment the field until it matches.
-	// While incrementing the field, a wrap-around brings it back to the beginning
-	// of the field list (since it is necessary to re-verify previous field
-	// values)
-
-	// Convert the given time into the schedule's timezone, if one is specified.
-	// Save the original timezone so we can convert back after we find a time.
-	// Note that schedules without a time zone specified (time.Local) are treated
-	// as local to the time provided.
-	origLocation := t.Location()
-	loc := s.Location
-	if loc == time.Local {
-		loc = t.Location()
-	}
-	if s.Location != time.Local {
-		t = t.In(s.Location)
-	}
-
-	// Start at the earliest possible time (the upcoming second).
-	t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond)
-
-	// This flag indicates whether a field has been incremented.
-	added := false
-
-	// If no time is found within five years, return zero.
-	yearLimit := t.Year() + 5
-
-WRAP:
-	if t.Year() > yearLimit {
-		return time.Time{}
-	}
-
-	// Find the first applicable month.
-	// If it's this month, then do nothing.
-	for 1<<uint(t.Month())&s.Month == 0 {
-		// If we have to add a month, reset the other parts to 0.
-		if !added {
-			added = true
-			// Otherwise, set the date at the beginning (since the current time is irrelevant).
-			t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc)
-		}
-		t = t.AddDate(0, 1, 0)
-
-		// Wrapped around.
-		if t.Month() == time.January {
-			goto WRAP
-		}
-	}
-
-	// Now get a day in that month.
-	//
-	// NOTE: This causes issues for daylight savings regimes where midnight does
-	// not exist.  For example: Sao Paulo has DST that transforms midnight on
-	// 11/3 into 1am. Handle that by noticing when the Hour ends up != 0.
-	for !dayMatches(s, t) {
-		if !added {
-			added = true
-			t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
-		}
-		t = t.AddDate(0, 0, 1)
-		// Notice if the hour is no longer midnight due to DST.
-		// Add an hour if it's 23, subtract an hour if it's 1.
-		if t.Hour() != 0 {
-			if t.Hour() > 12 {
-				t = t.Add(time.Duration(24-t.Hour()) * time.Hour)
-			} else {
-				t = t.Add(time.Duration(-t.Hour()) * time.Hour)
-			}
-		}
-
-		if t.Day() == 1 {
-			goto WRAP
-		}
-	}
-
-	for 1<<uint(t.Hour())&s.Hour == 0 {
-		if !added {
-			added = true
-			t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc)
-		}
-		t = t.Add(1 * time.Hour)
-
-		if t.Hour() == 0 {
-			goto WRAP
-		}
-	}
-
-	for 1<<uint(t.Minute())&s.Minute == 0 {
-		if !added {
-			added = true
-			t = t.Truncate(time.Minute)
-		}
-		t = t.Add(1 * time.Minute)
-
-		if t.Minute() == 0 {
-			goto WRAP
-		}
-	}
-
-	for 1<<uint(t.Second())&s.Second == 0 {
-		if !added {
-			added = true
-			t = t.Truncate(time.Second)
-		}
-		t = t.Add(1 * time.Second)
-
-		if t.Second() == 0 {
-			goto WRAP
-		}
-	}
-
-	return t.In(origLocation)
-}
-
-// dayMatches returns true if the schedule's day-of-week and day-of-month
-// restrictions are satisfied by the given time.
-func dayMatches(s *SpecSchedule, t time.Time) bool {
-	var (
-		domMatch = 1<<uint(t.Day())&s.Dom > 0
-		dowMatch = 1<<uint(t.Weekday())&s.Dow > 0
-	)
-	if s.Dom&starBit > 0 || s.Dow&starBit > 0 {
-		return domMatch && dowMatch
-	}
-	return domMatch || dowMatch
-}

backend/lib/cron/spec_test.go

@@ -1,300 +0,0 @@
-package cron
-
-import (
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestActivation(t *testing.T) {
-	tests := []struct {
-		time, spec string
-		expected   bool
-	}{
-		// Every fifteen minutes.
-		{"Mon Jul 9 15:00 2012", "0/15 * * * *", true},
-		{"Mon Jul 9 15:45 2012", "0/15 * * * *", true},
-		{"Mon Jul 9 15:40 2012", "0/15 * * * *", false},
-
-		// Every fifteen minutes, starting at 5 minutes.
-		{"Mon Jul 9 15:05 2012", "5/15 * * * *", true},
-		{"Mon Jul 9 15:20 2012", "5/15 * * * *", true},
-		{"Mon Jul 9 15:50 2012", "5/15 * * * *", true},
-
-		// Named months
-		{"Sun Jul 15 15:00 2012", "0/15 * * Jul *", true},
-		{"Sun Jul 15 15:00 2012", "0/15 * * Jun *", false},
-
-		// Everything set.
-		{"Sun Jul 15 08:30 2012", "30 08 ? Jul Sun", true},
-		{"Sun Jul 15 08:30 2012", "30 08 15 Jul ?", true},
-		{"Mon Jul 16 08:30 2012", "30 08 ? Jul Sun", false},
-		{"Mon Jul 16 08:30 2012", "30 08 15 Jul ?", false},
-
-		// Predefined schedules
-		{"Mon Jul 9 15:00 2012", "@hourly", true},
-		{"Mon Jul 9 15:04 2012", "@hourly", false},
-		{"Mon Jul 9 15:00 2012", "@daily", false},
-		{"Mon Jul 9 00:00 2012", "@daily", true},
-		{"Mon Jul 9 00:00 2012", "@weekly", false},
-		{"Sun Jul 8 00:00 2012", "@weekly", true},
-		{"Sun Jul 8 01:00 2012", "@weekly", false},
-		{"Sun Jul 8 00:00 2012", "@monthly", false},
-		{"Sun Jul 1 00:00 2012", "@monthly", true},
-
-		// Test interaction of DOW and DOM.
-		// If both are restricted, then only one needs to match.
-		{"Sun Jul 15 00:00 2012", "* * 1,15 * Sun", true},
-		{"Fri Jun 15 00:00 2012", "* * 1,15 * Sun", true},
-		{"Wed Aug 1 00:00 2012", "* * 1,15 * Sun", true},
-		{"Sun Jul 15 00:00 2012", "* * */10 * Sun", true}, // verifies #70
-
-		// However, if one has a star, then both need to match.
-		{"Sun Jul 15 00:00 2012", "* * * * Mon", false},
-		{"Mon Jul 9 00:00 2012", "* * 1,15 * *", false},
-		{"Sun Jul 15 00:00 2012", "* * 1,15 * *", true},
-		{"Sun Jul 15 00:00 2012", "* * */2 * Sun", true},
-	}
-
-	for _, test := range tests {
-		sched, err := ParseStandard(test.spec)
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-		actual := sched.Next(getTime(test.time).Add(-1 * time.Second))
-		expected := getTime(test.time)
-		if test.expected && expected != actual || !test.expected && expected == actual {
-			t.Errorf("Fail evaluating %s on %s: (expected) %s != %s (actual)",
-				test.spec, test.time, expected, actual)
-		}
-	}
-}
-
-func TestNext(t *testing.T) {
-	runs := []struct {
-		time, spec string
-		expected   string
-	}{
-		// Simple cases
-		{"Mon Jul 9 14:45 2012", "0 0/15 * * * *", "Mon Jul 9 15:00 2012"},
-		{"Mon Jul 9 14:59 2012", "0 0/15 * * * *", "Mon Jul 9 15:00 2012"},
-		{"Mon Jul 9 14:59:59 2012", "0 0/15 * * * *", "Mon Jul 9 15:00 2012"},
-
-		// Wrap around hours
-		{"Mon Jul 9 15:45 2012", "0 20-35/15 * * * *", "Mon Jul 9 16:20 2012"},
-
-		// Wrap around days
-		{"Mon Jul 9 23:46 2012", "0 */15 * * * *", "Tue Jul 10 00:00 2012"},
-		{"Mon Jul 9 23:45 2012", "0 20-35/15 * * * *", "Tue Jul 10 00:20 2012"},
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 * * * *", "Tue Jul 10 00:20:15 2012"},
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 1/2 * * *", "Tue Jul 10 01:20:15 2012"},
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 10-12 * * *", "Tue Jul 10 10:20:15 2012"},
-
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 1/2 */2 * *", "Thu Jul 11 01:20:15 2012"},
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 * 9-20 * *", "Wed Jul 10 00:20:15 2012"},
-		{"Mon Jul 9 23:35:51 2012", "15/35 20-35/15 * 9-20 Jul *", "Wed Jul 10 00:20:15 2012"},
-
-		// Wrap around months
-		{"Mon Jul 9 23:35 2012", "0 0 0 9 Apr-Oct ?", "Thu Aug 9 00:00 2012"},
-		{"Mon Jul 9 23:35 2012", "0 0 0 */5 Apr,Aug,Oct Mon", "Tue Aug 1 00:00 2012"},
-		{"Mon Jul 9 23:35 2012", "0 0 0 */5 Oct Mon", "Mon Oct 1 00:00 2012"},
-
-		// Wrap around years
-		{"Mon Jul 9 23:35 2012", "0 0 0 * Feb Mon", "Mon Feb 4 00:00 2013"},
-		{"Mon Jul 9 23:35 2012", "0 0 0 * Feb Mon/2", "Fri Feb 1 00:00 2013"},
-
-		// Wrap around minute, hour, day, month, and year
-		{"Mon Dec 31 23:59:45 2012", "0 * * * * *", "Tue Jan 1 00:00:00 2013"},
-
-		// Leap year
-		{"Mon Jul 9 23:35 2012", "0 0 0 29 Feb ?", "Mon Feb 29 00:00 2016"},
-
-		// Daylight savings time 2am EST (-5) -> 3am EDT (-4)
-		{"2012-03-11T00:00:00-0500", "TZ=America/New_York 0 30 2 11 Mar ?", "2013-03-11T02:30:00-0400"},
-
-		// hourly job
-		{"2012-03-11T00:00:00-0500", "TZ=America/New_York 0 0 * * * ?", "2012-03-11T01:00:00-0500"},
-		{"2012-03-11T01:00:00-0500", "TZ=America/New_York 0 0 * * * ?", "2012-03-11T03:00:00-0400"},
-		{"2012-03-11T03:00:00-0400", "TZ=America/New_York 0 0 * * * ?", "2012-03-11T04:00:00-0400"},
-		{"2012-03-11T04:00:00-0400", "TZ=America/New_York 0 0 * * * ?", "2012-03-11T05:00:00-0400"},
-
-		// hourly job using CRON_TZ
-		{"2012-03-11T00:00:00-0500", "CRON_TZ=America/New_York 0 0 * * * ?", "2012-03-11T01:00:00-0500"},
-		{"2012-03-11T01:00:00-0500", "CRON_TZ=America/New_York 0 0 * * * ?", "2012-03-11T03:00:00-0400"},
-		{"2012-03-11T03:00:00-0400", "CRON_TZ=America/New_York 0 0 * * * ?", "2012-03-11T04:00:00-0400"},
-		{"2012-03-11T04:00:00-0400", "CRON_TZ=America/New_York 0 0 * * * ?", "2012-03-11T05:00:00-0400"},
-
-		// 1am nightly job
-		{"2012-03-11T00:00:00-0500", "TZ=America/New_York 0 0 1 * * ?", "2012-03-11T01:00:00-0500"},
-		{"2012-03-11T01:00:00-0500", "TZ=America/New_York 0 0 1 * * ?", "2012-03-12T01:00:00-0400"},
-
-		// 2am nightly job (skipped)
-		{"2012-03-11T00:00:00-0500", "TZ=America/New_York 0 0 2 * * ?", "2012-03-12T02:00:00-0400"},
-
-		// Daylight savings time 2am EDT (-4) => 1am EST (-5)
-		{"2012-11-04T00:00:00-0400", "TZ=America/New_York 0 30 2 04 Nov ?", "2012-11-04T02:30:00-0500"},
-		{"2012-11-04T01:45:00-0400", "TZ=America/New_York 0 30 1 04 Nov ?", "2012-11-04T01:30:00-0500"},
-
-		// hourly job
-		{"2012-11-04T00:00:00-0400", "TZ=America/New_York 0 0 * * * ?", "2012-11-04T01:00:00-0400"},
-		{"2012-11-04T01:00:00-0400", "TZ=America/New_York 0 0 * * * ?", "2012-11-04T01:00:00-0500"},
-		{"2012-11-04T01:00:00-0500", "TZ=America/New_York 0 0 * * * ?", "2012-11-04T02:00:00-0500"},
-
-		// 1am nightly job (runs twice)
-		{"2012-11-04T00:00:00-0400", "TZ=America/New_York 0 0 1 * * ?", "2012-11-04T01:00:00-0400"},
-		{"2012-11-04T01:00:00-0400", "TZ=America/New_York 0 0 1 * * ?", "2012-11-04T01:00:00-0500"},
-		{"2012-11-04T01:00:00-0500", "TZ=America/New_York 0 0 1 * * ?", "2012-11-05T01:00:00-0500"},
-
-		// 2am nightly job
-		{"2012-11-04T00:00:00-0400", "TZ=America/New_York 0 0 2 * * ?", "2012-11-04T02:00:00-0500"},
-		{"2012-11-04T02:00:00-0500", "TZ=America/New_York 0 0 2 * * ?", "2012-11-05T02:00:00-0500"},
-
-		// 3am nightly job
-		{"2012-11-04T00:00:00-0400", "TZ=America/New_York 0 0 3 * * ?", "2012-11-04T03:00:00-0500"},
-		{"2012-11-04T03:00:00-0500", "TZ=America/New_York 0 0 3 * * ?", "2012-11-05T03:00:00-0500"},
-
-		// hourly job
-		{"TZ=America/New_York 2012-11-04T00:00:00-0400", "0 0 * * * ?", "2012-11-04T01:00:00-0400"},
-		{"TZ=America/New_York 2012-11-04T01:00:00-0400", "0 0 * * * ?", "2012-11-04T01:00:00-0500"},
-		{"TZ=America/New_York 2012-11-04T01:00:00-0500", "0 0 * * * ?", "2012-11-04T02:00:00-0500"},
-
-		// 1am nightly job (runs twice)
-		{"TZ=America/New_York 2012-11-04T00:00:00-0400", "0 0 1 * * ?", "2012-11-04T01:00:00-0400"},
-		{"TZ=America/New_York 2012-11-04T01:00:00-0400", "0 0 1 * * ?", "2012-11-04T01:00:00-0500"},
-		{"TZ=America/New_York 2012-11-04T01:00:00-0500", "0 0 1 * * ?", "2012-11-05T01:00:00-0500"},
-
-		// 2am nightly job
-		{"TZ=America/New_York 2012-11-04T00:00:00-0400", "0 0 2 * * ?", "2012-11-04T02:00:00-0500"},
-		{"TZ=America/New_York 2012-11-04T02:00:00-0500", "0 0 2 * * ?", "2012-11-05T02:00:00-0500"},
-
-		// 3am nightly job
-		{"TZ=America/New_York 2012-11-04T00:00:00-0400", "0 0 3 * * ?", "2012-11-04T03:00:00-0500"},
-		{"TZ=America/New_York 2012-11-04T03:00:00-0500", "0 0 3 * * ?", "2012-11-05T03:00:00-0500"},
-
-		// Unsatisfiable
-		{"Mon Jul 9 23:35 2012", "0 0 0 30 Feb ?", ""},
-		{"Mon Jul 9 23:35 2012", "0 0 0 31 Apr ?", ""},
-
-		// Monthly job
-		{"TZ=America/New_York 2012-11-04T00:00:00-0400", "0 0 3 3 * ?", "2012-12-03T03:00:00-0500"},
-
-		// Test the scenario of DST resulting in midnight not being a valid time.
-		// https://github.com/robfig/cron/issues/157
-		{"2018-10-17T05:00:00-0400", "TZ=America/Sao_Paulo 0 0 9 10 * ?", "2018-11-10T06:00:00-0500"},
-		{"2018-02-14T05:00:00-0500", "TZ=America/Sao_Paulo 0 0 9 22 * ?", "2018-02-22T07:00:00-0500"},
-	}
-
-	for _, c := range runs {
-		sched, err := secondParser.Parse(c.spec)
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-		actual := sched.Next(getTime(c.time))
-		expected := getTime(c.expected)
-		if !actual.Equal(expected) {
-			t.Errorf("%s, \"%s\": (expected) %v != %v (actual)", c.time, c.spec, expected, actual)
-		}
-	}
-}
-
-func TestErrors(t *testing.T) {
-	invalidSpecs := []string{
-		"xyz",
-		"60 0 * * *",
-		"0 60 * * *",
-		"0 0 * * XYZ",
-	}
-	for _, spec := range invalidSpecs {
-		_, err := ParseStandard(spec)
-		if err == nil {
-			t.Error("expected an error parsing: ", spec)
-		}
-	}
-}
-
-func getTime(value string) time.Time {
-	if value == "" {
-		return time.Time{}
-	}
-
-	var location = time.Local
-	if strings.HasPrefix(value, "TZ=") {
-		parts := strings.Fields(value)
-		loc, err := time.LoadLocation(parts[0][len("TZ="):])
-		if err != nil {
-			panic("could not parse location:" + err.Error())
-		}
-		location = loc
-		value = parts[1]
-	}
-
-	var layouts = []string{
-		"Mon Jan 2 15:04 2006",
-		"Mon Jan 2 15:04:05 2006",
-	}
-	for _, layout := range layouts {
-		if t, err := time.ParseInLocation(layout, value, location); err == nil {
-			return t
-		}
-	}
-	if t, err := time.ParseInLocation("2006-01-02T15:04:05-0700", value, location); err == nil {
-		return t
-	}
-	panic("could not parse time value " + value)
-}
-
-func TestNextWithTz(t *testing.T) {
-	runs := []struct {
-		time, spec string
-		expected   string
-	}{
-		// Failing tests
-		{"2016-01-03T13:09:03+0530", "14 14 * * *", "2016-01-03T14:14:00+0530"},
-		{"2016-01-03T04:09:03+0530", "14 14 * * ?", "2016-01-03T14:14:00+0530"},
-
-		// Passing tests
-		{"2016-01-03T14:09:03+0530", "14 14 * * *", "2016-01-03T14:14:00+0530"},
-		{"2016-01-03T14:00:00+0530", "14 14 * * ?", "2016-01-03T14:14:00+0530"},
-	}
-	for _, c := range runs {
-		sched, err := ParseStandard(c.spec)
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-		actual := sched.Next(getTimeTZ(c.time))
-		expected := getTimeTZ(c.expected)
-		if !actual.Equal(expected) {
-			t.Errorf("%s, \"%s\": (expected) %v != %v (actual)", c.time, c.spec, expected, actual)
-		}
-	}
-}
-
-func getTimeTZ(value string) time.Time {
-	if value == "" {
-		return time.Time{}
-	}
-	t, err := time.Parse("Mon Jan 2 15:04 2006", value)
-	if err != nil {
-		t, err = time.Parse("Mon Jan 2 15:04:05 2006", value)
-		if err != nil {
-			t, err = time.Parse("2006-01-02T15:04:05-0700", value)
-			if err != nil {
-				panic(err)
-			}
-		}
-	}
-
-	return t
-}
-
-// https://github.com/robfig/cron/issues/144
-func TestSlash0NoHang(t *testing.T) {
-	schedule := "TZ=America/New_York 15/0 * * * *"
-	_, err := ParseStandard(schedule)
-	if err == nil {
-		t.Error("expected an error on 0 increment")
-	}
-}

backend/lib/validate/mongo.go

@@ -1,10 +0,0 @@
-package validate
-
-import (
-	"github.com/globalsign/mgo/bson"
-	"github.com/go-playground/validator/v10"
-)
-
-func MongoID(sl validator.FieldLevel) bool {
-	return bson.IsObjectIdHex(sl.Field().String())
-}

backend/main.go

@@ -1,370 +1,9 @@
 package main
 
 import (
-	"context"
-	"crawlab/config"
-	"crawlab/database"
-	_ "crawlab/docs"
-	validate2 "crawlab/lib/validate"
-	"crawlab/middlewares"
-	"crawlab/model"
-	"crawlab/routes"
-	"crawlab/services"
-	"crawlab/services/challenge"
-	"crawlab/services/rpc"
-	"github.com/apex/log"
-	"github.com/gin-gonic/gin"
-	"github.com/gin-gonic/gin/binding"
-	"github.com/go-playground/validator/v10"
-	"github.com/olivere/elastic/v7"
-	"github.com/spf13/viper"
-	"github.com/swaggo/gin-swagger"
-	"github.com/swaggo/gin-swagger/swaggerFiles"
-	"net"
-	"net/http"
-	"os"
-	"os/signal"
-	"runtime/debug"
-	"syscall"
-	"time"
+	"crawlab/cmd"
 )
 
-var swagHandler gin.HandlerFunc
-
-func init() {
-	swagHandler = ginSwagger.WrapHandler(swaggerFiles.Handler)
-}
 func main() {
-	app := gin.New()
-	app.Use(gin.Logger(), gin.Recovery())
-	if v, ok := binding.Validator.Engine().(*validator.Validate); ok {
-		_ = v.RegisterValidation("bid", validate2.MongoID)
-	}
-
-	if swagHandler != nil {
-		app.GET("/swagger/*any", swagHandler)
-	}
-
-	// 初始化配置
-	if err := config.InitConfig(""); err != nil {
-		log.Error("init config error:" + err.Error())
-		panic(err)
-	}
-	log.Info("initialized config successfully")
-	// 初始化Mongodb数据库
-	if err := database.InitMongo(); err != nil {
-		log.Error("init mongodb error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized mongodb successfully")
-
-	// 初始化Redis数据库
-	if err := database.InitRedis(); err != nil {
-		log.Error("init redis error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized redis successfully")
-
-	// 初始化日志设置
-	if err := services.InitLogService(); err != nil {
-		log.Error("init log error:" + err.Error())
-		panic(err)
-	}
-	log.Info("initialized log successfully") // 初始化日志设置
-
-	// 初始化节点服务
-	if err := services.InitNodeService(); err != nil {
-		log.Error("init node service error:" + err.Error())
-		panic(err)
-	}
-	log.Info("initialized local node successfully")
-
-	if model.IsMaster() {
-
-		// 初始化定时任务
-		if err := services.InitScheduler(); err != nil {
-			log.Error("init scheduler error:" + err.Error())
-			debug.PrintStack()
-			panic(err)
-		}
-		log.Info("initialized schedule successfully")
-
-		// 初始化用户服务
-		if err := services.InitUserService(); err != nil {
-			log.Error("init user service error:" + err.Error())
-			debug.PrintStack()
-			panic(err)
-		}
-		log.Info("initialized user service successfully")
-
-		// 初始化依赖服务
-		if err := services.InitDepsFetcher(); err != nil {
-			log.Error("init dependency fetcher error:" + err.Error())
-			debug.PrintStack()
-			panic(err)
-		}
-		log.Info("initialized dependency fetcher successfully")
-
-		// 初始化挑战服务
-		if err := challenge.InitChallengeService(); err != nil {
-			log.Error("init challenge service error:" + err.Error())
-			debug.PrintStack()
-			panic(err)
-		}
-		log.Info("initialized challenge service successfully")
-
-		// 初始化清理服务
-		if err := services.InitCleanService(); err != nil {
-			log.Error("init clean service error:" + err.Error())
-			debug.PrintStack()
-			panic(err)
-		}
-		log.Info("initialized clean service successfully")
-	}
-
-	// 初始化任务执行器
-	if err := services.InitTaskExecutor(); err != nil {
-		log.Error("init task executor error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized task executor successfully")
-
-	// 初始化爬虫服务
-	if err := services.InitSpiderService(); err != nil {
-		log.Error("init spider service error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized spider service successfully")
-
-	// 初始化RPC服务
-	if err := rpc.InitRpcService(); err != nil {
-		log.Error("init rpc service error:" + err.Error())
-		debug.PrintStack()
-		panic(err)
-	}
-	log.Info("initialized rpc service successfully")
-
-	// 以下为主节点服务
-	if model.IsMaster() {
-		// 中间件
-		esClientStr := viper.GetString("setting.esClient")
-		if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
-			ctx := context.Background()
-			esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
-			if err != nil {
-				log.Error("Init es client Error:" + err.Error())
-			}
-			app.Use(middlewares.EsLog(ctx, esClient))
-		}
-		app.Use(middlewares.CORSMiddleware())
-		anonymousGroup := app.Group("/")
-		{
-			anonymousGroup.POST("/login", routes.Login)       // 用户登录
-			anonymousGroup.PUT("/users", routes.PutUser)      // 添加用户
-			anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
-			// release版本
-			anonymousGroup.GET("/version", routes.GetVersion)               // 获取发布的版本
-			anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本
-			// 文档
-			anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据
-		}
-		authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
-		{
-			// 节点
-			{
-				authGroup.GET("/nodes", routes.GetNodeList)                            // 节点列表
-				authGroup.GET("/nodes/:id", routes.GetNode)                            // 节点详情
-				authGroup.POST("/nodes/:id", routes.PostNode)                          // 修改节点
-				authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList)              // 节点任务列表
-				authGroup.GET("/nodes/:id/system", routes.GetSystemInfo)               // 节点任务列表
-				authGroup.DELETE("/nodes/:id", routes.DeleteNode)                      // 删除节点
-				authGroup.GET("/nodes/:id/langs", routes.GetLangList)                  // 节点语言环境列表
-				authGroup.GET("/nodes/:id/deps", routes.GetDepList)                    // 节点第三方依赖列表
-				authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
-				authGroup.POST("/nodes/:id/deps/install", routes.InstallDep)           // 节点安装依赖
-				authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep)       // 节点卸载依赖
-				authGroup.POST("/nodes/:id/langs/install", routes.InstallLang)         // 节点安装语言
-			}
-			// 爬虫
-			{
-				authGroup.GET("/spiders", routes.GetSpiderList)                                            // 爬虫列表
-				authGroup.GET("/spiders/:id", routes.GetSpider)                                            // 爬虫详情
-				authGroup.PUT("/spiders", routes.PutSpider)                                                // 添加爬虫
-				authGroup.POST("/spiders", routes.UploadSpider)                                            // 上传爬虫
-				authGroup.POST("/spiders/:id", routes.PostSpider)                                          // 修改爬虫
-				authGroup.POST("/spiders/:id/publish", routes.PublishSpider)                               // 发布爬虫
-				authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId)                           // 上传爬虫（ID）
-				authGroup.DELETE("/spiders", routes.DeleteSelectedSpider)                                  // 删除选择的爬虫
-				authGroup.DELETE("/spiders/:id", routes.DeleteSpider)                                      // 删除爬虫
-				authGroup.POST("/spiders/:id/copy", routes.CopySpider)                                     // 拷贝爬虫
-				authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks)                                 // 爬虫任务列表
-				authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree)                          // 爬虫文件目录树读取
-				authGroup.GET("/spiders/:id/file", routes.GetSpiderFile)                                   // 爬虫文件读取
-				authGroup.POST("/spiders/:id/file", routes.PostSpiderFile)                                 // 爬虫文件更改
-				authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile)                                   // 爬虫文件创建
-				authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir)                                     // 爬虫目录创建
-				authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile)                             // 爬虫文件删除
-				authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile)                        // 爬虫文件重命名
-				authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir)                                     // 爬虫目录
-				authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)                                 // 爬虫统计数据
-				authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules)                         // 爬虫定时任务
-				authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders)                // Scrapy 爬虫名称列表
-				authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders)                // Scrapy 爬虫创建爬虫
-				authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings)              // Scrapy 爬虫设置
-				authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings)            // Scrapy 爬虫修改设置
-				authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems)                    // Scrapy 爬虫 items
-				authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems)                  // Scrapy 爬虫修改 items
-				authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines)            // Scrapy 爬虫 pipelines
-				authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
-				authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit)                          // 爬虫 Git 同步
-				authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit)                        // 爬虫 Git 重置
-				authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider)                             // 停止所选爬虫任务
-				authGroup.POST("/spiders-run", routes.RunSelectedSpider)                                   // 运行所选爬虫
-				authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider)                  // 批量设置爬虫项目
-			}
-			// 可配置爬虫
-			{
-				authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)           // 获取可配置爬虫配置
-				authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig)         // 更改可配置爬虫配置
-				authGroup.PUT("/config_spiders", routes.PutConfigSpider)                            // 添加可配置爬虫
-				authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)                      // 修改可配置爬虫
-				authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)             // 上传可配置爬虫
-				authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
-				authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList)      // 获取可配置爬虫模版列表
-			}
-			// 任务
-			{
-				authGroup.GET("/tasks", routes.GetTaskList)                                 // 任务列表
-				authGroup.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
-				authGroup.PUT("/tasks", routes.PutTask)                                     // 派发任务
-				authGroup.PUT("/tasks/batch", routes.PutBatchTasks)                         // 批量派发任务
-				authGroup.DELETE("/tasks/:id", routes.DeleteTask)                           // 删除任务
-				authGroup.DELETE("/tasks", routes.DeleteSelectedTask)                       // 删除多个任务
-				authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus)             // 删除指定状态的任务
-				authGroup.POST("/tasks/:id/cancel", routes.CancelTask)                      // 取消任务
-				authGroup.GET("/tasks/:id/log", routes.GetTaskLog)                          // 任务日志
-				authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog)               // 任务错误日志
-				authGroup.GET("/tasks/:id/results", routes.GetTaskResults)                  // 任务结果
-				authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
-				authGroup.POST("/tasks/:id/restart", routes.RestartTask)                    // 重新开始任务
-				authGroup.POST("/tasks-cancel", routes.CancelSelectedTask)                  // 批量取消任务
-				authGroup.POST("/tasks-restart", routes.RestartSelectedTask)                // 批量重试任务
-			}
-			// 系统任务/脚本
-			{
-				authGroup.PUT("/system-tasks", routes.PutSystemTask)      // 运行系统任务
-				authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表
-			}
-			// 定时任务
-			{
-				authGroup.GET("/schedules", routes.GetScheduleList)                  // 定时任务列表
-				authGroup.GET("/schedules/:id", routes.GetSchedule)                  // 定时任务详情
-				authGroup.PUT("/schedules", routes.PutSchedule)                      // 创建定时任务
-				authGroup.PUT("/schedules/batch", routes.PutBatchSchedules)          // 批量创建定时任务
-				authGroup.POST("/schedules/:id", routes.PostSchedule)                // 修改定时任务
-				authGroup.DELETE("/schedules/:id", routes.DeleteSchedule)            // 删除定时任务
-				authGroup.DELETE("/schedules", routes.DeleteBatchSchedules)          // 批量删除定时任务
-				authGroup.POST("/schedules/:id/disable", routes.DisableSchedule)     // 禁用定时任务
-				authGroup.POST("/schedules/:id/enable", routes.EnableSchedule)       // 启用定时任务
-				authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态
-			}
-			// 用户
-			{
-				authGroup.GET("/users", routes.GetUserList)                        // 用户列表
-				authGroup.GET("/users/:id", routes.GetUser)                        // 用户详情
-				authGroup.POST("/users/:id", routes.PostUser)                      // 更改用户
-				authGroup.DELETE("/users/:id", routes.DeleteUser)                  // 删除用户
-				authGroup.PUT("/users-add", routes.PutUser)                        // 添加用户
-				authGroup.GET("/me", routes.GetMe)                                 // 获取自己账户
-				authGroup.POST("/me", routes.PostMe)                               // 修改自己账户
-				authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码
-			}
-			// 系统
-			{
-				authGroup.GET("/system/deps/:lang", routes.GetAllDepList)             // 节点所有第三方依赖列表
-				authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
-			}
-			// 全局变量
-			{
-				authGroup.GET("/variables", routes.GetVariableList)      // 列表
-				authGroup.PUT("/variable", routes.PutVariable)           // 新增
-				authGroup.POST("/variable/:id", routes.PostVariable)     // 修改
-				authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除
-			}
-			// 项目
-			{
-				authGroup.GET("/projects", routes.GetProjectList)       // 列表
-				authGroup.GET("/projects/tags", routes.GetProjectTags)  // 项目标签
-				authGroup.PUT("/projects", routes.PutProject)           // 修改
-				authGroup.POST("/projects/:id", routes.PostProject)     // 新增
-				authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除
-			}
-			// 挑战
-			{
-				authGroup.GET("/challenges", routes.GetChallengeList)          // 挑战列表
-				authGroup.POST("/challenges-check", routes.CheckChallengeList) // 检查挑战列表
-			}
-			// 操作
-			{
-				//authGroup.GET("/actions", routes.GetActionList)   // 操作列表
-				//authGroup.GET("/actions/:id", routes.GetAction)   // 操作
-				authGroup.PUT("/actions", routes.PutAction) // 新增操作
-				//authGroup.POST("/actions/:id", routes.PostAction) // 修改操作
-			}
-			// API Token
-			{
-				authGroup.GET("/tokens", routes.GetTokens)          // 获取 Tokens
-				authGroup.PUT("/tokens", routes.PutToken)           // 添加 Token
-				authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token
-			}
-			// 统计数据
-			authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
-			// 文件
-			authGroup.GET("/file", routes.GetFile) // 获取文件
-			// Git
-			authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支
-			authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
-			authGroup.GET("/git/commits", routes.GetGitCommits)         // 获取 Git Commits
-			authGroup.POST("/git/checkout", routes.PostGitCheckout)     // 获取 Git Commits
-			// 爬虫市场 / 仓库
-			{
-				authGroup.GET("/repos", routes.GetRepoList)               // 获取仓库列表
-				authGroup.GET("/repos/sub-dir", routes.GetRepoSubDirList) // 获取仓库子目录
-				authGroup.POST("/repos/download", routes.DownloadRepo)    // 下载仓库
-			}
-		}
-	}
-
-	// 路由ping
-	app.GET("/ping", routes.Ping)
-
-	// 运行服务器
-	host := viper.GetString("server.host")
-	port := viper.GetString("server.port")
-	address := net.JoinHostPort(host, port)
-	srv := &http.Server{
-		Handler: app,
-		Addr:    address,
-	}
-	go func() {
-		if err := srv.ListenAndServe(); err != nil {
-			if err != http.ErrServerClosed {
-				log.Error("run server error:" + err.Error())
-			} else {
-				log.Info("server graceful down")
-			}
-		}
-	}()
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
-	<-quit
-	ctx2, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-	if err := srv.Shutdown(ctx2); err != nil {
-		log.Error("run server error:" + err.Error())
-	}
+	_ = cmd.Execute()
 }

backend/main_legacy.go

@@ -0,0 +1,252 @@
+package main
+
+import "crawlab/apps"
+
+func main() {
+	api := apps.NewApi()
+	api.Init()
+	api.Run()
+
+	//if model.IsMaster() {
+	//	// 初始化定时任务
+	//	if err := services.InitScheduler(); err != nil {
+	//		log.Error("init scheduler error:" + err.Error())
+	//		debug.PrintStack()
+	//		panic(err)
+	//	}
+	//	log.Info("initialized schedule successfully")
+	//
+	//	// 初始化用户服务
+	//	if err := services.InitUserService(); err != nil {
+	//		log.Error("init user service error:" + err.Error())
+	//		debug.PrintStack()
+	//		panic(err)
+	//	}
+	//	log.Info("initialized user service successfully")
+	//
+	//	// 初始化依赖服务
+	//	if err := services.InitDepsFetcher(); err != nil {
+	//		log.Error("init dependency fetcher error:" + err.Error())
+	//		debug.PrintStack()
+	//		panic(err)
+	//	}
+	//	log.Info("initialized dependency fetcher successfully")
+	//
+	//	// 初始化清理服务
+	//	if err := services.InitCleanService(); err != nil {
+	//		log.Error("init clean service error:" + err.Error())
+	//		debug.PrintStack()
+	//		panic(err)
+	//	}
+	//	log.Info("initialized clean service successfully")
+	//}
+	//
+	//// 初始化任务执行器
+	//if err := services.InitTaskExecutor(); err != nil {
+	//	log.Error("init task executor error:" + err.Error())
+	//	debug.PrintStack()
+	//	panic(err)
+	//}
+	//log.Info("initialized task executor successfully")
+	//
+	//// 初始化爬虫服务
+	//if err := services.InitSpiderService(); err != nil {
+	//	log.Error("init spider service error:" + err.Error())
+	//	debug.PrintStack()
+	//	panic(err)
+	//}
+	//log.Info("initialized spider service successfully")
+	//
+	//// 初始化RPC服务
+	//if err := rpc.InitRpcService(); err != nil {
+	//	log.Error("init rpc service error:" + err.Error())
+	//	debug.PrintStack()
+	//	panic(err)
+	//}
+	//log.Info("initialized rpc service successfully")
+
+	// 以下为主节点服务
+	//if model.IsMaster() {
+	//	// 中间件
+	//	esClientStr := viper.GetString("setting.esClient")
+	//	if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" {
+	//		ctx := context.Background()
+	//		esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false))
+	//		if err != nil {
+	//			log.Error("Init es client Error:" + err.Error())
+	//		}
+	//		app.Use(middlewares.EsLog(ctx, esClient))
+	//	}
+	//	app.Use(middlewares.CORSMiddleware())
+	//	anonymousGroup := app.Group("/")
+	//	{
+	//		anonymousGroup.POST("/login", routes.Login)       // 用户登录
+	//		anonymousGroup.PUT("/users", routes.PutUser)      // 添加用户
+	//		anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息
+	//		// release版本
+	//		anonymousGroup.GET("/version", routes.GetVersion)               // 获取发布的版本
+	//		anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本
+	//		// 文档
+	//		anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据
+	//	}
+	//	authGroup := app.Group("/", middlewares.AuthorizationMiddleware())
+	//	{
+	//		// 节点
+	//		{
+	//			authGroup.GET("/nodes", routes.GetNodeList)                            // 节点列表
+	//			authGroup.GET("/nodes/:id", routes.GetNode)                            // 节点详情
+	//			authGroup.POST("/nodes/:id", routes.PostNode)                          // 修改节点
+	//			authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList)              // 节点任务列表
+	//			authGroup.GET("/nodes/:id/system", routes.GetSystemInfo)               // 节点任务列表
+	//			authGroup.DELETE("/nodes/:id", routes.DeleteNode)                      // 删除节点
+	//			authGroup.GET("/nodes/:id/langs", routes.GetLangList)                  // 节点语言环境列表
+	//			authGroup.GET("/nodes/:id/deps", routes.GetDepList)                    // 节点第三方依赖列表
+	//			authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
+	//			authGroup.POST("/nodes/:id/deps/install", routes.InstallDep)           // 节点安装依赖
+	//			authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep)       // 节点卸载依赖
+	//			authGroup.POST("/nodes/:id/langs/install", routes.InstallLang)         // 节点安装语言
+	//		}
+	//		// 爬虫
+	//		{
+	//			authGroup.GET("/spiders", routes.GetSpiderList)                                            // 爬虫列表
+	//			authGroup.GET("/spiders/:id", routes.GetSpider)                                            // 爬虫详情
+	//			authGroup.PUT("/spiders", routes.PutSpider)                                                // 添加爬虫
+	//			authGroup.POST("/spiders", routes.UploadSpider)                                            // 上传爬虫
+	//			authGroup.POST("/spiders/:id", routes.PostSpider)                                          // 修改爬虫
+	//			authGroup.POST("/spiders/:id/publish", routes.PublishSpider)                               // 发布爬虫
+	//			authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId)                           // 上传爬虫（ID）
+	//			authGroup.DELETE("/spiders", routes.DeleteSelectedSpider)                                  // 删除选择的爬虫
+	//			authGroup.DELETE("/spiders/:id", routes.DeleteSpider)                                      // 删除爬虫
+	//			authGroup.POST("/spiders/:id/copy", routes.CopySpider)                                     // 拷贝爬虫
+	//			authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks)                                 // 爬虫任务列表
+	//			authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree)                          // 爬虫文件目录树读取
+	//			authGroup.GET("/spiders/:id/file", routes.GetSpiderFile)                                   // 爬虫文件读取
+	//			authGroup.POST("/spiders/:id/file", routes.PostSpiderFile)                                 // 爬虫文件更改
+	//			authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile)                                   // 爬虫文件创建
+	//			authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir)                                     // 爬虫目录创建
+	//			authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile)                             // 爬虫文件删除
+	//			authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile)                        // 爬虫文件重命名
+	//			authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir)                                     // 爬虫目录
+	//			authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats)                                 // 爬虫统计数据
+	//			authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules)                         // 爬虫定时任务
+	//			authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders)                // Scrapy 爬虫名称列表
+	//			authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders)                // Scrapy 爬虫创建爬虫
+	//			authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings)              // Scrapy 爬虫设置
+	//			authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings)            // Scrapy 爬虫修改设置
+	//			authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems)                    // Scrapy 爬虫 items
+	//			authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems)                  // Scrapy 爬虫修改 items
+	//			authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines)            // Scrapy 爬虫 pipelines
+	//			authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines
+	//			authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit)                          // 爬虫 Git 同步
+	//			authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit)                        // 爬虫 Git 重置
+	//			authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider)                             // 停止所选爬虫任务
+	//			authGroup.POST("/spiders-run", routes.RunSelectedSpider)                                   // 运行所选爬虫
+	//			authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider)                  // 批量设置爬虫项目
+	//		}
+	//		// 可配置爬虫
+	//		{
+	//			authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig)           // 获取可配置爬虫配置
+	//			authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig)         // 更改可配置爬虫配置
+	//			authGroup.PUT("/config_spiders", routes.PutConfigSpider)                            // 添加可配置爬虫
+	//			authGroup.POST("/config_spiders/:id", routes.PostConfigSpider)                      // 修改可配置爬虫
+	//			authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider)             // 上传可配置爬虫
+	//			authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫
+	//			authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList)      // 获取可配置爬虫模版列表
+	//		}
+	//		// 任务
+	//		{
+	//			authGroup.GET("/tasks", routes.GetTaskList)           // 任务列表
+	//			authGroup.GET("/tasks/:id", routes.GetTask)           // 任务详情
+	//			authGroup.PUT("/tasks", routes.PutTask)               // 派发任务
+	//			authGroup.PUT("/tasks/batch", routes.PutBatchTasks)   // 批量派发任务
+	//			authGroup.DELETE("/tasks/:id", routes.DeleteTask)     // 删除任务
+	//			authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务
+	//			//authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus)             // 删除指定状态的任务
+	//			authGroup.POST("/tasks/:id/cancel", routes.CancelTask)                      // 取消任务
+	//			authGroup.GET("/tasks/:id/log", routes.GetTaskLog)                          // 任务日志
+	//			authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog)               // 任务错误日志
+	//			authGroup.GET("/tasks/:id/results", routes.GetTaskResults)                  // 任务结果
+	//			authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果
+	//			authGroup.POST("/tasks/:id/restart", routes.RestartTask)                    // 重新开始任务
+	//			authGroup.POST("/tasks-cancel", routes.CancelSelectedTask)                  // 批量取消任务
+	//			authGroup.POST("/tasks-restart", routes.RestartSelectedTask)                // 批量重试任务
+	//		}
+	//		// 系统任务/脚本
+	//		{
+	//			authGroup.PUT("/system-tasks", routes.PutSystemTask)      // 运行系统任务
+	//			authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表
+	//		}
+	//		// 定时任务
+	//		{
+	//			authGroup.GET("/schedules", routes.GetScheduleList)                  // 定时任务列表
+	//			authGroup.GET("/schedules/:id", routes.GetSchedule)                  // 定时任务详情
+	//			authGroup.PUT("/schedules", routes.PutSchedule)                      // 创建定时任务
+	//			authGroup.PUT("/schedules/batch", routes.PutBatchSchedules)          // 批量创建定时任务
+	//			authGroup.POST("/schedules/:id", routes.PostSchedule)                // 修改定时任务
+	//			authGroup.DELETE("/schedules/:id", routes.DeleteSchedule)            // 删除定时任务
+	//			authGroup.DELETE("/schedules", routes.DeleteBatchSchedules)          // 批量删除定时任务
+	//			authGroup.POST("/schedules/:id/disable", routes.DisableSchedule)     // 禁用定时任务
+	//			authGroup.POST("/schedules/:id/enable", routes.EnableSchedule)       // 启用定时任务
+	//			authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态
+	//		}
+	//		// 用户
+	//		{
+	//			authGroup.GET("/users", routes.GetUserList)                        // 用户列表
+	//			authGroup.GET("/users/:id", routes.GetUser)                        // 用户详情
+	//			authGroup.POST("/users/:id", routes.PostUser)                      // 更改用户
+	//			authGroup.DELETE("/users/:id", routes.DeleteUser)                  // 删除用户
+	//			authGroup.PUT("/users-add", routes.PutUser)                        // 添加用户
+	//			authGroup.GET("/me", routes.GetMe)                                 // 获取自己账户
+	//			authGroup.POST("/me", routes.PostMe)                               // 修改自己账户
+	//			authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码
+	//		}
+	//		// 系统
+	//		{
+	//			authGroup.GET("/system/deps/:lang", routes.GetAllDepList)             // 节点所有第三方依赖列表
+	//			authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
+	//		}
+	//		// 全局变量
+	//		{
+	//			authGroup.GET("/variables", routes.GetVariableList)      // 列表
+	//			authGroup.PUT("/variable", routes.PutVariable)           // 新增
+	//			authGroup.POST("/variable/:id", routes.PostVariable)     // 修改
+	//			authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除
+	//		}
+	//		// 项目
+	//		{
+	//			authGroup.GET("/projects", routes.GetProjectList)       // 列表
+	//			authGroup.GET("/projects/tags", routes.GetProjectTags)  // 项目标签
+	//			authGroup.PUT("/projects", routes.PutProject)           // 修改
+	//			authGroup.POST("/projects/:id", routes.PostProject)     // 新增
+	//			authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除
+	//		}
+	//		// 操作
+	//		{
+	//			//authGroup.GET("/actions", routes.GetActionList)   // 操作列表
+	//			//authGroup.GET("/actions/:id", routes.GetAction)   // 操作
+	//			authGroup.PUT("/actions", routes.PutAction) // 新增操作
+	//			//authGroup.POST("/actions/:id", routes.PostAction) // 修改操作
+	//		}
+	//		// API Token
+	//		{
+	//			authGroup.GET("/tokens", routes.GetTokens)          // 获取 Tokens
+	//			authGroup.PUT("/tokens", routes.PutToken)           // 添加 Token
+	//			authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token
+	//		}
+	//		// 统计数据
+	//		authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据
+	//		// 文件
+	//		authGroup.GET("/file", routes.GetFile) // 获取文件
+	//		// Git
+	//		authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支
+	//		authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥
+	//		authGroup.GET("/git/commits", routes.GetGitCommits)         // 获取 Git Commits
+	//		authGroup.POST("/git/checkout", routes.PostGitCheckout)     // 获取 Git Commits
+	//	}
+	//}
+
+	// 路由ping
+	//app.GET("/ping", routes.Ping)
+
+	// 运行服务器
+}

backend/middlewares/auth.go

@@ -1,48 +0,0 @@
-package middlewares
-
-import (
-	"crawlab/constants"
-	"crawlab/routes"
-	"crawlab/services"
-	"github.com/gin-gonic/gin"
-	"net/http"
-	"strings"
-)
-
-func AuthorizationMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		// 获取token string
-		tokenStr := c.GetHeader("Authorization")
-
-		// 校验token
-		user, err := services.CheckToken(tokenStr)
-
-		// 校验失败，返回错误响应
-		if err != nil {
-			c.AbortWithStatusJSON(http.StatusUnauthorized, routes.Response{
-				Status:  "ok",
-				Message: "unauthorized",
-				Error:   "unauthorized",
-			})
-			return
-		}
-
-		// 如果为普通权限，校验请求地址是否符合要求
-		if user.Role == constants.RoleNormal {
-			if strings.HasPrefix(strings.ToLower(c.Request.URL.Path), "/users") {
-				c.AbortWithStatusJSON(http.StatusUnauthorized, routes.Response{
-					Status:  "ok",
-					Message: "unauthorized",
-					Error:   "unauthorized",
-				})
-				return
-			}
-		}
-
-		// 设置用户
-		c.Set(constants.ContextUser, &user)
-
-		// 校验成功
-		c.Next()
-	}
-}

backend/middlewares/cors.go

@@ -1,19 +0,0 @@
-package middlewares
-
-import "github.com/gin-gonic/gin"
-
-func CORSMiddleware() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
-		c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
-		c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
-		c.Writer.Header().Set("Access-Control-Allow-Methods", "DELETE, POST, OPTIONS, GET, PUT")
-
-		if c.Request.Method == "OPTIONS" {
-			c.AbortWithStatus(204)
-			return
-		}
-
-		c.Next()
-	}
-}

backend/middlewares/es_log.go

@@ -1,54 +0,0 @@
-package middlewares
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/olivere/elastic/v7"
-	"github.com/satori/go.uuid"
-	"github.com/spf13/viper"
-	"strconv"
-	"time"
-)
-
-func EsLog(ctx context.Context, esClient *elastic.Client) gin.HandlerFunc {
-
-	return func(c *gin.Context) {
-		// 开始时间
-		crawlabIndex := viper.GetString("setting.crawlabLogIndex")
-		start := time.Now()
-		// 处理请求
-		c.Next()
-		// 结束时间
-		end := time.Now()
-		//执行时间
-		latency := strconv.FormatInt(end.Sub(start).Nanoseconds()/1000, 10)
-		path := c.Request.URL.Path
-
-		clientIP := c.ClientIP()
-		method := c.Request.Method
-		statusCode := strconv.Itoa(c.Writer.Status())
-		buf := new(bytes.Buffer)
-		buf.ReadFrom(c.Request.Body)
-		b := buf.String()
-		accessLog := "costTime:" + latency + "ms--" + "StatusCode:" + statusCode + "--" + "Method:" + method + "--" + "ClientIp:" + clientIP + "--" +
-			"RequestURI:" + path + "--" + "Host:" + c.Request.Host + "--" + "UserAgent--" + c.Request.UserAgent() + "--RequestBody:" +
-			string(b)
-		WriteMsg(ctx, crawlabIndex, esClient, time.Now(), accessLog)
-	}
-
-}
-
-// WriteMsg will write the msg and level into es
-func WriteMsg(ctx context.Context, crawlabIndex string, es *elastic.Client, when time.Time, msg string) error {
-	vals := make(map[string]interface{})
-	vals["@timestamp"] = when.Format(time.RFC3339)
-	vals["@msg"] = msg
-	uid := uuid.NewV4().String()
-	_, err := es.Index().Index(crawlabIndex).Id(uid).BodyJson(vals).Refresh("wait_for").Do(ctx)
-	if err != nil {
-		fmt.Println(err)
-	}
-	return err
-}

backend/mock/base.go

@@ -1,16 +0,0 @@
-package mock
-
-type Response struct {
-	Status  string      `json:"status"`
-	Message string      `json:"message"`
-	Data    interface{} `json:"data"`
-	Error   string      `json:"error"`
-}
-
-type ListResponse struct {
-	Status  string      `json:"status"`
-	Message string      `json:"message"`
-	Total   int         `json:"total"`
-	Data    interface{} `json:"data"`
-	Error   string      `json:"error"`
-}

backend/mock/file.go

@@ -1,8 +0,0 @@
-package mock
-
-type File struct {
-	Name  string `json:"name"`
-	Path  string `json:"path"`
-	IsDir bool   `json:"is_dir"`
-	Size  int64  `json:"size"`
-}

backend/mock/node.go

@@ -1,222 +0,0 @@
-package mock
-
-import (
-	"crawlab/entity"
-	"crawlab/model"
-	"crawlab/services"
-	"github.com/apex/log"
-	"github.com/gin-gonic/gin"
-	"github.com/globalsign/mgo/bson"
-	"net/http"
-	"time"
-)
-
-var NodeList = []model.Node{
-	{
-		Id:           bson.ObjectId("5d429e6c19f7abede924fee2"),
-		Ip:           "10.32.35.15",
-		Name:         "test1",
-		Status:       "online",
-		Port:         "8081",
-		Mac:          "ac:12:df:12:fd",
-		Description:  "For test1",
-		IsMaster:     true,
-		UpdateTs:     time.Now(),
-		CreateTs:     time.Now(),
-		UpdateTsUnix: time.Now().Unix(),
-	},
-	{
-		Id:           bson.ObjectId("5d429e6c19f7abede924fe22"),
-		Ip:           "10.32.35.12",
-		Name:         "test2",
-		Status:       "online",
-		Port:         "8082",
-		Mac:          "ac:12:df:12:vh",
-		Description:  "For test2",
-		IsMaster:     true,
-		UpdateTs:     time.Now(),
-		CreateTs:     time.Now(),
-		UpdateTsUnix: time.Now().Unix(),
-	},
-}
-
-var TaskList = []model.Task{
-	{
-		Id:              "1234",
-		SpiderId:        bson.ObjectId("5d429e6c19f7abede924fee2"),
-		StartTs:         time.Now(),
-		FinishTs:        time.Now(),
-		Status:          "进行中",
-		NodeId:          bson.ObjectId("5d429e6c19f7abede924fee2"),
-		LogPath:         "./log",
-		Cmd:             "scrapy crawl test",
-		Error:           "",
-		ResultCount:     0,
-		WaitDuration:    10.0,
-		RuntimeDuration: 10,
-		TotalDuration:   20,
-		SpiderName:      "test",
-		NodeName:        "test",
-		CreateTs:        time.Now(),
-		UpdateTs:        time.Now(),
-	},
-	{
-		Id:              "5678",
-		SpiderId:        bson.ObjectId("5d429e6c19f7abede924fee2"),
-		StartTs:         time.Now(),
-		FinishTs:        time.Now(),
-		Status:          "进行中",
-		NodeId:          bson.ObjectId("5d429e6c19f7abede924fee2"),
-		LogPath:         "./log",
-		Cmd:             "scrapy crawl test2",
-		Error:           "",
-		ResultCount:     0,
-		WaitDuration:    10.0,
-		RuntimeDuration: 10,
-		TotalDuration:   20,
-		SpiderName:      "test",
-		NodeName:        "test",
-		CreateTs:        time.Now(),
-		UpdateTs:        time.Now(),
-	},
-}
-
-var dataList = []services.Data{
-	{
-		Mac:          "ac:12:fc:fd:ds:dd",
-		Ip:           "192.10.2.1",
-		Master:       true,
-		UpdateTs:     time.Now(),
-		UpdateTsUnix: time.Now().Unix(),
-	},
-	{
-		Mac:          "22:12:fc:fd:ds:dd",
-		Ip:           "182.10.2.2",
-		Master:       true,
-		UpdateTs:     time.Now(),
-		UpdateTsUnix: time.Now().Unix(),
-	},
-}
-
-var executeble = []entity.Executable{
-	{
-		Path:        "/test",
-		FileName:    "test.py",
-		DisplayName: "test.py",
-	},
-}
-var systemInfo = entity.SystemInfo{ARCH: "x86",
-	OS:          "linux",
-	Hostname:    "test",
-	NumCpu:      4,
-	Executables: executeble,
-}
-
-func GetNodeList(c *gin.Context) {
-	nodes := NodeList
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    nodes,
-	})
-}
-
-func GetNode(c *gin.Context) {
-	var result model.Node
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	for _, node := range NodeList {
-		if node.Id == bson.ObjectId(id) {
-			result = node
-		}
-	}
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    result,
-	})
-}
-
-func Ping(c *gin.Context) {
-	data := dataList[0]
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    data,
-	})
-}
-
-func PostNode(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	var oldItem model.Node
-	for _, node := range NodeList {
-		if node.Id == bson.ObjectId(id) {
-			oldItem = node
-		}
-
-	}
-	log.Info(id)
-	var newItem model.Node
-	if err := c.ShouldBindJSON(&newItem); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-	newItem.Id = oldItem.Id
-
-	log.Info("Post Node success")
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func GetNodeTaskList(c *gin.Context) {
-
-	tasks := TaskList
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    tasks,
-	})
-}
-
-func DeleteNode(c *gin.Context) {
-	id := bson.ObjectId("5d429e6c19f7abede924fee2")
-
-	for _, node := range NodeList {
-		if node.Id == id {
-			log.Infof("Delete a node")
-		}
-	}
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func GetSystemInfo(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	sysInfo := systemInfo
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    sysInfo,
-	})
-}

backend/mock/node_test.go

@@ -1,198 +0,0 @@
-package mock
-
-import (
-	"bytes"
-	"crawlab/model"
-	"encoding/json"
-	"github.com/gin-gonic/gin"
-	"github.com/globalsign/mgo/bson"
-	. "github.com/smartystreets/goconvey/convey"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-)
-
-var app *gin.Engine
-
-// 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务
-func init() {
-	app = gin.Default()
-
-	// mock Test
-	// 节点相关的API
-	app.GET("/ping", Ping)
-	app.GET("/nodes", GetNodeList)               // 节点列表
-	app.GET("/nodes/:id", GetNode)               // 节点详情
-	app.POST("/nodes/:id", PostNode)             // 修改节点
-	app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表
-	app.GET("/nodes/:id/system", GetSystemInfo)  // 节点任务列表
-	app.DELETE("/nodes/:id", DeleteNode)         // 删除节点
-	//// 爬虫
-	app.GET("/stats/home", GetHomeStats) // 首页统计数据
-	// 定时任务
-	app.GET("/schedules", GetScheduleList)                         // 定时任务列表
-	app.GET("/schedules/:id", GetSchedule)                         // 定时任务详情
-	app.PUT("/schedules", PutSchedule)                             // 创建定时任务
-	app.POST("/schedules/:id", PostSchedule)                       // 修改定时任务
-	app.DELETE("/schedules/:id", DeleteSchedule)                   // 删除定时任务
-	app.GET("/tasks", GetTaskList)                                 // 任务列表
-	app.GET("/tasks/:id", GetTask)                                 // 任务详情
-	app.PUT("/tasks", PutTask)                                     // 派发任务
-	app.DELETE("/tasks/:id", DeleteTask)                           // 删除任务
-	app.GET("/tasks/:id/results", GetTaskResults)                  // 任务结果
-	app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果
-	app.GET("/spiders", GetSpiderList)                             // 爬虫列表
-	app.GET("/spiders/:id", GetSpider)                             // 爬虫详情
-	app.POST("/spiders/:id", PostSpider)                           // 修改爬虫
-	app.DELETE("/spiders/:id", DeleteSpider)                       // 删除爬虫
-	app.GET("/spiders/:id/tasks", GetSpiderTasks)                  // 爬虫任务列表
-	app.GET("/spiders/:id/dir", GetSpiderDir)                      // 爬虫目录
-}
-
-//mock test, test data in ./mock
-func TestGetNodeList(t *testing.T) {
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/nodes", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-
-	Convey("Test API GetNodeList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetNode(t *testing.T) {
-	var resp Response
-	var mongoId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	t.Log(resp.Data)
-	Convey("Test API GetNode", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-			So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
-		})
-	})
-}
-
-func TestPing(t *testing.T) {
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/ping", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API ping", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetNodeTaskList(t *testing.T) {
-	var resp Response
-	var mongoId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API GetNodeTaskList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestDeleteNode(t *testing.T) {
-	var resp Response
-
-	var mongoId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API DeleteNode", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestPostNode(t *testing.T) {
-	var newItem = model.Node{
-		Id:           bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		Ip:           "10.32.35.15",
-		Name:         "test1",
-		Status:       "online",
-		Port:         "8081",
-		Mac:          "ac:12:df:12:fd",
-		Description:  "For test1",
-		IsMaster:     true,
-		UpdateTs:     time.Now(),
-		CreateTs:     time.Now(),
-		UpdateTsUnix: time.Now().Unix(),
-	}
-
-	var resp Response
-	body, _ := json.Marshal(newItem)
-
-	var mongoId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("POST", "nodes/"+mongoId, bytes.NewReader(body))
-
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	t.Log(resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API PostNode", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetSystemInfo(t *testing.T) {
-	var resp Response
-	var mongoId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API GetSystemInfo", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}

backend/mock/schedule.go

@@ -1,136 +0,0 @@
-package mock
-
-import (
-	"crawlab/constants"
-	"crawlab/model"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/globalsign/mgo/bson"
-	"net/http"
-	"time"
-)
-
-var NodeIdss = []bson.ObjectId{bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-	bson.ObjectIdHex("5d429e6c19f7abede924fee1")}
-
-var scheduleList = []model.Schedule{
-	{
-		Id:       bson.ObjectId("5d429e6c19f7abede924fee2"),
-		Name:     "test schedule",
-		SpiderId: "123",
-		NodeIds:  NodeIdss,
-		Cron:     "***1*",
-		EntryId:  10,
-		// 前端展示
-		SpiderName: "test scedule",
-
-		CreateTs: time.Now(),
-		UpdateTs: time.Now(),
-	},
-	{
-		Id:       bson.ObjectId("xx429e6c19f7abede924fee2"),
-		Name:     "test schedule2",
-		SpiderId: "234",
-		NodeIds:  NodeIdss,
-		Cron:     "***1*",
-		EntryId:  10,
-		// 前端展示
-		SpiderName: "test scedule2",
-
-		CreateTs: time.Now(),
-		UpdateTs: time.Now(),
-	},
-}
-
-func GetScheduleList(c *gin.Context) {
-	results := scheduleList
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    results,
-	})
-}
-
-func GetSchedule(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	var result model.Schedule
-	for _, sch := range scheduleList {
-		if sch.Id == bson.ObjectId(id) {
-			result = sch
-		}
-	}
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    result,
-	})
-}
-
-func PostSchedule(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	var oldItem model.Schedule
-	for _, sch := range scheduleList {
-		if sch.Id == bson.ObjectId(id) {
-			oldItem = sch
-		}
-
-	}
-
-	var newItem model.Schedule
-	if err := c.ShouldBindJSON(&newItem); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-	newItem.Id = oldItem.Id
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func PutSchedule(c *gin.Context) {
-	var item model.Schedule
-
-	// 绑定数据模型
-	if err := c.ShouldBindJSON(&item); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-
-	// 如果node_id为空，则置为空ObjectId
-	for _, NodeId := range item.NodeIds {
-		if NodeId == "" {
-			NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
-		}
-	}
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func DeleteSchedule(c *gin.Context) {
-	id := bson.ObjectIdHex("5d429e6c19f7abede924fee2")
-	for _, sch := range scheduleList {
-		if sch.Id == id {
-			fmt.Println("delete a schedule")
-		}
-	}
-	fmt.Println(id)
-	fmt.Println("update schedule")
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}

backend/mock/schedule_test.go

@@ -1,141 +0,0 @@
-package mock
-
-import (
-	"bytes"
-	"crawlab/model"
-	"crawlab/utils"
-	"encoding/json"
-	"github.com/globalsign/mgo/bson"
-	. "github.com/smartystreets/goconvey/convey"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestGetScheduleList(t *testing.T) {
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/schedules", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	t.Log(resp.Data)
-	Convey("Test API GetScheduleList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetSchedule(t *testing.T) {
-	var mongoId = "5d429e6c19f7abede924fee2"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API GetSchedule", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-			So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
-		})
-	})
-}
-
-func TestDeleteSchedule(t *testing.T) {
-	var mongoId = "5d429e6c19f7abede924fee2"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil)
-	app.ServeHTTP(w, req)
-
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-
-	Convey("Test DeleteSchedule", t, func() {
-		Convey("Test resp status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-		})
-	})
-}
-
-func TestPostSchedule(t *testing.T) {
-	var newItem = model.Schedule{
-		Id:       bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		Name:     "test schedule",
-		SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		NodeIds:  NodeIdss,
-		Cron:     "***1*",
-		EntryId:  10,
-		// 前端展示
-		SpiderName: "test scedule",
-
-		CreateTs: time.Now(),
-		UpdateTs: time.Now(),
-	}
-
-	var resp Response
-	var mongoId = "5d429e6c19f7abede924fee2"
-	body, _ := json.Marshal(newItem)
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("POST", "/schedules/"+mongoId, strings.NewReader(utils.BytesToString(body)))
-	app.ServeHTTP(w, req)
-
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	t.Log(resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API PostSchedule", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-
-}
-
-func TestPutSchedule(t *testing.T) {
-	var newItem = model.Schedule{
-		Id:       bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		Name:     "test schedule",
-		SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		NodeIds:  NodeIdss,
-		Cron:     "***1*",
-		EntryId:  10,
-		// 前端展示
-		SpiderName: "test scedule",
-
-		CreateTs: time.Now(),
-		UpdateTs: time.Now(),
-	}
-
-	var resp Response
-	body, _ := json.Marshal(newItem)
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("PUT", "/schedules", bytes.NewReader(body))
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	t.Log(resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API PutSchedule", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-
-}

backend/mock/spider.go

@@ -1,187 +0,0 @@
-package mock
-
-import (
-	"crawlab/constants"
-	"crawlab/model"
-	"github.com/apex/log"
-	"github.com/gin-gonic/gin"
-	"github.com/globalsign/mgo/bson"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"time"
-)
-
-var SpiderList = []model.Spider{
-	{
-		Id:          bson.ObjectId("5d429e6c19f7abede924fee2"),
-		Name:        "For test",
-		DisplayName: "test",
-		Type:        "test",
-		Col:         "test",
-		Site:        "www.baidu.com",
-		Envs:        nil,
-		Src:         "../app/spiders",
-		Cmd:         "scrapy crawl test",
-		LastRunTs:   time.Now(),
-		CreateTs:    time.Now(),
-		UpdateTs:    time.Now(),
-		UserId:      constants.ObjectIdNull,
-	},
-}
-
-func GetSpiderList(c *gin.Context) {
-
-	// mock get spider list from database
-	results := SpiderList
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    results,
-	})
-}
-
-func GetSpider(c *gin.Context) {
-	id := c.Param("id")
-	var result model.Spider
-
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-
-	for _, spider := range SpiderList {
-		if spider.Id == bson.ObjectId(id) {
-			result = spider
-		}
-	}
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    result,
-	})
-}
-
-func PostSpider(c *gin.Context) {
-	id := c.Param("id")
-
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-	}
-
-	var item model.Spider
-	if err := c.ShouldBindJSON(&item); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-
-	log.Info("modify the item")
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-func GetSpiderDir(c *gin.Context) {
-	// 爬虫ID
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	// 目录相对路径
-	path := c.Query("path")
-	var spi model.Spider
-
-	// 获取爬虫
-	for _, spider := range SpiderList {
-		if spider.Id == bson.ObjectId(id) {
-			spi = spider
-		}
-	}
-
-	// 获取目录下文件列表
-	f, err := ioutil.ReadDir(filepath.Join(spi.Src, path))
-	if err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-
-	// 遍历文件列表
-	var fileList []model.File
-	for _, file := range f {
-		fileList = append(fileList, model.File{
-			Name:  file.Name(),
-			IsDir: file.IsDir(),
-			Size:  file.Size(),
-			Path:  filepath.Join(path, file.Name()),
-		})
-	}
-
-	// 返回结果
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    fileList,
-	})
-}
-
-func GetSpiderTasks(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	var spider model.Spider
-	for _, spi := range SpiderList {
-		if spi.Id == bson.ObjectId(id) {
-			spider = spi
-		}
-	}
-
-	var tasks model.Task
-	for _, task := range TaskList {
-		if task.SpiderId == spider.Id {
-			tasks = task
-		}
-	}
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    tasks,
-	})
-}
-
-func DeleteSpider(c *gin.Context) {
-	id := c.Param("id")
-
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-
-	// 获取该爬虫,get this spider
-	var spider model.Spider
-	for _, spi := range SpiderList {
-		if spi.Id == bson.ObjectId(id) {
-			spider = spi
-		}
-	}
-
-	// 删除爬虫文件目录,delete the spider dir
-	if err := os.RemoveAll(spider.Src); err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-
-	// 从数据库中删除该爬虫,delete this spider from database
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}

backend/mock/spider_test.go

@@ -1,139 +0,0 @@
-package mock
-
-import (
-	"bytes"
-	"crawlab/constants"
-	"crawlab/model"
-	"encoding/json"
-	"github.com/globalsign/mgo/bson"
-	. "github.com/smartystreets/goconvey/convey"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-)
-
-func TestGetSpiderList(t *testing.T) {
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/spiders", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp faild")
-	}
-	Convey("Test API GetSpiderList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetSpider(t *testing.T) {
-	var resp Response
-	var spiderId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API GetSpider", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestPostSpider(t *testing.T) {
-	var spider = model.Spider{
-		Id:          bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		Name:        "For test",
-		DisplayName: "test",
-		Type:        "test",
-		Col:         "test",
-		Site:        "www.baidu.com",
-		Envs:        nil,
-		Src:         "/app/spider",
-		Cmd:         "scrapy crawl test",
-		LastRunTs:   time.Now(),
-		CreateTs:    time.Now(),
-		UpdateTs:    time.Now(),
-		UserId:      constants.ObjectIdNull,
-	}
-	var resp Response
-	var spiderId = "5d429e6c19f7abede924fee2"
-	w := httptest.NewRecorder()
-	body, _ := json.Marshal(spider)
-	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, bytes.NewReader(body))
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API PostSpider", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-
-}
-
-func TestGetSpiderDir(t *testing.T) {
-	var spiderId = "5d429e6c19f7abede924fee2"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API GetSpiderDir", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-
-}
-
-func TestGetSpiderTasks(t *testing.T) {
-	var spiderId = "5d429e6c19f7abede924fee2"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API GetSpiderTasks", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestDeleteSpider(t *testing.T) {
-	var spiderId = "5d429e6c19f7abede924fee2"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API DeleteSpider", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}

backend/mock/stats.go

@@ -1,62 +0,0 @@
-package mock
-
-import (
-	"crawlab/model"
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-var taskDailyItems = []model.TaskDailyItem{
-	{
-		Date:               "2019/08/19",
-		TaskCount:          2,
-		AvgRuntimeDuration: 1000,
-	},
-	{
-		Date:               "2019/08/20",
-		TaskCount:          3,
-		AvgRuntimeDuration: 10130,
-	},
-}
-
-func GetHomeStats(c *gin.Context) {
-	type DataOverview struct {
-		TaskCount       int `json:"task_count"`
-		SpiderCount     int `json:"spider_count"`
-		ActiveNodeCount int `json:"active_node_count"`
-		ScheduleCount   int `json:"schedule_count"`
-	}
-
-	type Data struct {
-		Overview DataOverview          `json:"overview"`
-		Daily    []model.TaskDailyItem `json:"daily"`
-	}
-
-	// 任务总数
-	taskCount := 10
-
-	// 在线节点总数
-	activeNodeCount := 4
-
-	// 爬虫总数
-	spiderCount := 5
-	// 定时任务数
-	scheduleCount := 2
-
-	// 每日任务数
-	items := taskDailyItems
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data: Data{
-			Overview: DataOverview{
-				ActiveNodeCount: activeNodeCount,
-				TaskCount:       taskCount,
-				SpiderCount:     spiderCount,
-				ScheduleCount:   scheduleCount,
-			},
-			Daily: items,
-		},
-	})
-}

backend/mock/stats_test.go

@@ -1,29 +0,0 @@
-package mock
-
-import (
-	"encoding/json"
-	"fmt"
-	. "github.com/smartystreets/goconvey/convey"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-)
-
-func TestGetHomeStats(t *testing.T) {
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/stats/home", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	fmt.Println(resp.Data)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-
-	Convey("Test API GetHomeStats", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}

backend/mock/system.go

@@ -1 +0,0 @@
-package mock

backend/mock/task.go

@@ -1,236 +0,0 @@
-package mock
-
-import (
-	"bytes"
-	"crawlab/constants"
-	"crawlab/model"
-	"crawlab/utils"
-	"encoding/csv"
-	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/globalsign/mgo/bson"
-	"github.com/satori/go.uuid"
-	"net/http"
-)
-
-type TaskListRequestData struct {
-	PageNum  int    `form:"page_num"`
-	PageSize int    `form:"page_size"`
-	NodeId   string `form:"node_id"`
-	SpiderId string `form:"spider_id"`
-}
-
-type TaskResultsRequestData struct {
-	PageNum  int `form:"page_num"`
-	PageSize int `form:"page_size"`
-}
-
-func GetTaskList(c *gin.Context) {
-	// 绑定数据
-	data := TaskListRequestData{}
-
-	if err := c.ShouldBindQuery(&data); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-	if data.PageNum == 0 {
-		data.PageNum = 1
-	}
-	if data.PageSize == 0 {
-		data.PageNum = 10
-	}
-
-	// 过滤条件
-	query := bson.M{}
-	if data.NodeId != "" {
-		query["node_id"] = bson.ObjectIdHex(data.NodeId)
-	}
-	if data.SpiderId != "" {
-		query["spider_id"] = bson.ObjectIdHex(data.SpiderId)
-	}
-
-	// 获取任务列表
-	tasks := TaskList
-
-	// 获取总任务数
-	total := len(TaskList)
-
-	c.JSON(http.StatusOK, ListResponse{
-		Status:  "ok",
-		Message: "success",
-		Total:   total,
-		Data:    tasks,
-	})
-}
-
-func GetTask(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	var result model.Task
-	for _, task := range TaskList {
-		if task.Id == id {
-			result = task
-		}
-	}
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-		Data:    result,
-	})
-}
-
-func PutTask(c *gin.Context) {
-	// 生成任务ID,generate task ID
-	id := uuid.NewV4()
-
-	// 绑定数据
-	var t model.Task
-	if err := c.ShouldBindJSON(&t); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-	t.Id = id.String()
-	t.Status = constants.StatusPending
-
-	// 如果没有传入node_id，则置为null
-	if t.NodeId.Hex() == "" {
-		t.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
-	}
-
-	// 将任务存入数据库,put the task into database
-	fmt.Println("put the task into database")
-
-	// 加入任务队列, put the task into task queue
-	fmt.Println("put the task into task queue")
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func DeleteTask(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	for _, task := range TaskList {
-		if task.Id == id {
-			fmt.Println("delete the task")
-		}
-	}
-
-	c.JSON(http.StatusOK, Response{
-		Status:  "ok",
-		Message: "success",
-	})
-}
-
-func GetTaskResults(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	// 绑定数据
-	data := TaskResultsRequestData{}
-	if err := c.ShouldBindQuery(&data); err != nil {
-		HandleError(http.StatusBadRequest, c, err)
-		return
-	}
-
-	// 获取任务
-	var task model.Task
-	for _, ta := range TaskList {
-		if ta.Id == id {
-			task = ta
-		}
-	}
-
-	fmt.Println(task)
-	// 获取结果
-	var results interface{}
-	total := len(TaskList)
-
-	c.JSON(http.StatusOK, ListResponse{
-		Status:  "ok",
-		Message: "success",
-		Data:    results,
-		Total:   total,
-	})
-}
-
-func DownloadTaskResultsCsv(c *gin.Context) {
-	id := c.Param("id")
-	if !bson.IsObjectIdHex(id) {
-		HandleErrorF(http.StatusBadRequest, c, "invalid id")
-		return
-	}
-	// 获取任务
-	var task model.Task
-	for _, ta := range TaskList {
-		if ta.Id == id {
-			task = ta
-		}
-	}
-	fmt.Println(task)
-
-	// 获取结果
-	var results []interface {
-	}
-
-	// 字段列表
-	var columns []string
-	if len(results) == 0 {
-		columns = []string{}
-	} else {
-		item := results[0].(bson.M)
-		for key := range item {
-			columns = append(columns, key)
-		}
-	}
-
-	// 缓冲
-	bytesBuffer := &bytes.Buffer{}
-
-	// 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码
-	bytesBuffer.WriteString("\xEF\xBB\xBF")
-
-	writer := csv.NewWriter(bytesBuffer)
-
-	// 写入表头
-	if err := writer.Write(columns); err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
-		return
-	}
-
-	// 写入内容
-	for _, result := range results {
-		// 将result转换为[]string
-		item := result.(bson.M)
-		var values []string
-		for _, col := range columns {
-			value := utils.InterfaceToString(item[col])
-			values = append(values, value)
-		}
-
-		// 写入数据
-		if err := writer.Write(values); err != nil {
-			HandleError(http.StatusInternalServerError, c, err)
-			return
-		}
-	}
-
-	// 此时才会将缓冲区数据写入
-	writer.Flush()
-
-	// 设置下载的文件名
-	c.Writer.Header().Set("Content-Disposition", "attachment;filename=data.csv")
-
-	// 设置文件类型以及输出数据
-	c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes())
-}

backend/mock/task_test.go

@@ -1,138 +0,0 @@
-package mock
-
-import (
-	"bytes"
-	"crawlab/model"
-	"encoding/json"
-	"github.com/globalsign/mgo/bson"
-	. "github.com/smartystreets/goconvey/convey"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-)
-
-func TestGetTaskList(t *testing.T) {
-	//var teskListRequestFrom = TaskListRequestData{
-	//	PageNum:  2,
-	//	PageSize: 10,
-	//	NodeId:   "434221grfsf",
-	//	SpiderId: "fdfewqrftea",
-	//}
-
-	var resp ListResponse
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-
-	Convey("Test API GetNodeList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-			So(resp.Total, ShouldEqual, 2)
-		})
-	})
-}
-
-func TestGetTask(t *testing.T) {
-	var resp Response
-	var taskId = "1234"
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-	Convey("Test API GetTask", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestPutTask(t *testing.T) {
-	var newItem = model.Task{
-		Id:              "1234",
-		SpiderId:        bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		StartTs:         time.Now(),
-		FinishTs:        time.Now(),
-		Status:          "online",
-		NodeId:          bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
-		LogPath:         "./log",
-		Cmd:             "scrapy crawl test",
-		Error:           "",
-		ResultCount:     0,
-		WaitDuration:    10.0,
-		RuntimeDuration: 10,
-		TotalDuration:   20,
-		SpiderName:      "test",
-		NodeName:        "test",
-		CreateTs:        time.Now(),
-		UpdateTs:        time.Now(),
-	}
-
-	var resp Response
-	body, _ := json.Marshal(&newItem)
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("PUT", "/tasks", bytes.NewReader(body))
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API PutTask", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestDeleteTask(t *testing.T) {
-	taskId := "1234"
-	var resp Response
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("unmarshal resp failed")
-	}
-	Convey("Test API DeleteTask", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-		})
-	})
-}
-
-func TestGetTaskResults(t *testing.T) {
-	//var teskListResultFrom = TaskResultsRequestData{
-	//	PageNum:  2,
-	//	PageSize: 1,
-	//}
-	taskId := "1234"
-
-	var resp ListResponse
-	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil)
-	app.ServeHTTP(w, req)
-	err := json.Unmarshal(w.Body.Bytes(), &resp)
-	if err != nil {
-		t.Fatal("Unmarshal resp failed")
-	}
-
-	Convey("Test API GetNodeList", t, func() {
-		Convey("Test response status", func() {
-			So(resp.Status, ShouldEqual, "ok")
-			So(resp.Message, ShouldEqual, "success")
-			So(resp.Total, ShouldEqual, 2)
-		})
-	})
-}

backend/mock/user.go

@@ -1 +0,0 @@
-package mock