feat: added modules

core/.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = tab
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[{*.yaml,*.yml,package.json}]
+indent_size = 2
+indent_style = space

core/.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: "Test"
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main, develop ]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-20.04
+    services:
+      mongo:
+        image: mongo:5
+        ports:
+          - 27017:27017
+    env:
+      CRAWLAB_SERVER_PORT: 9999
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '^1.22'
+      - name: Run unit tests
+        run: |
+          mods=(\
+            "github.com/crawlab-team/crawlab/core/controllers" \
+            "github.com/crawlab-team/crawlab/core/models/client" \
+            "github.com/crawlab-team/crawlab/core/models/service" \
+          )
+          for pkg in ${mods[@]}; do
+            go test ${pkg}
+          done

core/.gitignore

@@ -0,0 +1,10 @@
+.idea
+.DS_Store
+vendor/
+tmp/
+build/
+dist/
+*.log
+gen/
+*.exe
+*.txt

core/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

core/README.md

@@ -0,0 +1,2 @@
+# crawlab-core
+Backend core modules for Crawlab

core/apps/api.go

@@ -0,0 +1,126 @@
+package apps
+
+import (
+	"context"
+	"errors"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/routes"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net"
+	"net/http"
+	"time"
+)
+
+func init() {
+	// set gin mode
+	if viper.GetString("gin.mode") == "" {
+		gin.SetMode(gin.ReleaseMode)
+	} else {
+		gin.SetMode(viper.GetString("gin.mode"))
+	}
+}
+
+type Api struct {
+	// dependencies
+	interfaces.WithConfigPath
+
+	// internals
+	app   *gin.Engine
+	ln    net.Listener
+	srv   *http.Server
+	ready bool
+}
+
+func (app *Api) Init() {
+	// initialize controllers
+	_ = initModule("controllers", controllers.InitControllers)
+
+	// initialize middlewares
+	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
+
+	// initialize routes
+	_ = app.initModuleWithApp("routes", routes.InitRoutes)
+}
+
+func (app *Api) Start() {
+	// address
+	host := viper.GetString("server.host")
+	port := viper.GetString("server.port")
+	address := net.JoinHostPort(host, port)
+
+	// http server
+	app.srv = &http.Server{
+		Handler: app.app,
+		Addr:    address,
+	}
+
+	// listen
+	var err error
+	app.ln, err = net.Listen("tcp", address)
+	if err != nil {
+		panic(err)
+	}
+	app.ready = true
+
+	// serve
+	if err := http.Serve(app.ln, app.app); err != nil {
+		if !errors.Is(err, http.ErrServerClosed) {
+			log.Error("run server error:" + err.Error())
+		} else {
+			log.Info("server graceful down")
+		}
+	}
+}
+
+func (app *Api) Wait() {
+	DefaultWait()
+}
+
+func (app *Api) Stop() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := app.srv.Shutdown(ctx); err != nil {
+		log.Error("run server error:" + err.Error())
+	}
+}
+
+func (app *Api) GetGinEngine() *gin.Engine {
+	return app.app
+}
+
+func (app *Api) GetHttpServer() *http.Server {
+	return app.srv
+}
+
+func (app *Api) Ready() (ok bool) {
+	return app.ready
+}
+
+func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
+	return initModule(name, func() error {
+		return fn(app.app)
+	})
+}
+
+func NewApi() *Api {
+	api := &Api{
+		app: gin.New(),
+	}
+	api.Init()
+	return api
+}
+
+var api *Api
+
+func GetApi() *Api {
+	if api != nil {
+		return api
+	}
+	api = NewApi()
+	return api
+}

core/apps/api_v2.go

@@ -0,0 +1,122 @@
+package apps
+
+import (
+	"context"
+	"errors"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net"
+	"net/http"
+	"time"
+)
+
+func init() {
+	// set gin mode
+	if viper.GetString("gin.mode") == "" {
+		gin.SetMode(gin.ReleaseMode)
+	} else {
+		gin.SetMode(viper.GetString("gin.mode"))
+	}
+}
+
+type ApiV2 struct {
+	// dependencies
+	interfaces.WithConfigPath
+
+	// internals
+	app   *gin.Engine
+	ln    net.Listener
+	srv   *http.Server
+	ready bool
+}
+
+func (app *ApiV2) Init() {
+	// initialize middlewares
+	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
+
+	// initialize routes
+	_ = app.initModuleWithApp("routes", controllers.InitRoutes)
+}
+
+func (app *ApiV2) Start() {
+	// address
+	host := viper.GetString("server.host")
+	port := viper.GetString("server.port")
+	address := net.JoinHostPort(host, port)
+
+	// http server
+	app.srv = &http.Server{
+		Handler: app.app,
+		Addr:    address,
+	}
+
+	// listen
+	var err error
+	app.ln, err = net.Listen("tcp", address)
+	if err != nil {
+		panic(err)
+	}
+	app.ready = true
+
+	// serve
+	if err := http.Serve(app.ln, app.app); err != nil {
+		if !errors.Is(err, http.ErrServerClosed) {
+			log.Error("run server error:" + err.Error())
+		} else {
+			log.Info("server graceful down")
+		}
+	}
+}
+
+func (app *ApiV2) Wait() {
+	DefaultWait()
+}
+
+func (app *ApiV2) Stop() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := app.srv.Shutdown(ctx); err != nil {
+		log.Error("run server error:" + err.Error())
+	}
+}
+
+func (app *ApiV2) GetGinEngine() *gin.Engine {
+	return app.app
+}
+
+func (app *ApiV2) GetHttpServer() *http.Server {
+	return app.srv
+}
+
+func (app *ApiV2) Ready() (ok bool) {
+	return app.ready
+}
+
+func (app *ApiV2) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
+	return initModule(name, func() error {
+		return fn(app.app)
+	})
+}
+
+func NewApiV2() *ApiV2 {
+	api := &ApiV2{
+		app: gin.New(),
+	}
+	api.Init()
+	return api
+}
+
+var apiV2 *ApiV2
+
+func GetApiV2() *ApiV2 {
+	if apiV2 != nil {
+		return apiV2
+	}
+	apiV2 = NewApiV2()
+	return apiV2
+}

core/apps/docker.go

@@ -0,0 +1,199 @@
+package apps
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/sys_exec"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+type Docker struct {
+	// parent
+	parent ServerApp
+
+	// dependencies
+	interfaces.WithConfigPath
+
+	// seaweedfs log
+	fsLogFilePath string
+	fsLogFile     *os.File
+	fsReady       bool
+}
+
+func (app *Docker) Init() {
+	var err error
+	app.fsLogFile, err = os.OpenFile(app.fsLogFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0777))
+	if err != nil {
+		trace.PrintError(err)
+	}
+
+	// replace paths
+	if err := app.replacePaths(); err != nil {
+		panic(err)
+	}
+
+	// start nginx
+	go app.startNginx()
+
+	// start seaweedfs
+	go app.startSeaweedFs()
+}
+
+func (app *Docker) Start() {
+	// import demo
+	//if utils.IsDemo() && utils.InitializedDemo() {
+	//	go app.importDemo()
+	//}
+}
+
+func (app *Docker) Wait() {
+	DefaultWait()
+}
+
+func (app *Docker) Stop() {
+}
+
+func (app *Docker) GetParent() (parent ServerApp) {
+	return app.parent
+}
+
+func (app *Docker) SetParent(parent ServerApp) {
+	app.parent = parent
+}
+
+func (app *Docker) Ready() (ok bool) {
+	return app.fsReady &&
+		app.parent.GetApi().Ready()
+}
+
+func (app *Docker) replacePaths() (err error) {
+	// read
+	indexHtmlPath := "/app/dist/index.html"
+	indexHtmlBytes, err := os.ReadFile(indexHtmlPath)
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	indexHtml := string(indexHtmlBytes)
+
+	// replace paths
+	baseUrl := viper.GetString("base.url")
+	if baseUrl != "" {
+		indexHtml = app._replacePath(indexHtml, "js", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "css", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "window.VUE_APP_API_BASE_URL = '", baseUrl)
+	}
+
+	// replace path of baidu tongji
+	initBaiduTongji := viper.GetString("string")
+	if initBaiduTongji != "" {
+		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_BAIDU_TONGJI = ''", fmt.Sprintf("window.VUE_APP_INIT_BAIDU_TONGJI = '%s'", initBaiduTongji))
+	}
+
+	// replace path of umeng
+	initUmeng := viper.GetString("string")
+	if initUmeng != "" {
+		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_UMENG = ''", fmt.Sprintf("window.VUE_APP_INIT_UMENG = '%s'", initUmeng))
+	}
+
+	// write
+	if err := os.WriteFile(indexHtmlPath, []byte(indexHtml), os.FileMode(0766)); err != nil {
+		return trace.TraceError(err)
+	}
+
+	return nil
+}
+
+func (app *Docker) _replacePath(text, path, baseUrl string) (res string) {
+	text = strings.ReplaceAll(text, path, fmt.Sprintf("%s/%s", baseUrl, path))
+	return text
+}
+
+func (app *Docker) startNginx() {
+	cmd := exec.Command("service", "nginx", "start")
+	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
+		for scanner.Scan() {
+			line := fmt.Sprintf("[nginx] %s\n", scanner.Text())
+			_, _ = os.Stdout.WriteString(line)
+		}
+	})
+	if err := cmd.Run(); err != nil {
+		trace.PrintError(err)
+	}
+}
+
+func (app *Docker) startSeaweedFs() {
+	seaweedFsDataPath := "/data/seaweedfs"
+	if !utils.Exists(seaweedFsDataPath) {
+		_ = os.MkdirAll(seaweedFsDataPath, os.FileMode(0777))
+	}
+	cmd := exec.Command("weed", "server",
+		"-dir", "/data",
+		"-master.dir", seaweedFsDataPath,
+		"-volume.dir.idx", seaweedFsDataPath,
+		"-ip", "localhost",
+		"-volume.port", "9999",
+		"-volume.minFreeSpace", "1GiB",
+		"-filer",
+	)
+	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
+		for scanner.Scan() {
+			line := fmt.Sprintf("[seaweedfs] %s\n", scanner.Text())
+			_, _ = app.fsLogFile.WriteString(line)
+		}
+	})
+	go func() {
+		if err := cmd.Run(); err != nil {
+			trace.PrintError(err)
+		}
+	}()
+	for {
+		_, err := req.Get("http://localhost:8888")
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		app.fsReady = true
+		return
+	}
+}
+
+func (app *Docker) importDemo() {
+	for {
+		if app.Ready() {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	_ = utils.ImportDemo()
+}
+
+func NewDocker(svr ServerApp) *Docker {
+	dck := &Docker{
+		parent:        svr,
+		fsLogFilePath: "/var/log/weed.log",
+	}
+
+	dck.Init()
+
+	return dck
+}
+
+var dck *Docker
+
+func GetDocker(svr ServerApp) *Docker {
+	if dck != nil {
+		return dck
+	}
+	dck = NewDocker(svr)
+	return dck
+}

core/apps/interfaces.go

@@ -0,0 +1,39 @@
+package apps
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+type App interface {
+	Init()
+	Start()
+	Wait()
+	Stop()
+}
+
+type ApiApp interface {
+	App
+	GetGinEngine() (engine *gin.Engine)
+	GetHttpServer() (svr *http.Server)
+	Ready() (ok bool)
+}
+
+type NodeApp interface {
+	App
+	interfaces.WithConfigPath
+}
+
+type ServerApp interface {
+	NodeApp
+	GetApi() (api ApiApp)
+	GetNodeService() (masterSvc interfaces.NodeService)
+}
+
+type DockerApp interface {
+	App
+	GetParent() (parent NodeApp)
+	SetParent(parent NodeApp)
+	Ready() (ok bool)
+}

core/apps/server.go

@@ -0,0 +1,149 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/node/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/spf13/viper"
+	"net/http"
+	_ "net/http/pprof"
+)
+
+func init() {
+	injectModules()
+}
+
+type Server struct {
+	// settings
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+	nodeSvc interfaces.NodeService
+
+	// modules
+	api *Api
+	dck *Docker
+
+	// internals
+	quit chan int
+}
+
+func (app *Server) SetGrpcAddress(address interfaces.Address) {
+	app.grpcAddress = address
+}
+
+func (app *Server) GetApi() (api ApiApp) {
+	return app.api
+}
+
+func (app *Server) GetNodeService() (svc interfaces.NodeService) {
+	return app.nodeSvc
+}
+
+func (app *Server) Init() {
+	// log node info
+	app.logNodeInfo()
+
+	if utils.IsMaster() {
+
+		// initialize controllers
+		if err := controllers.InitControllers(); err != nil {
+			panic(err)
+		}
+	}
+
+	// pprof
+	app.initPprof()
+}
+
+func (app *Server) Start() {
+	if utils.IsMaster() {
+		// start docker app
+		if utils.IsDocker() {
+			go app.dck.Start()
+		}
+
+		// start api
+		go app.api.Start()
+	}
+
+	// start node service
+	go app.nodeSvc.Start()
+}
+
+func (app *Server) Wait() {
+	<-app.quit
+}
+
+func (app *Server) Stop() {
+	app.api.Stop()
+	app.quit <- 1
+}
+
+func (app *Server) logNodeInfo() {
+	log.Infof("current node type: %s", utils.GetNodeType())
+	if utils.IsDocker() {
+		log.Infof("running in docker container")
+	}
+}
+
+func (app *Server) initPprof() {
+	if viper.GetBool("pprof") {
+		go func() {
+			fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		}()
+	}
+}
+
+func NewServer() (app NodeApp) {
+	// server
+	svr := &Server{
+		WithConfigPath: config.NewConfigPathService(),
+		quit:           make(chan int, 1),
+	}
+
+	// service options
+	var svcOpts []service.Option
+	if svr.grpcAddress != nil {
+		svcOpts = append(svcOpts, service.WithAddress(svr.grpcAddress))
+	}
+
+	// master modules
+	if utils.IsMaster() {
+		// api
+		svr.api = GetApi()
+
+		// docker
+		if utils.IsDocker() {
+			svr.dck = GetDocker(svr)
+		}
+	}
+
+	// node service
+	var err error
+	if utils.IsMaster() {
+		svr.nodeSvc, err = service.NewMasterService(svcOpts...)
+	} else {
+		svr.nodeSvc, err = service.NewWorkerService(svcOpts...)
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	return svr
+}
+
+var server NodeApp
+
+func GetServer() NodeApp {
+	if server != nil {
+		return server
+	}
+	server = NewServer()
+	return server
+}

core/apps/server_test.go

@@ -0,0 +1,29 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"os"
+	"testing"
+	"time"
+)
+
+func init() {
+	_ = os.Setenv("CRAWLAB_DEMO", "false")
+}
+
+func TestServer_Start(t *testing.T) {
+	svr := GetServer()
+
+	// start
+	go Start(svr)
+	time.Sleep(1 * time.Second)
+
+	res, err := req.Get(fmt.Sprintf("http://localhost:%s/system-info", viper.GetString("server.port")))
+	require.Nil(t, err)
+	resStr, err := res.ToString()
+	require.Nil(t, err)
+	require.Contains(t, resStr, "success")
+}

core/apps/server_v2.go

@@ -0,0 +1,126 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/node/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/spf13/viper"
+	"net/http"
+	_ "net/http/pprof"
+)
+
+type ServerV2 struct {
+	// settings
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+
+	// modules
+	nodeSvc interfaces.NodeService
+	api     *ApiV2
+	dck     *Docker
+
+	// internals
+	quit chan int
+}
+
+func (app *ServerV2) Init() {
+	// log node info
+	app.logNodeInfo()
+
+	// pprof
+	app.initPprof()
+}
+
+func (app *ServerV2) Start() {
+	if utils.IsMaster() {
+		// start docker app
+		if utils.IsDocker() {
+			go app.dck.Start()
+		}
+
+		// start api
+		go app.api.Start()
+	}
+
+	// start node service
+	go app.nodeSvc.Start()
+}
+
+func (app *ServerV2) Wait() {
+	<-app.quit
+}
+
+func (app *ServerV2) Stop() {
+	app.api.Stop()
+	app.quit <- 1
+}
+
+func (app *ServerV2) GetApi() ApiApp {
+	return app.api
+}
+
+func (app *ServerV2) GetNodeService() interfaces.NodeService {
+	return app.nodeSvc
+}
+
+func (app *ServerV2) logNodeInfo() {
+	log.Infof("current node type: %s", utils.GetNodeType())
+	if utils.IsDocker() {
+		log.Infof("running in docker container")
+	}
+}
+
+func (app *ServerV2) initPprof() {
+	if viper.GetBool("pprof") {
+		go func() {
+			fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		}()
+	}
+}
+
+func NewServerV2() (app NodeApp) {
+	// server
+	svr := &ServerV2{
+		WithConfigPath: config.NewConfigPathService(),
+		quit:           make(chan int, 1),
+	}
+
+	// master modules
+	if utils.IsMaster() {
+		// api
+		svr.api = GetApiV2()
+
+		// docker
+		if utils.IsDocker() {
+			svr.dck = GetDocker(svr)
+		}
+	}
+
+	// node service
+	var err error
+	if utils.IsMaster() {
+		svr.nodeSvc, err = service.NewMasterServiceV2()
+	} else {
+		svr.nodeSvc, err = service.NewWorkerServiceV2()
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	return svr
+}
+
+var serverV2 NodeApp
+
+func GetServerV2() NodeApp {
+	if serverV2 != nil {
+		return serverV2
+	}
+	serverV2 = NewServerV2()
+	return serverV2
+}

core/apps/utils.go

@@ -0,0 +1,92 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/color"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/container"
+	grpcclient "github.com/crawlab-team/crawlab/core/grpc/client"
+	grpcserver "github.com/crawlab-team/crawlab/core/grpc/server"
+	modelsclient "github.com/crawlab-team/crawlab/core/models/client"
+	modelsservice "github.com/crawlab-team/crawlab/core/models/service"
+	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
+	"github.com/crawlab-team/crawlab/core/schedule"
+	"github.com/crawlab-team/crawlab/core/spider/admin"
+	"github.com/crawlab-team/crawlab/core/stats"
+	"github.com/crawlab-team/crawlab/core/task/handler"
+	"github.com/crawlab-team/crawlab/core/task/scheduler"
+	taskstats "github.com/crawlab-team/crawlab/core/task/stats"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+)
+
+func Start(app App) {
+	start(app)
+}
+
+func start(app App) {
+	app.Init()
+	go app.Start()
+	app.Wait()
+	app.Stop()
+}
+
+func DefaultWait() {
+	utils.DefaultWait()
+}
+
+func initModule(name string, fn func() error) (err error) {
+	if err := fn(); err != nil {
+		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
+		_ = trace.TraceError(err)
+		panic(err)
+	}
+	log.Info(fmt.Sprintf("initialized %s successfully", name))
+	return nil
+}
+
+func initApp(name string, app App) {
+	_ = initModule(name, func() error {
+		app.Init()
+		return nil
+	})
+}
+
+var injectors = []interface{}{
+	modelsservice.GetService,
+	modelsclient.NewServiceDelegate,
+	modelsclient.NewNodeServiceDelegate,
+	modelsclient.NewSpiderServiceDelegate,
+	modelsclient.NewTaskServiceDelegate,
+	modelsclient.NewTaskStatServiceDelegate,
+	modelsclient.NewEnvironmentServiceDelegate,
+	grpcclient.NewClient,
+	grpcclient.NewPool,
+	grpcserver.GetServer,
+	grpcserver.NewModelDelegateServer,
+	grpcserver.NewModelBaseServiceServer,
+	grpcserver.NewNodeServer,
+	grpcserver.NewTaskServer,
+	grpcserver.NewMessageServer,
+	config.NewConfigPathService,
+	user.GetUserService,
+	schedule.GetScheduleService,
+	admin.GetSpiderAdminService,
+	stats.GetStatsService,
+	nodeconfig.NewNodeConfigService,
+	taskstats.GetTaskStatsService,
+	color.NewService,
+	scheduler.GetTaskSchedulerService,
+	handler.GetTaskHandlerService,
+}
+
+func injectModules() {
+	c := container.GetContainer()
+	for _, injector := range injectors {
+		if err := c.Provide(injector); err != nil {
+			panic(err)
+		}
+	}
+}

core/cmd/root.go

@@ -0,0 +1,33 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var (
+	// Used for flags.
+	cfgFile string
+
+	rootCmd = &cobra.Command{
+		Use:   "crawlab",
+		Short: "CLI tool for Crawlab",
+		Long: `The CLI tool is for controlling against Crawlab.
+Crawlab is a distributed web crawler and task admin platform
+aimed at making web crawling and task management easier.
+`,
+	}
+)
+
+// Execute executes the root command.
+func Execute() error {
+	return rootCmd.Execute()
+}
+
+// GetRootCmd get rootCmd instance
+func GetRootCmd() *cobra.Command {
+	return rootCmd
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "c", "", "Use Custom Config File")
+}

core/cmd/server.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/crawlab-team/crawlab/core/apps"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(serverCmd)
+}
+
+var serverCmd = &cobra.Command{
+	Use:     "server",
+	Aliases: []string{"s"},
+	Short:   "Start Crawlab server",
+	Long:    `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		// app
+		//svr := apps.GetServer(opts...)
+		svr := apps.GetServerV2()
+
+		// start
+		apps.Start(svr)
+	},
+}

core/cmd/server_test.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"github.com/crawlab-team/crawlab/core/apps"
+	"os"
+	"testing"
+)
+
+func TestCmdServer(t *testing.T) {
+	_ = os.Setenv("CRAWLAB_PPROF", "true")
+
+	// app
+	svr := apps.GetServerV2()
+
+	// start
+	apps.Start(svr)
+}

core/color/service.go

@@ -0,0 +1,82 @@
+package color
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/data"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/go-trace"
+	"math/rand"
+	"strconv"
+	"strings"
+)
+
+func NewService() (svc interfaces.ColorService, err error) {
+	var cl []*entity.Color
+	cm := map[string]*entity.Color{}
+
+	if err := json.Unmarshal([]byte(data.ColorsDataText), &cl); err != nil {
+		return nil, trace.TraceError(err)
+	}
+
+	for _, c := range cl {
+		cm[c.Name] = c
+	}
+
+	return &Service{
+		cl: cl,
+		cm: cm,
+	}, nil
+}
+
+type Service struct {
+	cl []*entity.Color
+	cm map[string]*entity.Color
+}
+
+func (svc *Service) Inject() (err error) {
+	return nil
+}
+
+func (svc *Service) GetByName(name string) (res interfaces.Color, err error) {
+	res, ok := svc.cm[name]
+	if !ok {
+		return res, errors.ErrorModelNotFound
+	}
+	return res, err
+}
+
+func (svc *Service) GetRandom() (res interfaces.Color, err error) {
+	if len(svc.cl) == 0 {
+		hexStr, err := svc.getRandomColorHex()
+		if err != nil {
+			return res, err
+		}
+		return &entity.Color{Hex: hexStr}, nil
+	}
+
+	idx := rand.Intn(len(svc.cl))
+	return svc.cl[idx], nil
+}
+
+func (svc *Service) getRandomColorHex() (res string, err error) {
+	n := 6
+	arr := make([]string, n)
+	for i := 0; i < n; i++ {
+		arr[i], err = svc.getRandomHexChar()
+		if err != nil {
+			return res, err
+		}
+	}
+	return strings.Join(arr, ""), nil
+}
+
+func (svc *Service) getRandomHexChar() (res string, err error) {
+	n := rand.Intn(16)
+	b := []byte(strconv.Itoa(n))
+	h := make([]byte, 1)
+	hex.Encode(h, b)
+	return string(h), nil
+}

core/config/base.go

@@ -0,0 +1,21 @@
+package config
+
+import (
+	"github.com/mitchellh/go-homedir"
+	"github.com/spf13/viper"
+	"path/filepath"
+)
+
+var HomeDirPath, _ = homedir.Dir()
+
+const configDirName = ".crawlab"
+
+const configName = "config.json"
+
+func GetConfigPath() string {
+	if viper.GetString("metadata") != "" {
+		MetadataPath := viper.GetString("metadata")
+		return filepath.Join(MetadataPath, configName)
+	}
+	return filepath.Join(HomeDirPath, configDirName, configName)
+}

core/config/config.go

@@ -0,0 +1,88 @@
+package config
+
+import (
+	"bytes"
+	"github.com/apex/log"
+	"github.com/crawlab-team/go-trace"
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/viper"
+	"strings"
+)
+
+func init() {
+	// config instance
+	c := Config{Name: ""}
+
+	// init config file
+	if err := c.Init(); err != nil {
+		log.Warn("unable to init config")
+		return
+	}
+
+	// watch config change and load responsively
+	c.WatchConfig()
+
+	// init log level
+	c.initLogLevel()
+}
+
+type Config struct {
+	Name string
+}
+
+type InitConfigOptions struct {
+	Name string
+}
+
+func (c *Config) WatchConfig() {
+	viper.WatchConfig()
+	viper.OnConfigChange(func(e fsnotify.Event) {
+		log.Infof("Config file changed: %s", e.Name)
+	})
+}
+
+func (c *Config) Init() (err error) {
+	// config
+	if c.Name != "" {
+		viper.SetConfigFile(c.Name) // if config file is set, load it accordingly
+	} else {
+		viper.AddConfigPath("./conf") // if no config file is set, load by default
+		viper.SetConfigName("config")
+	}
+
+	// config type as yaml
+	viper.SetConfigType("yaml") // default yaml
+
+	// auto env
+	viper.AutomaticEnv() // load matched environment variables
+
+	// env prefix
+	viper.SetEnvPrefix("CRAWLAB") // environment variable prefix as CRAWLAB
+
+	// replacer
+	replacer := strings.NewReplacer(".", "_")
+	viper.SetEnvKeyReplacer(replacer)
+
+	// read default config
+	defaultConfBuf := bytes.NewBufferString(DefaultConfigYaml)
+	if err := viper.ReadConfig(defaultConfBuf); err != nil {
+		return trace.TraceError(err)
+	}
+
+	// merge config
+	if err := viper.MergeInConfig(); err != nil { // viper parsing config file
+		return err
+	}
+
+	return nil
+}
+
+func (c *Config) initLogLevel() {
+	// set log level
+	logLevel := viper.GetString("log.level")
+	l, err := log.ParseLevel(logLevel)
+	if err != nil {
+		l = log.InfoLevel
+	}
+	log.SetLevel(l)
+}

core/config/config_test.go

@@ -0,0 +1,11 @@
+package config
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestInitConfig(t *testing.T) {
+	err := InitConfig()
+	require.Nil(t, err)
+}

core/config/default_config.go

@@ -0,0 +1,40 @@
+package config
+
+var DefaultConfigYaml = `
+info:
+  version: v0.6.3
+  edition: global.edition.community
+mongo:
+  host: localhost
+  port: 27017
+  db: crawlab_test
+  username: ""
+  password: ""
+  authSource: "admin"
+server:
+  host: 0.0.0.0
+  port: 8000
+spider:
+  fs: "/spiders"
+  workspace: "/workspace"
+  repo: "/repo"
+task:
+  workers: 16
+  cancelWaitSeconds: 30
+grpc:
+  address: localhost:9666
+  server:
+    address: 0.0.0.0:9666
+  authKey: Crawlab2021!
+fs:
+  filer:
+    proxy: http://localhost:8888
+    url: http://localhost:8000/filer
+    authKey: Crawlab2021!
+node:
+  master: Y
+api:
+  endpoint: http://localhost:8000
+log:
+  path: /var/log/crawlab
+`

core/config/path.go

@@ -0,0 +1,23 @@
+package config
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+type PathService struct {
+	cfgPath string
+}
+
+func (svc *PathService) GetConfigPath() (path string) {
+	return svc.cfgPath
+}
+
+func (svc *PathService) SetConfigPath(path string) {
+	svc.cfgPath = path
+}
+
+func NewConfigPathService() (svc interfaces.WithConfigPath) {
+	svc = &PathService{}
+	svc.SetConfigPath(GetConfigPath())
+	return svc
+}

core/config/version.go

@@ -0,0 +1,12 @@
+package config
+
+import "strings"
+
+const Version = "v0.6.3"
+
+func GetVersion() (v string) {
+	if strings.HasPrefix(Version, "v") {
+		return Version
+	}
+	return "v" + Version
+}

core/constants/action.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	ActionTypeVisit          = "visit"
+	ActionTypeInstallDep     = "install_dep"
+	ActionTypeInstallLang    = "install_lang"
+	ActionTypeViewDisclaimer = "view_disclaimer"
+)

core/constants/anchor.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	AnchorStartStage = "START_STAGE"
+	AnchorStartUrl   = "START_URL"
+	AnchorItems      = "ITEMS"
+	AnchorParsers    = "PARSERS"
+)

core/constants/auth.go

@@ -0,0 +1,7 @@
+package constants
+
+const (
+	OwnerTypeAll    = "all"
+	OwnerTypeMe     = "me"
+	OwnerTypePublic = "public"
+)

core/constants/cache.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	CacheColName  = "cache"
+	CacheColKey   = "k"
+	CacheColValue = "v"
+	CacheColTime  = "t"
+)

core/constants/channels.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	ChannelAllNode = "nodes:public"
+
+	ChannelWorkerNode = "nodes:"
+
+	ChannelMasterNode = "nodes:master"
+)

core/constants/common.go

@@ -0,0 +1,6 @@
+package constants
+
+const (
+	ASCENDING  = "asc"
+	DESCENDING = "dsc"
+)

core/constants/config_spider.go

@@ -0,0 +1,6 @@
+package constants
+
+const (
+	EngineScrapy = "scrapy"
+	EngineColly  = "colly"
+)

core/constants/data_collection.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	DataCollectionKey = "_col"
+)

core/constants/data_field.go

@@ -0,0 +1,12 @@
+package constants
+
+const (
+	DataFieldTypeGeneral  = "general"
+	DataFieldTypeNumeric  = "numeric"
+	DataFieldTypeDate     = "date"
+	DataFieldTypeCurrency = "currency"
+	DataFieldTypeUrl      = "url"
+	DataFieldTypeImage    = "image"
+	DataFieldTypeAudio    = "audio"
+	DataFieldTypeVideo    = "video"
+)

core/constants/database.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	ColJob = "jobs"
+)

core/constants/delegate.go

@@ -0,0 +1 @@
+package constants

core/constants/ds.go

@@ -0,0 +1,31 @@
+package constants
+
+const (
+	DataSourceTypeMongo         = "mongo"
+	DataSourceTypeMysql         = "mysql"
+	DataSourceTypePostgresql    = "postgresql"
+	DataSourceTypeMssql         = "mssql"
+	DataSourceTypeSqlite        = "sqlite"
+	DataSourceTypeCockroachdb   = "cockroachdb"
+	DataSourceTypeElasticSearch = "elasticsearch"
+	DataSourceTypeKafka         = "kafka"
+)
+
+const (
+	DefaultHost = "localhost"
+)
+
+const (
+	DefaultMongoPort         = "27017"
+	DefaultMysqlPort         = "3306"
+	DefaultPostgresqlPort    = "5432"
+	DefaultMssqlPort         = "1433"
+	DefaultCockroachdbPort   = "26257"
+	DefaultElasticsearchPort = "9200"
+	DefaultKafkaPort         = "9092"
+)
+
+const (
+	DataSourceStatusOnline  = "on"
+	DataSourceStatusOffline = "off"
+)

core/constants/encrypt.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	DefaultEncryptServerKey = "0123456789abcdef"
+)

core/constants/errors.go

@@ -0,0 +1,30 @@
+package constants
+
+import (
+	"errors"
+)
+
+var (
+	//ErrorMongoError                = e.NewSystemOPError(1001, "system error:[mongo]%s", http.StatusInternalServerError)
+	//ErrorUserNotFound              = e.NewBusinessError(10001, "user not found.", http.StatusUnauthorized)
+	//ErrorUsernameOrPasswordInvalid = e.NewBusinessError(11001, "username or password invalid", http.StatusUnauthorized)
+	ErrAlreadyExists    = errors.New("already exists")
+	ErrNotExists        = errors.New("not exists")
+	ErrForbidden        = errors.New("forbidden")
+	ErrInvalidOperation = errors.New("invalid operation")
+	ErrInvalidOptions   = errors.New("invalid options")
+	ErrNoTasksAvailable = errors.New("no tasks available")
+	ErrInvalidType      = errors.New("invalid type")
+	ErrInvalidSignal    = errors.New("invalid signal")
+	ErrEmptyValue       = errors.New("empty value")
+	ErrTaskError        = errors.New("task error")
+	ErrTaskLost         = errors.New("task lost")
+	ErrTaskCancelled    = errors.New("task cancelled")
+	ErrUnableToCancel   = errors.New("unable to cancel")
+	ErrUnableToDispose  = errors.New("unable to dispose")
+	ErrAlreadyDisposed  = errors.New("already disposed")
+	ErrStopped          = errors.New("stopped")
+	ErrMissingCol       = errors.New("missing col")
+	ErrInvalidValue     = errors.New("invalid value")
+	ErrInvalidCronSpec  = errors.New("invalid cron spec")
+)

core/constants/event.go

@@ -0,0 +1,6 @@
+package constants
+
+const (
+	GrpcEventServiceTypeRegister = "register"
+	GrpcEventServiceTypeSend     = "send"
+)

core/constants/export.go

@@ -0,0 +1,6 @@
+package constants
+
+const (
+	ExportTypeCsv  = "csv"
+	ExportTypeJson = "json"
+)

core/constants/file.go

@@ -0,0 +1,5 @@
+package constants
+
+const EmptyFileData = " "
+
+const FsKeepFileName = ".gitkeep"

core/constants/filer.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	DefaultFilerAuthKey = "Crawlab2021!"
+)

core/constants/filter.go

@@ -0,0 +1,28 @@
+package constants
+
+const (
+	FilterQueryFieldConditions = "conditions"
+	FilterQueryFieldAll        = "all"
+)
+
+const (
+	FilterObjectTypeString  = "string"
+	FilterObjectTypeNumber  = "number"
+	FilterObjectTypeBoolean = "boolean"
+)
+
+const (
+	FilterOpNotSet           = "ns"
+	FilterOpContains         = "c"
+	FilterOpNotContains      = "nc"
+	FilterOpRegex            = "r"
+	FilterOpEqual            = "eq"
+	FilterOpNotEqual         = "ne"
+	FilterOpIn               = "in"
+	FilterOpNotIn            = "nin"
+	FilterOpGreaterThan      = "gt"
+	FilterOpLessThan         = "lt"
+	FilterOpGreaterThanEqual = "gte"
+	FilterOpLessThanEqual    = "lte"
+	FilterOpSearch           = "s"
+)

core/constants/git.go

@@ -0,0 +1,16 @@
+package constants
+
+const (
+	GitAuthTypeHttp = "http"
+	GitAuthTypeSsh  = "ssh"
+)
+
+const (
+	GitRemoteNameUpstream = "upstream"
+	GitRemoteNameOrigin   = "origin"
+)
+
+const (
+	GitBranchMaster = "master"
+	GitBranchMain   = "main"
+)

core/constants/grpc.go

@@ -0,0 +1,17 @@
+package constants
+
+const (
+	DefaultGrpcServerHost       = ""
+	DefaultGrpcServerPort       = "9666"
+	DefaultGrpcClientRemoteHost = "localhost"
+	DefaultGrpcClientRemotePort = DefaultGrpcServerPort
+	DefaultGrpcAuthKey          = "Crawlab2021!"
+)
+
+const (
+	GrpcHeaderAuthorization = "authorization"
+)
+
+const (
+	GrpcSubscribeTypeNode = "node"
+)

core/constants/http.go

@@ -0,0 +1,11 @@
+package constants
+
+const (
+	HttpResponseStatusOk       = "ok"
+	HttpResponseMessageSuccess = "success"
+	HttpResponseMessageError   = "error"
+)
+
+const (
+	HttpContentTypeApplicationJson = "application/json"
+)

core/constants/log.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	ErrorRegexPattern = "(?:[ :,.]|^)((?:error|exception|traceback)s?)(?:[ :,.]|$)"
+)

core/constants/message.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	MsgTypeGetLog        = "get-log"
+	MsgTypeGetSystemInfo = "get-sys-info"
+	MsgTypeCancelTask    = "cancel-task"
+	MsgTypeRemoveLog     = "remove-log"
+	MsgTypeRemoveSpider  = "remove-spider"
+)

core/constants/node.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	NodeStatusUnregistered = "u"
+	NodeStatusRegistered   = "r"
+	NodeStatusOnline       = "on"
+	NodeStatusOffline      = "off"
+)

core/constants/notification.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	NotificationTriggerTaskFinish       = "task_finish"
+	NotificationTriggerTaskError        = "task_error"
+	NotificationTriggerTaskEmptyResults = "task_empty_results"
+	NotificationTriggerTaskNever        = "task_never"
+)

core/constants/pagination.go

@@ -0,0 +1,4 @@
+package constants
+
+var PaginationDefaultPage = 1
+var PaginationDefaultSize = 10

core/constants/register.go

@@ -0,0 +1,8 @@
+package constants
+
+const (
+	RegisterTypeMac        = "mac"
+	RegisterTypeIp         = "ip"
+	RegisterTypeHostname   = "hostname"
+	RegisterTypeCustomName = "customName"
+)

core/constants/results.go

@@ -0,0 +1,10 @@
+package constants
+
+const (
+	HashKey = "_h"
+)
+
+const (
+	DedupTypeIgnore    = "ignore"
+	DedupTypeOverwrite = "overwrite"
+)

core/constants/rpc.go

@@ -0,0 +1,12 @@
+package constants
+
+const (
+	RpcInstallLang          = "install_lang"
+	RpcInstallDep           = "install_dep"
+	RpcUninstallDep         = "uninstall_dep"
+	RpcGetInstalledDepList  = "get_installed_dep_list"
+	RpcGetLang              = "get_lang"
+	RpcCancelTask           = "cancel_task"
+	RpcGetSystemInfoService = "get_system_info"
+	RpcRemoveSpider         = "remove_spider"
+)

core/constants/schedule.go

@@ -0,0 +1,10 @@
+package constants
+
+const (
+	ScheduleStatusStop    = "stopped"
+	ScheduleStatusRunning = "running"
+	ScheduleStatusError   = "error"
+
+	ScheduleStatusErrorNotFoundNode   = "Not Found Node"
+	ScheduleStatusErrorNotFoundSpider = "Not Found Spider"
+)

core/constants/scrapy.go

@@ -0,0 +1,5 @@
+package constants
+
+const ScrapyProtectedStageNames = ""
+
+const ScrapyProtectedFieldNames = "_id,task_id,ts"

core/constants/signal.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	SignalQuit = iota
+)

core/constants/sort.go

@@ -0,0 +1,5 @@
+package constants
+
+const (
+	SortQueryField = "sort"
+)

core/constants/system.go

@@ -0,0 +1,25 @@
+package constants
+
+const (
+	Windows = "windows"
+	Linux   = "linux"
+	Darwin  = "darwin"
+)
+
+const (
+	Python = "python"
+	Nodejs = "node"
+	Java   = "java"
+)
+
+const (
+	InstallStatusNotInstalled    = "not-installed"
+	InstallStatusInstalling      = "installing"
+	InstallStatusInstallingOther = "installing-other"
+	InstallStatusInstalled       = "installed"
+)
+
+const (
+	LangTypeLang      = "lang"
+	LangTypeWebDriver = "webdriver"
+)

core/constants/task.go

@@ -0,0 +1,39 @@
+package constants
+
+const (
+	TaskStatusPending   = "pending"
+	TaskStatusRunning   = "running"
+	TaskStatusFinished  = "finished"
+	TaskStatusError     = "error"
+	TaskStatusCancelled = "cancelled"
+	TaskStatusAbnormal  = "abnormal"
+)
+
+const (
+	RunTypeAllNodes      = "all-nodes"
+	RunTypeRandom        = "random"
+	RunTypeSelectedNodes = "selected-nodes"
+)
+
+const (
+	TaskTypeSpider = "spider"
+	TaskTypeSystem = "system"
+)
+
+type TaskSignal int
+
+const (
+	TaskSignalFinish TaskSignal = iota
+	TaskSignalCancel
+	TaskSignalError
+	TaskSignalLost
+)
+
+const (
+	TaskListQueuePrefixPublic = "tasks:public"
+	TaskListQueuePrefixNodes  = "tasks:nodes"
+)
+
+const (
+	TaskKey = "_tid"
+)

core/constants/user.go

@@ -0,0 +1,15 @@
+package constants
+
+const (
+	RoleAdmin  = "admin"
+	RoleNormal = "normal"
+)
+
+const (
+	DefaultAdminUsername = "admin"
+	DefaultAdminPassword = "admin"
+)
+
+const (
+	UserContextKey = "user"
+)

core/constants/variable.go

@@ -0,0 +1,9 @@
+package constants
+
+const (
+	String  = "string"
+	Number  = "number"
+	Boolean = "boolean"
+	Array   = "array"
+	Object  = "object"
+)

core/container/container.go

@@ -0,0 +1,11 @@
+package container
+
+import (
+	"go.uber.org/dig"
+)
+
+var c = dig.New()
+
+func GetContainer() *dig.Container {
+	return c
+}

core/controllers/base.go

@@ -0,0 +1,69 @@
+package controllers
+
+import "github.com/gin-gonic/gin"
+
+const (
+	ControllerIdNode = iota << 1
+	ControllerIdProject
+	ControllerIdSpider
+	ControllerIdTask
+	ControllerIdJob
+	ControllerIdSchedule
+	ControllerIdUser
+	ControllerIdSetting
+	ControllerIdToken
+	ControllerIdVariable
+	ControllerIdTag
+	ControllerIdLogin
+	ControllerIdColor
+	ControllerIdDataSource
+	ControllerIdDataCollection
+	ControllerIdResult
+	ControllerIdStats
+	ControllerIdFiler
+	ControllerIdGit
+	ControllerIdRole
+	ControllerIdPermission
+	ControllerIdExport
+	ControllerIdNotification
+	ControllerIdFilter
+	ControllerIdEnvironment
+	ControllerIdSync
+
+	ControllerIdVersion
+	ControllerIdI18n
+	ControllerIdSystemInfo
+	ControllerIdDemo
+)
+
+type ControllerId int
+
+type BasicController interface {
+	Get(c *gin.Context)
+	Post(c *gin.Context)
+	Put(c *gin.Context)
+	Delete(c *gin.Context)
+}
+
+type ListController interface {
+	BasicController
+	GetList(c *gin.Context)
+	PutList(c *gin.Context)
+	PostList(c *gin.Context)
+	DeleteList(c *gin.Context)
+}
+
+type Action struct {
+	Method      string
+	Path        string
+	HandlerFunc gin.HandlerFunc
+}
+
+type ActionController interface {
+	Actions() (actions []Action)
+}
+
+type ListActionController interface {
+	ListController
+	ActionController
+}

core/controllers/base_v2.go

@@ -0,0 +1,226 @@
+package controllers
+
+import (
+	"errors"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+type BaseControllerV2[T any] struct {
+	modelSvc *service.ModelServiceV2[T]
+	actions  []Action
+}
+
+func (ctr *BaseControllerV2[T]) GetById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	model, err := ctr.modelSvc.GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, model)
+}
+
+func (ctr *BaseControllerV2[T]) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		ctr.getAll(c)
+		return
+	}
+
+	// get list
+	ctr.getList(c)
+}
+
+func (ctr *BaseControllerV2[T]) Post(c *gin.Context) {
+	var model T
+	if err := c.ShouldBindJSON(&model); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	u := GetUserFromContextV2(c)
+	m := any(&model).(interfaces.ModelV2)
+	m.SetId(primitive.NewObjectID())
+	m.SetCreated(u.Id)
+	m.SetUpdated(u.Id)
+	col := ctr.modelSvc.GetCol()
+	res, err := col.GetCollection().InsertOne(col.GetContext(), m)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	result, err := ctr.modelSvc.GetById(res.InsertedID.(primitive.ObjectID))
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, result)
+}
+
+func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	var model T
+	if err := c.ShouldBindJSON(&model); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+	m := any(&model).(interfaces.ModelV2)
+	m.SetId(primitive.NewObjectID())
+	m.SetUpdated(u.Id)
+
+	if err := ctr.modelSvc.ReplaceById(id, model); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	result, err := ctr.modelSvc.GetById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, result)
+}
+
+func (ctr *BaseControllerV2[T]) PatchList(c *gin.Context) {
+	type Payload struct {
+		Ids    []primitive.ObjectID `json:"ids"`
+		Update bson.M               `json:"update"`
+	}
+
+	var payload Payload
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// query
+	query := bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}
+
+	// update
+	if err := ctr.modelSvc.UpdateMany(query, bson.M{"$set": payload.Update}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) DeleteById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr.modelSvc.DeleteById(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) {
+	type Payload struct {
+		Ids []primitive.ObjectID `json:"ids"`
+	}
+
+	var payload Payload
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr.modelSvc.DeleteMany(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) {
+	models, err := ctr.modelSvc.GetMany(nil, &mongo.FindOptions{
+		Sort: bson.D{{"_id", -1}},
+	})
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	total, err := ctr.modelSvc.Count(nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithListData(c, models, total)
+}
+
+func (ctr *BaseControllerV2[T]) getList(c *gin.Context) {
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if errors.Is(err, mongo2.ErrNoDocuments) {
+			HandleSuccessWithListData(c, nil, 0)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// total count
+	total, err := ctr.modelSvc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, models, total)
+}
+
+func NewControllerV2[T any](actions ...Action) *BaseControllerV2[T] {
+	ctr := &BaseControllerV2[T]{
+		modelSvc: service.NewModelServiceV2[T](),
+		actions:  actions,
+	}
+	return ctr
+}

core/controllers/base_v2_test.go

@@ -0,0 +1,165 @@
+package controllers_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/spf13/viper"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+
+}
+
+// TestModel is a simple struct to be used as a model in tests
+type TestModel models.TestModel
+
+var TestToken string
+
+// SetupTestDB sets up the test database
+func SetupTestDB() {
+	viper.Set("mongo.db", "testdb")
+	modelSvc := service.NewModelServiceV2[models.UserV2]()
+	u := models.UserV2{
+		Username: "admin",
+	}
+	id, err := modelSvc.InsertOne(u)
+	if err != nil {
+		panic(err)
+	}
+	u.SetId(id)
+
+	userSvc, err := user.GetUserServiceV2()
+	if err != nil {
+		panic(err)
+	}
+	token, err := userSvc.MakeToken(&u)
+	if err != nil {
+		panic(err)
+	}
+	TestToken = token
+}
+
+// SetupRouter sets up the gin router for testing
+func SetupRouter() *gin.Engine {
+	router := gin.Default()
+	return router
+}
+
+// CleanupTestDB cleans up the test database
+func CleanupTestDB() {
+	mongo.GetMongoDb("testdb").Drop(context.Background())
+}
+
+func TestBaseControllerV2_GetById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Insert a test document
+	id, err := service.NewModelServiceV2[TestModel]().InsertOne(TestModel{Name: "test"})
+	assert.NoError(t, err)
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.GET("/testmodels/:id", ctr.GetById)
+
+	// Create a test request
+	req, _ := http.NewRequest("GET", "/testmodels/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	var response controllers.Response[TestModel]
+	err = json.Unmarshal(w.Body.Bytes(), &response)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", response.Data.Name)
+}
+
+func TestBaseControllerV2_Post(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.POST("/testmodels", ctr.Post)
+
+	// Create a test request
+	testModel := TestModel{Name: "test"}
+	jsonValue, _ := json.Marshal(testModel)
+	req, _ := http.NewRequest("POST", "/testmodels", bytes.NewBuffer(jsonValue))
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	var response controllers.Response[TestModel]
+	err := json.Unmarshal(w.Body.Bytes(), &response)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", response.Data.Name)
+
+	// Check if the document was inserted into the database
+	result, err := service.NewModelServiceV2[TestModel]().GetById(response.Data.Id)
+	assert.NoError(t, err)
+	assert.Equal(t, "test", result.Name)
+}
+
+func TestBaseControllerV2_DeleteById(t *testing.T) {
+	SetupTestDB()
+	defer CleanupTestDB()
+
+	// Insert a test document
+	id, err := service.NewModelServiceV2[TestModel]().InsertOne(TestModel{Name: "test"})
+	assert.NoError(t, err)
+
+	// Initialize the controller
+	ctr := controllers.NewControllerV2[TestModel]()
+
+	// Set up the router
+	router := SetupRouter()
+	router.Use(middlewares.AuthorizationMiddlewareV2())
+	router.DELETE("/testmodels/:id", ctr.DeleteById)
+
+	// Create a test request
+	req, _ := http.NewRequest("DELETE", "/testmodels/"+id.Hex(), nil)
+	req.Header.Set("Authorization", TestToken)
+	w := httptest.NewRecorder()
+
+	// Serve the request
+	router.ServeHTTP(w, req)
+
+	// Check the response
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	// Check if the document was deleted from the database
+	_, err = service.NewModelServiceV2[TestModel]().GetById(id)
+	assert.Error(t, err)
+}

core/controllers/binder.go

@@ -0,0 +1,14 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+)
+
+type BinderInterface interface {
+	Bind(c *gin.Context) (res interfaces.Model, err error)
+	BindList(c *gin.Context) (res []interfaces.Model, err error)
+	BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error)
+	BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error)
+}

core/controllers/binder_json.go

@@ -0,0 +1,208 @@
+package controllers
+
+import (
+	"encoding/json"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+)
+
+func NewJsonBinder(id ControllerId) (b *JsonBinder) {
+	return &JsonBinder{
+		id: id,
+	}
+}
+
+type JsonBinder struct {
+	id ControllerId
+}
+
+func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) {
+	// declare
+	m := models.NewModelMap()
+
+	switch b.id {
+	case ControllerIdNode:
+		err = c.ShouldBindJSON(&m.Node)
+		return &m.Node, err
+	case ControllerIdProject:
+		err = c.ShouldBindJSON(&m.Project)
+		return &m.Project, err
+	case ControllerIdSpider:
+		err = c.ShouldBindJSON(&m.Spider)
+		return &m.Spider, err
+	case ControllerIdTask:
+		err = c.ShouldBindJSON(&m.Task)
+		return &m.Task, err
+	case ControllerIdJob:
+		err = c.ShouldBindJSON(&m.Job)
+		return &m.Job, err
+	case ControllerIdSchedule:
+		err = c.ShouldBindJSON(&m.Schedule)
+		return &m.Schedule, err
+	case ControllerIdUser:
+		err = c.ShouldBindJSON(&m.User)
+		return &m.User, nil
+	case ControllerIdSetting:
+		err = c.ShouldBindJSON(&m.Setting)
+		return &m.Setting, nil
+	case ControllerIdToken:
+		err = c.ShouldBindJSON(&m.Token)
+		return &m.Token, nil
+	case ControllerIdVariable:
+		err = c.ShouldBindJSON(&m.Variable)
+		return &m.Variable, nil
+	case ControllerIdTag:
+		err = c.ShouldBindJSON(&m.Tag)
+		return &m.Tag, nil
+	case ControllerIdDataSource:
+		err = c.ShouldBindJSON(&m.DataSource)
+		return &m.DataSource, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollection)
+		return &m.DataCollection, nil
+	case ControllerIdGit:
+		err = c.ShouldBindJSON(&m.Git)
+		return &m.Git, nil
+	case ControllerIdRole:
+		err = c.ShouldBindJSON(&m.Role)
+		return &m.Role, nil
+	case ControllerIdPermission:
+		err = c.ShouldBindJSON(&m.Permission)
+		return &m.Permission, nil
+	case ControllerIdEnvironment:
+		err = c.ShouldBindJSON(&m.Environment)
+		return &m.Environment, nil
+	default:
+		return nil, errors.ErrorControllerInvalidControllerId
+	}
+}
+
+func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) {
+	// declare
+	m := models.NewModelListMap()
+
+	// bind
+	switch b.id {
+	case ControllerIdNode:
+		err = c.ShouldBindJSON(&m.Nodes)
+		return m.Nodes, err
+	case ControllerIdProject:
+		err = c.ShouldBindJSON(&m.Projects)
+		return m.Projects, err
+	case ControllerIdSpider:
+		err = c.ShouldBindJSON(&m.Spiders)
+		return m.Spiders, err
+	case ControllerIdTask:
+		err = c.ShouldBindJSON(&m.Tasks)
+		return m.Tasks, err
+	case ControllerIdJob:
+		err = c.ShouldBindJSON(&m.Jobs)
+		return m.Jobs, err
+	case ControllerIdSchedule:
+		err = c.ShouldBindJSON(&m.Schedules)
+		return m.Schedules, err
+	case ControllerIdUser:
+		err = c.ShouldBindJSON(&m.Users)
+		return m.Users, nil
+	case ControllerIdSetting:
+		err = c.ShouldBindJSON(&m.Settings)
+		return m.Settings, nil
+	case ControllerIdToken:
+		err = c.ShouldBindJSON(&m.Tokens)
+		return m.Tokens, nil
+	case ControllerIdVariable:
+		err = c.ShouldBindJSON(&m.Variables)
+		return m.Variables, nil
+	case ControllerIdTag:
+		err = c.ShouldBindJSON(&m.Tags)
+		return m.Tags, nil
+	case ControllerIdDataSource:
+		err = c.ShouldBindJSON(&m.DataSources)
+		return m.DataSources, nil
+	case ControllerIdDataCollection:
+		err = c.ShouldBindJSON(&m.DataCollections)
+		return m.DataCollections, nil
+	case ControllerIdGit:
+		err = c.ShouldBindJSON(&m.Gits)
+		return m.Gits, nil
+	case ControllerIdRole:
+		err = c.ShouldBindJSON(&m.Roles)
+		return m.Roles, nil
+	case ControllerIdEnvironment:
+		err = c.ShouldBindJSON(&m.Environments)
+		return m.Environments, nil
+	default:
+		return nil, errors.ErrorControllerInvalidControllerId
+	}
+}
+
+func (b *JsonBinder) BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) {
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		return payload, err
+	}
+	return payload, nil
+}
+
+func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) {
+	// declare
+	m := models.NewModelMap()
+
+	// bind
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		return payload, nil, err
+	}
+
+	// validate
+	if len(payload.Ids) == 0 ||
+		len(payload.Fields) == 0 {
+		return payload, nil, errors.ErrorControllerRequestPayloadInvalid
+	}
+
+	// unmarshall
+	switch b.id {
+	case ControllerIdNode:
+		err = json.Unmarshal([]byte(payload.Data), &m.Node)
+		return payload, &m.Node, err
+	case ControllerIdProject:
+		err = json.Unmarshal([]byte(payload.Data), &m.Project)
+		return payload, &m.Project, err
+	case ControllerIdSpider:
+		err = json.Unmarshal([]byte(payload.Data), &m.Spider)
+		return payload, &m.Spider, err
+	case ControllerIdTask:
+		err = json.Unmarshal([]byte(payload.Data), &m.Task)
+		return payload, &m.Task, err
+	case ControllerIdJob:
+		err = json.Unmarshal([]byte(payload.Data), &m.Job)
+		return payload, &m.Job, err
+	case ControllerIdSchedule:
+		err = json.Unmarshal([]byte(payload.Data), &m.Schedule)
+		return payload, &m.Schedule, err
+	case ControllerIdUser:
+		err = json.Unmarshal([]byte(payload.Data), &m.User)
+		return payload, &m.User, err
+	case ControllerIdSetting:
+		err = json.Unmarshal([]byte(payload.Data), &m.Setting)
+		return payload, &m.Setting, err
+	case ControllerIdToken:
+		err = json.Unmarshal([]byte(payload.Data), &m.Token)
+		return payload, &m.Token, err
+	case ControllerIdVariable:
+		err = json.Unmarshal([]byte(payload.Data), &m.Variable)
+		return payload, &m.Variable, err
+	case ControllerIdDataSource:
+		err = json.Unmarshal([]byte(payload.Data), &m.DataSource)
+		return payload, &m.DataSource, err
+	case ControllerIdDataCollection:
+		err = json.Unmarshal([]byte(payload.Data), &m.DataCollection)
+		return payload, &m.DataCollection, err
+	case ControllerIdEnvironment:
+		err = json.Unmarshal([]byte(payload.Data), &m.Environment)
+		return payload, &m.Environment, err
+	default:
+		return payload, nil, errors.ErrorControllerInvalidControllerId
+	}
+}

core/controllers/data_collection.go

@@ -0,0 +1,103 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var DataCollectionController *dataCollectionController
+
+func getDataCollectionActions() []Action {
+	ctx := newDataCollectionContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/indexes",
+			HandlerFunc: ctx.postIndexes,
+		},
+	}
+}
+
+type dataCollectionController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *dataCollectionContext
+}
+
+type dataCollectionContext struct {
+	modelSvc  service.ModelService
+	resultSvc interfaces.ResultService
+}
+
+func (ctx *dataCollectionContext) postIndexes(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	dc, err := ctx.modelSvc.GetDataCollectionById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	for _, f := range dc.Fields {
+		if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{
+			Keys: f.Key,
+		}); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccess(c)
+}
+
+var _dataCollectionCtx *dataCollectionContext
+
+func newDataCollectionContext() *dataCollectionContext {
+	if _dataCollectionCtx != nil {
+		return _dataCollectionCtx
+	}
+
+	// context
+	ctx := &dataCollectionContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+	) {
+		ctx.modelSvc = modelSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	_dataCollectionCtx = ctx
+
+	return ctx
+}
+
+func newDataCollectionController() *dataCollectionController {
+	actions := getDataCollectionActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions)
+	ctx := newDataCollectionContext()
+
+	return &dataCollectionController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/data_source.go

@@ -0,0 +1,148 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/ds"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	interfaces2 "github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var DataSourceController *dataSourceController
+
+func getDataSourceActions() []Action {
+	ctx := newDataSourceContext()
+	return []Action{
+		{
+			Path:        "/:id/change-password",
+			Method:      http.MethodPost,
+			HandlerFunc: ctx.changePassword,
+		},
+	}
+}
+
+type dataSourceController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *dataSourceContext
+}
+
+func (ctr *dataSourceController) Post(c *gin.Context) {
+	// data source
+	var _ds models.DataSource
+	if err := c.ShouldBindJSON(&_ds); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// add data source to db
+	if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error {
+		if err := delegate.NewModelDelegate(&_ds).Add(); err != nil {
+			return trace.TraceError(err)
+		}
+		pwd, err := utils.EncryptAES(_ds.Password)
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		p := models.Password{Id: _ds.Id, Password: pwd}
+		if err := delegate.NewModelDelegate(&p).Add(); err != nil {
+			return trace.TraceError(err)
+		}
+		return nil
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// check data source status
+	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
+
+	HandleSuccess(c)
+}
+
+func (ctr *dataSourceController) Put(c *gin.Context) {
+	// data source
+	var _ds models.DataSource
+	if err := c.ShouldBindJSON(&_ds); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := delegate.NewModelDelegate(&_ds).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// check data source status
+	go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }()
+}
+
+type dataSourceContext struct {
+	dsSvc interfaces.DataSourceService
+}
+
+var _dataSourceCtx *dataSourceContext
+
+func newDataSourceContext() *dataSourceContext {
+	if _dataSourceCtx != nil {
+		return _dataSourceCtx
+	}
+	dsSvc, err := ds.GetDataSourceService()
+	if err != nil {
+		panic(err)
+	}
+	_dataSourceCtx = &dataSourceContext{
+		dsSvc: dsSvc,
+	}
+	return _dataSourceCtx
+}
+
+func (ctx *dataSourceContext) changePassword(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var payload map[string]string
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	password, ok := payload["password"]
+	if !ok {
+		HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields)
+		return
+	}
+	if err := ctx.dsSvc.ChangePassword(id, password); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func newDataSourceController() *dataSourceController {
+	actions := getDataSourceActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions)
+	ctx := newDataSourceContext()
+
+	return &dataSourceController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/delegate_action.go

@@ -0,0 +1,17 @@
+package controllers
+
+func NewActionControllerDelegate(id ControllerId, actions []Action) (d *ActionControllerDelegate) {
+	return &ActionControllerDelegate{
+		id:      id,
+		actions: actions,
+	}
+}
+
+type ActionControllerDelegate struct {
+	id      ControllerId
+	actions []Action
+}
+
+func (ctr *ActionControllerDelegate) Actions() (actions []Action) {
+	return ctr.actions
+}

core/controllers/delegate_basic.go

@@ -0,0 +1,99 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	delegate2 "github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func NewBasicControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *BasicControllerDelegate) {
+	return &BasicControllerDelegate{
+		id:  id,
+		svc: svc,
+	}
+}
+
+type BasicControllerDelegate struct {
+	id  ControllerId
+	svc interfaces.ModelBaseService
+}
+
+func (d *BasicControllerDelegate) Get(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := d.svc.GetById(id)
+	if err == mongo2.ErrNoDocuments {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Post(c *gin.Context) {
+	doc, err := NewJsonBinder(d.id).Bind(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Put(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := NewJsonBinder(d.id).Bind(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if doc.GetId() != id {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+	_, err = d.svc.GetById(id)
+	if err != nil {
+		HandleErrorNotFound(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, doc)
+}
+
+func (d *BasicControllerDelegate) Delete(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	doc, err := d.svc.GetById(oid)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Delete(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}

core/controllers/delegate_list.go

@@ -0,0 +1,222 @@
+package controllers
+
+import (
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"reflect"
+	"time"
+)
+
+func NewListControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *ListControllerDelegate) {
+	if svc == nil {
+		panic(errors.ErrorControllerNoModelService)
+	}
+
+	return &ListControllerDelegate{
+		id:  id,
+		svc: svc,
+		bc:  NewBasicControllerDelegate(id, svc),
+	}
+}
+
+type ListControllerDelegate struct {
+	id  ControllerId
+	svc interfaces.ModelBaseService
+	bc  BasicController
+}
+
+func (d *ListControllerDelegate) Get(c *gin.Context) {
+	d.bc.Get(c)
+}
+
+func (d *ListControllerDelegate) Post(c *gin.Context) {
+	d.bc.Post(c)
+}
+
+func (d *ListControllerDelegate) Put(c *gin.Context) {
+	d.bc.Put(c)
+}
+
+func (d *ListControllerDelegate) Delete(c *gin.Context) {
+	d.bc.Delete(c)
+}
+
+func (d *ListControllerDelegate) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		d.getAll(c)
+		return
+	}
+
+	// get list and total
+	l, total, err := d.getList(c)
+	if err != nil {
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, l, total)
+}
+
+func (d *ListControllerDelegate) PostList(c *gin.Context) {
+	// bind
+	docs, err := NewJsonBinder(d.id).BindList(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// success ids
+	var ids []primitive.ObjectID
+
+	// reflect
+	switch reflect.TypeOf(docs).Kind() {
+	case reflect.Slice, reflect.Array:
+		s := reflect.ValueOf(docs)
+		for i := 0; i < s.Len(); i++ {
+			item := s.Index(i)
+			if !item.CanAddr() {
+				HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
+				return
+			}
+			ptr := item.Addr()
+			doc, ok := ptr.Interface().(interfaces.Model)
+			if !ok {
+				HandleErrorInternalServerError(c, errors.ErrorModelInvalidType)
+				return
+			}
+			if err := delegate.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil {
+				_ = trace.TraceError(err)
+				continue
+			}
+			ids = append(ids, doc.GetId())
+		}
+	}
+
+	// check
+	items, err := utils.GetArrayItems(docs)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if len(ids) < len(items) {
+		HandleErrorInternalServerError(c, errors.ErrorControllerAddError)
+		return
+	}
+
+	// success
+	HandleSuccessWithData(c, docs)
+}
+
+func (d *ListControllerDelegate) PutList(c *gin.Context) {
+	payload, doc, err := NewJsonBinder(d.id).BindBatchRequestPayloadWithStringData(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// query
+	query := bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}
+
+	// update
+	if err := d.svc.UpdateDoc(query, doc, payload.Fields); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (d *ListControllerDelegate) DeleteList(c *gin.Context) {
+	payload, err := NewJsonBinder(d.id).BindBatchRequestPayload(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := d.svc.DeleteList(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (d *ListControllerDelegate) getAll(c *gin.Context) {
+	// get list
+	tic := time.Now()
+	log.Debugf("getAll -> d.svc.GetMany:start")
+	list, err := d.svc.GetList(nil, &mongo.FindOptions{
+		Sort: bson.D{{"_id", -1}},
+	})
+	if err != nil {
+		if err == mongo2.ErrNoDocuments {
+			HandleErrorNotFound(c, err)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+	log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
+	tic = time.Now()
+
+	// total count
+	tic = time.Now()
+	log.Debugf("getAll -> d.svc.Count:start")
+	total, err := d.svc.Count(nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds())
+
+	// response
+	HandleSuccessWithListData(c, list, total)
+}
+
+func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) {
+	// params
+	pagination := MustGetPagination(c)
+	query := MustGetFilterQuery(c)
+	sort := MustGetSortOption(c)
+
+	// get list
+	l, err = d.svc.GetList(query, &mongo.FindOptions{
+		Sort:  sort,
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+		} else {
+			HandleErrorInternalServerError(c, err)
+		}
+		return
+	}
+
+	// total count
+	total, err = d.svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	return l, total, nil
+}

core/controllers/delegate_list_action.go

@@ -0,0 +1,17 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+)
+
+func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) {
+	return &ListActionControllerDelegate{
+		NewListControllerDelegate(id, svc),
+		NewActionControllerDelegate(id, actions),
+	}
+}
+
+type ListActionControllerDelegate struct {
+	ListController
+	ActionController
+}

core/controllers/demo.go

@@ -0,0 +1,73 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func getDemoActions() []Action {
+	ctx := newDemoContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/import",
+			HandlerFunc: ctx.import_,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/reimport",
+			HandlerFunc: ctx.reimport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/cleanup",
+			HandlerFunc: ctx.cleanup,
+		},
+	}
+}
+
+type demoContext struct {
+}
+
+func (ctx *demoContext) import_(c *gin.Context) {
+	if err := utils.ImportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *demoContext) reimport(c *gin.Context) {
+	if err := utils.ReimportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *demoContext) cleanup(c *gin.Context) {
+	if err := utils.ReimportDemo(); err != nil {
+		trace.PrintError(err)
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+var _demoCtx *demoContext
+
+func newDemoContext() *demoContext {
+	if _demoCtx != nil {
+		return _demoCtx
+	}
+
+	_demoCtx = &demoContext{}
+
+	return _demoCtx
+}
+
+var DemoController ActionController

core/controllers/environment.go

@@ -0,0 +1,57 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+)
+
+var EnvironmentController *environmentController
+
+var EnvironmentActions []Action
+
+type environmentController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *environmentContext
+}
+
+type environmentContext struct {
+	modelSvc service.ModelService
+	userSvc  interfaces.UserService
+}
+
+func newEnvironmentContext() *environmentContext {
+	// context
+	ctx := &environmentContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		userSvc interfaces.UserService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newEnvironmentController() *environmentController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
+	d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions)
+	ctx := newEnvironmentContext()
+
+	return &environmentController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/export.go

@@ -0,0 +1,127 @@
+package controllers
+
+import (
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/export"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+var ExportController ActionController
+
+func getExportActions() []Action {
+	ctx := newExportContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:type",
+			HandlerFunc: ctx.postExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id",
+			HandlerFunc: ctx.getExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id/download",
+			HandlerFunc: ctx.getExportDownload,
+		},
+	}
+}
+
+type exportContext struct {
+	csvSvc  interfaces.ExportService
+	jsonSvc interfaces.ExportService
+}
+
+func (ctx *exportContext) postExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportTarget := c.Query("target")
+	exportFilter, _ := GetFilter(c)
+
+	var exportId string
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter)
+	case constants.ExportTypeJson:
+		exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exportId)
+}
+
+func (ctx *exportContext) getExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = ctx.csvSvc.GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = ctx.jsonSvc.GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exp)
+}
+
+func (ctx *exportContext) getExportDownload(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = ctx.csvSvc.GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = ctx.jsonSvc.GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	switch exportType {
+	case constants.ExportTypeCsv:
+		c.Header("Content-Type", "text/csv")
+	case constants.ExportTypeJson:
+		c.Header("Content-Type", "text/plain")
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
+	c.File(exp.GetDownloadPath())
+}
+
+func newExportContext() *exportContext {
+	return &exportContext{
+		csvSvc:  export.GetCsvService(),
+		jsonSvc: export.GetJsonService(),
+	}
+}

core/controllers/export_v2.go

@@ -0,0 +1,87 @@
+package controllers
+
+import (
+	"errors"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/export"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+)
+
+func PostExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportTarget := c.Query("target")
+	exportFilter, _ := GetFilter(c)
+
+	var exportId string
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exportId, err = export.GetCsvService().Export(exportType, exportTarget, exportFilter)
+	case constants.ExportTypeJson:
+		exportId, err = export.GetJsonService().Export(exportType, exportTarget, exportFilter)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+		return
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exportId)
+}
+
+func GetExport(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, exp)
+}
+
+func GetExportDownload(c *gin.Context) {
+	exportType := c.Param("type")
+	exportId := c.Param("id")
+
+	var exp interfaces.Export
+	var err error
+	switch exportType {
+	case constants.ExportTypeCsv:
+		exp, err = export.GetCsvService().GetExport(exportId)
+	case constants.ExportTypeJson:
+		exp, err = export.GetJsonService().GetExport(exportId)
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	switch exportType {
+	case constants.ExportTypeCsv:
+		c.Header("Content-Type", "text/csv")
+	case constants.ExportTypeJson:
+		c.Header("Content-Type", "text/plain")
+	default:
+		HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType)))
+	}
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath()))
+	c.File(exp.GetDownloadPath())
+}

core/controllers/filer.go

@@ -0,0 +1,130 @@
+package controllers
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/gin-gonic/gin"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"net/http"
+	"strings"
+)
+
+var FilerController ActionController
+
+func getFilerActions() []Action {
+	filerCtx := newFilerContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+		{
+			Method:      http.MethodDelete,
+			Path:        "*path",
+			HandlerFunc: filerCtx.do,
+		},
+	}
+}
+
+type filerContext struct {
+	endpoint string
+}
+
+func (ctx *filerContext) do(c *gin.Context) {
+	// request path
+	requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1)
+
+	// request url
+	requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath)
+	if c.Request.URL.RawQuery != "" {
+		requestUrl += "?" + c.Request.URL.RawQuery
+	}
+
+	// request body
+	bufR := bufio.NewScanner(c.Request.Body)
+	requestBody := req.BodyJSON(bufR.Bytes())
+
+	// request file uploads
+	var requestFileUploads []req.FileUpload
+	form, err := c.MultipartForm()
+	if err == nil {
+		for k, v := range form.File {
+			for _, fh := range v {
+				f, err := fh.Open()
+				if err != nil {
+					HandleErrorInternalServerError(c, err)
+					return
+				}
+				requestFileUploads = append(requestFileUploads, req.FileUpload{
+					FileName:  fh.Filename,
+					FieldName: k,
+					File:      f,
+				})
+			}
+		}
+	}
+
+	// request header
+	requestHeader := req.Header{}
+	for k, v := range c.Request.Header {
+		if len(v) > 0 {
+			requestHeader[k] = v[0]
+		}
+	}
+
+	// perform request
+	res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// status code check
+	statusCode := res.Response().StatusCode
+	if statusCode == http.StatusNotFound {
+		HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound)
+		return
+	}
+
+	// response
+	for k, v := range res.Response().Header {
+		if len(v) > 0 {
+			c.Header(k, v[0])
+		}
+	}
+	_, _ = c.Writer.Write(res.Bytes())
+	c.AbortWithStatus(statusCode)
+}
+
+var _filerCtx *filerContext
+
+func newFilerContext() *filerContext {
+	if _filerCtx != nil {
+		return _filerCtx
+	}
+
+	ctx := &filerContext{
+		endpoint: "http://localhost:8888",
+	}
+
+	if viper.GetString("fs.filer.proxy") != "" {
+		ctx.endpoint = viper.GetString("fs.filer.proxy")
+	}
+
+	_filerCtx = ctx
+
+	return ctx
+}

core/controllers/filter.go

@@ -0,0 +1,100 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var FilterController ActionController
+
+func getFilterActions() []Action {
+	ctx := newFilterContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value/:label",
+			HandlerFunc: ctx.getColFieldOptions,
+		},
+	}
+}
+
+type filterContext struct {
+}
+
+func (ctx *filterContext) getColFieldOptions(c *gin.Context) {
+	colName := c.Param("col")
+	value := c.Param("value")
+	if value == "" {
+		value = "_id"
+	}
+	label := c.Param("label")
+	if label == "" {
+		label = "name"
+	}
+	query := MustGetFilterQuery(c)
+	pipelines := mongo2.Pipeline{}
+	if query != nil {
+		pipelines = append(pipelines, bson.D{{"$match", query}})
+	}
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$group",
+				bson.M{
+					"_id": bson.M{
+						"value": "$" + value,
+						"label": "$" + label,
+					},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$project",
+				bson.M{
+					"value": "$_id.value",
+					"label": bson.M{"$toString": "$_id.label"},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$sort",
+				bson.D{
+					{"value", 1},
+				},
+			},
+		},
+	)
+	var options []entity.FilterSelectOption
+	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, options)
+}
+
+func newFilterContext() *filterContext {
+	return &filterContext{}
+}

core/controllers/filter_v2.go

@@ -0,0 +1,69 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/mongo"
+	"github.com/crawlab-team/crawlab/core/entity"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetFilterColFieldOptions(c *gin.Context) {
+	colName := c.Param("col")
+	value := c.Param("value")
+	if value == "" {
+		value = "_id"
+	}
+	label := c.Param("label")
+	if label == "" {
+		label = "name"
+	}
+	query := MustGetFilterQuery(c)
+	pipelines := mongo2.Pipeline{}
+	if query != nil {
+		pipelines = append(pipelines, bson.D{{"$match", query}})
+	}
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$group",
+				bson.M{
+					"_id": bson.M{
+						"value": "$" + value,
+						"label": "$" + label,
+					},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$project",
+				bson.M{
+					"value": "$_id.value",
+					"label": bson.M{"$toString": "$_id.label"},
+				},
+			},
+		},
+	)
+	pipelines = append(
+		pipelines,
+		bson.D{
+			{
+				"$sort",
+				bson.D{
+					{"value", 1},
+				},
+			},
+		},
+	)
+	var options []entity.FilterSelectOption
+	if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, options)
+}

core/controllers/git.go

@@ -0,0 +1,3 @@
+package controllers
+
+var GitController ListController

core/controllers/http.go

@@ -0,0 +1,16 @@
+package controllers
+
+type Response[T any] struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	Data    T      `json:"data"`
+	Error   string `json:"error"`
+}
+
+type ListResponse[T any] struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	Total   int    `json:"total"`
+	Data    []T    `json:"data"`
+	Error   string `json:"error"`
+}

core/controllers/init.go

@@ -0,0 +1,42 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/service"
+)
+
+func InitControllers() (err error) {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		return err
+	}
+
+	NodeController = newNodeController()
+	ProjectController = newProjectController()
+	SpiderController = newSpiderController()
+	TaskController = newTaskController()
+	UserController = newUserController()
+	TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag))
+	SettingController = newSettingController()
+	LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions())
+	DataCollectionController = newDataCollectionController()
+	ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions())
+	ScheduleController = newScheduleController()
+	StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions())
+	TokenController = newTokenController()
+	FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions())
+	GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit))
+	VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions())
+	SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions())
+	DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions())
+	RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole))
+	PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission))
+	ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions())
+	NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions())
+	FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions())
+	SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions())
+	DataSourceController = newDataSourceController()
+	EnvironmentController = newEnvironmentController()
+
+	return nil
+}

core/controllers/login.go

@@ -0,0 +1,64 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+var LoginController ActionController
+
+func getLoginActions() []Action {
+	loginCtx := newLoginContext()
+	return []Action{
+		{Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login},
+		{Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout},
+	}
+}
+
+type loginContext struct {
+	userSvc interfaces.UserService
+}
+
+func (ctx *loginContext) login(c *gin.Context) {
+	var u models.User
+	if err := c.ShouldBindJSON(&u); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{
+		Username: u.Username,
+		Password: u.Password,
+	})
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	c.Set(constants.UserContextKey, loggedInUser)
+	HandleSuccessWithData(c, token)
+}
+
+func (ctx *loginContext) logout(c *gin.Context) {
+	c.Set(constants.UserContextKey, nil)
+	HandleSuccess(c)
+}
+
+func newLoginContext() *loginContext {
+	// context
+	ctx := &loginContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		userSvc interfaces.UserService,
+	) {
+		ctx.userSvc = userSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}

core/controllers/login_v2.go

@@ -0,0 +1,36 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/gin-gonic/gin"
+)
+
+func PostLogin(c *gin.Context) {
+	var payload struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}
+	if err := c.ShouldBindJSON(&payload); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	userSvc, err := user.GetUserServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	token, loggedInUser, err := userSvc.Login(payload.Username, payload.Password)
+	if err != nil {
+		HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized)
+		return
+	}
+	c.Set(constants.UserContextKey, loggedInUser)
+	HandleSuccessWithData(c, token)
+}
+
+func PostLogout(c *gin.Context) {
+	c.Set(constants.UserContextKey, nil)
+	HandleSuccess(c)
+}

core/controllers/node.go

@@ -0,0 +1,94 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/constants"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/go-trace"
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var NodeController *nodeController
+
+type nodeController struct {
+	ListControllerDelegate
+}
+
+func (ctr *nodeController) Post(c *gin.Context) {
+	var n models.Node
+	if err := c.ShouldBindJSON(&n); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	if err := ctr._post(c, &n); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func (ctr *nodeController) PostList(c *gin.Context) {
+	// bind
+	var docs []models.Node
+	if err := c.ShouldBindJSON(&docs); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// success ids
+	var ids []primitive.ObjectID
+
+	// iterate nodes
+	for _, n := range docs {
+		if err := ctr._post(c, &n); err != nil {
+			trace.PrintError(err)
+			continue
+		}
+		ids = append(ids, n.Id)
+	}
+
+	// success
+	HandleSuccessWithData(c, docs)
+}
+
+func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) {
+	// set default key
+	if n.Key == "" {
+		id, err := uuid.NewUUID()
+		if err != nil {
+			return trace.TraceError(err)
+		}
+		n.Key = id.String()
+	}
+
+	// set default status
+	if n.Status == "" {
+		n.Status = constants.NodeStatusUnregistered
+	}
+
+	// add
+	if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil {
+		return trace.TraceError(err)
+	}
+
+	return nil
+}
+
+func newNodeController() *nodeController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode))
+
+	return &nodeController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/notification.go

@@ -0,0 +1,158 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/notification"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+)
+
+var NotificationController ActionController
+
+func getNotificationActions() []Action {
+	ctx := newNotificationContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/settings",
+			HandlerFunc: ctx.GetSettingList,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.GetSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings",
+			HandlerFunc: ctx.PostSetting,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.PutSetting,
+		},
+		{
+			Method:      http.MethodDelete,
+			Path:        "/settings/:id",
+			HandlerFunc: ctx.DeleteSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings/:id/enable",
+			HandlerFunc: ctx.EnableSetting,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/settings/:id/disable",
+			HandlerFunc: ctx.DisableSetting,
+		},
+	}
+}
+
+type notificationContext struct {
+	svc *notification.Service
+}
+
+func (ctx *notificationContext) GetSettingList(c *gin.Context) {
+	query := MustGetFilterQuery(c)
+	pagination := MustGetPagination(c)
+	sort := MustGetSortOption(c)
+	res, total, err := ctx.svc.GetSettingList(query, pagination, sort)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithListData(c, res, total)
+}
+
+func (ctx *notificationContext) GetSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	res, err := ctx.svc.GetSetting(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccessWithData(c, res)
+}
+
+func (ctx *notificationContext) PostSetting(c *gin.Context) {
+	var s notification.NotificationSetting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.PosSetting(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) PutSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s notification.NotificationSetting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.PutSetting(id, s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) DeleteSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.DeleteSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) EnableSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.EnableSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *notificationContext) DisableSetting(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := ctx.svc.DisableSetting(id); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func newNotificationContext() *notificationContext {
+	ctx := &notificationContext{
+		svc: notification.GetService(),
+	}
+	return ctx
+}

core/controllers/permission.go

@@ -0,0 +1,3 @@
+package controllers
+
+var PermissionController ListController

core/controllers/project.go

@@ -0,0 +1,103 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+var ProjectController *projectController
+
+type projectController struct {
+	ListControllerDelegate
+}
+
+func (ctr *projectController) GetList(c *gin.Context) {
+	// get all if query field "all" is set true
+	all := MustGetFilterAll(c)
+	if all {
+		ctr.getAll(c)
+		return
+	}
+
+	// get list
+	list, total, err := ctr.getList(c)
+	if err != nil {
+		return
+	}
+	data := list.GetModels()
+
+	// check empty list
+	if len(list.GetModels()) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// project ids
+	var ids []primitive.ObjectID
+
+	// count cache
+	cache := map[primitive.ObjectID]int{}
+
+	// iterate
+	for _, d := range data {
+		p, ok := d.(*models.Project)
+		if !ok {
+			HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType)
+			return
+		}
+		ids = append(ids, p.Id)
+		cache[p.Id] = 0
+	}
+
+	// spiders
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	spiders, err := modelSvc.GetSpiderList(bson.M{
+		"project_id": bson.M{
+			"$in": ids,
+		},
+	}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	for _, s := range spiders {
+		_, ok := cache[s.ProjectId]
+		if !ok {
+			HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache)
+			return
+		}
+		cache[s.ProjectId]++
+	}
+
+	// assign
+	var projects []models.Project
+	for _, d := range data {
+		p := d.(*models.Project)
+		p.Spiders = cache[p.Id]
+		projects = append(projects, *p)
+	}
+
+	HandleSuccessWithListData(c, projects, total)
+}
+
+func newProjectController() *projectController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject))
+
+	return &projectController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/result.go

@@ -0,0 +1,150 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+	"net/http"
+)
+
+var ResultController ActionController
+
+func getResultActions() []Action {
+	var resultCtx = newResultContext()
+	return []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: resultCtx.getList,
+		},
+	}
+}
+
+type resultContext struct {
+	modelSvc service.ModelService
+}
+
+func (ctx *resultContext) getList(c *gin.Context) {
+	// data collection id
+	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// data source id
+	var dsId primitive.ObjectID
+	dsIdStr := c.Query("data_source_id")
+	if dsIdStr != "" {
+		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+	}
+
+	// data collection
+	dc, err := ctx.modelSvc.GetDataCollectionById(dcId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// data source
+	ds, err := ctx.modelSvc.GetDataSourceById(dsId)
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			ds = &models.DataSource{}
+		} else {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	// spider
+	sq := bson.M{
+		"col_id":         dc.Id,
+		"data_source_id": ds.Id,
+	}
+	s, err := ctx.modelSvc.GetSpider(sq, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// service
+	svc, err := result.GetResultService(s.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := ctx.getListQuery(c)
+
+	// get results
+	data, err := svc.List(query, &generic.ListOptions{
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate results
+	if len(data) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// total count
+	total, err := svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}
+
+func newResultContext() *resultContext {
+	// context
+	ctx := &resultContext{}
+
+	var err error
+	ctx.modelSvc, err = service.NewService()
+	if err != nil {
+		panic(err)
+	}
+
+	return ctx
+}

core/controllers/result_v2.go

@@ -0,0 +1,119 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab-db/generic"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/result"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	mongo2 "go.mongodb.org/mongo-driver/mongo"
+)
+
+func GetResultList(c *gin.Context) {
+	// data collection id
+	dcId, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	// data source id
+	var dsId primitive.ObjectID
+	dsIdStr := c.Query("data_source_id")
+	if dsIdStr != "" {
+		dsId, err = primitive.ObjectIDFromHex(dsIdStr)
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+	}
+
+	// data collection
+	dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// data source
+	ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId)
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			ds = &models.DataSourceV2{}
+		} else {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	// spider
+	sq := bson.M{
+		"col_id":         dc.Id,
+		"data_source_id": ds.Id,
+	}
+	s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// service
+	svc, err := result.GetResultService(s.Id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// params
+	pagination := MustGetPagination(c)
+	query := getResultListQuery(c)
+
+	// get results
+	data, err := svc.List(query, &generic.ListOptions{
+		Sort:  []generic.ListSort{{"_id", generic.SortDirectionDesc}},
+		Skip:  pagination.Size * (pagination.Page - 1),
+		Limit: pagination.Size,
+	})
+	if err != nil {
+		if err.Error() == mongo2.ErrNoDocuments.Error() {
+			HandleSuccessWithListData(c, nil, 0)
+			return
+		}
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// validate results
+	if len(data) == 0 {
+		HandleSuccessWithListData(c, nil, 0)
+		return
+	}
+
+	// total count
+	total, err := svc.Count(query)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// response
+	HandleSuccessWithListData(c, data, total)
+}
+
+func getResultListQuery(c *gin.Context) (q generic.ListQuery) {
+	f, err := GetFilter(c)
+	if err != nil {
+		return q
+	}
+	for _, cond := range f.Conditions {
+		q = append(q, generic.ListQueryCondition{
+			Key:   cond.Key,
+			Op:    cond.Op,
+			Value: utils.NormalizeObjectId(cond.Value),
+		})
+	}
+	return q
+}

core/controllers/role.go

@@ -0,0 +1,3 @@
+package controllers
+
+var RoleController ListController

core/controllers/router_v2.go

@@ -0,0 +1,387 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+type RouterGroups struct {
+	AuthGroup      *gin.RouterGroup
+	AnonymousGroup *gin.RouterGroup
+}
+
+func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) {
+	return &RouterGroups{
+		AuthGroup:      app.Group("/", middlewares.AuthorizationMiddlewareV2()),
+		AnonymousGroup: app.Group("/"),
+	}
+}
+
+func RegisterController[T any](group *gin.RouterGroup, basePath string, ctr *BaseControllerV2[T]) {
+	actionPaths := make(map[string]bool)
+	for _, action := range ctr.actions {
+		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
+		path := basePath + action.Path
+		key := action.Method + " - " + path
+		actionPaths[key] = true
+	}
+	registerBuiltinHandler(group, http.MethodGet, basePath+"", ctr.GetList, actionPaths)
+	registerBuiltinHandler(group, http.MethodGet, basePath+"/:id", ctr.GetById, actionPaths)
+	registerBuiltinHandler(group, http.MethodPost, basePath+"", ctr.Post, actionPaths)
+	registerBuiltinHandler(group, http.MethodPut, basePath+"/:id", ctr.PutById, actionPaths)
+	registerBuiltinHandler(group, http.MethodPatch, basePath+"", ctr.PatchList, actionPaths)
+	registerBuiltinHandler(group, http.MethodDelete, basePath+"/:id", ctr.DeleteById, actionPaths)
+	registerBuiltinHandler(group, http.MethodDelete, basePath+"", ctr.DeleteList, actionPaths)
+}
+
+func RegisterActions(group *gin.RouterGroup, basePath string, actions []Action) {
+	for _, action := range actions {
+		group.Handle(action.Method, basePath+action.Path, action.HandlerFunc)
+	}
+}
+
+func registerBuiltinHandler(group *gin.RouterGroup, method, path string, handlerFunc gin.HandlerFunc, existingActionPaths map[string]bool) {
+	key := method + " - " + path
+	_, ok := existingActionPaths[key]
+	if ok {
+		return
+	}
+	group.Handle(method, path, handlerFunc)
+}
+
+func InitRoutes(app *gin.Engine) (err error) {
+	// routes groups
+	groups := NewRouterGroups(app)
+
+	RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
+	RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]())
+	RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
+	RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]())
+	RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
+	RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]())
+	RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]())
+	RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]())
+	RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]())
+	RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostSchedule,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutScheduleById,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/enable",
+			HandlerFunc: PostScheduleEnable,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/disable",
+			HandlerFunc: PostScheduleDisable,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2](
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetSpiderById,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetSpiderList,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostSpider,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutSpiderById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id",
+			HandlerFunc: DeleteSpiderById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "",
+			HandlerFunc: DeleteSpiderList,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/list",
+			HandlerFunc: GetSpiderListDir,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/get",
+			HandlerFunc: GetSpiderFile,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/files/info",
+			HandlerFunc: GetSpiderFileInfo,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save",
+			HandlerFunc: PostSpiderSaveFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save/batch",
+			HandlerFunc: PostSpiderSaveFiles,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/save/dir",
+			HandlerFunc: PostSpiderSaveDir,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/rename",
+			HandlerFunc: PostSpiderRenameFile,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id/files",
+			HandlerFunc: DeleteSpiderFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/copy",
+			HandlerFunc: PostSpiderCopyFile,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/files/export",
+			HandlerFunc: PostSpiderExport,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/run",
+			HandlerFunc: PostSpiderRun,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/git",
+			HandlerFunc: GetSpiderGit,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/git/remote-refs",
+			HandlerFunc: GetSpiderGitRemoteRefs,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/checkout",
+			HandlerFunc: PostSpiderGitCheckout,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/pull",
+			HandlerFunc: PostSpiderGitPull,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/git/commit",
+			HandlerFunc: PostSpiderGitCommit,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/data-source",
+			HandlerFunc: GetSpiderDataSource,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/data-source/:ds_id",
+			HandlerFunc: PostSpiderDataSource,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2](
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetTaskById,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetTaskList,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "/:id",
+			HandlerFunc: DeleteTaskById,
+		},
+		Action{
+			Method:      http.MethodDelete,
+			Path:        "",
+			HandlerFunc: DeleteList,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/run",
+			HandlerFunc: PostTaskRun,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/restart",
+			HandlerFunc: PostTaskRestart,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/cancel",
+			HandlerFunc: PostTaskCancel,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/logs",
+			HandlerFunc: GetTaskLogs,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/:id/data",
+			HandlerFunc: GetTaskData,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostToken,
+		},
+	))
+	RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2](
+		Action{
+			Method:      http.MethodPost,
+			Path:        "",
+			HandlerFunc: PostUser,
+		},
+		Action{
+			Method:      http.MethodPost,
+			Path:        "/:id/change-password",
+			HandlerFunc: PostUserChangePassword,
+		},
+		Action{
+			Method:      http.MethodGet,
+			Path:        "/me",
+			HandlerFunc: GetUserMe,
+		},
+		Action{
+			Method:      http.MethodPut,
+			Path:        "/me",
+			HandlerFunc: PutUserById,
+		},
+	))
+
+	RegisterActions(groups.AuthGroup, "/results", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetResultList,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/export", []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:type",
+			HandlerFunc: PostExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id",
+			HandlerFunc: GetExport,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:type/:id/download",
+			HandlerFunc: GetExportDownload,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/filters", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/:col/:value/:label",
+			HandlerFunc: GetFilterColFieldOptions,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/settings", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/:id",
+			HandlerFunc: GetSetting,
+		},
+		{
+			Method:      http.MethodPut,
+			Path:        "/:id",
+			HandlerFunc: PutSetting,
+		},
+	})
+	RegisterActions(groups.AuthGroup, "/stats", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "/overview",
+			HandlerFunc: GetStatsOverview,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/daily",
+			HandlerFunc: GetStatsDaily,
+		},
+		{
+			Method:      http.MethodGet,
+			Path:        "/tasks",
+			HandlerFunc: GetStatsTasks,
+		},
+	})
+
+	RegisterActions(groups.AnonymousGroup, "/system-info", []Action{
+		{
+			Path:        "",
+			Method:      http.MethodGet,
+			HandlerFunc: GetSystemInfo,
+		},
+	})
+	RegisterActions(groups.AnonymousGroup, "/version", []Action{
+		{
+			Method:      http.MethodGet,
+			Path:        "",
+			HandlerFunc: GetVersion,
+		},
+	})
+	RegisterActions(groups.AnonymousGroup, "/", []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/login",
+			HandlerFunc: PostLogin,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/logout",
+			HandlerFunc: PostLogout,
+		},
+	})
+
+	return nil
+}

core/controllers/router_v2_test.go

@@ -0,0 +1,91 @@
+package controllers_test
+
+import (
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestRouterGroups(t *testing.T) {
+	router := gin.Default()
+	groups := controllers.NewRouterGroups(router)
+
+	assertions := []struct {
+		group *gin.RouterGroup
+		name  string
+	}{
+		{groups.AuthGroup, "AuthGroup"},
+		{groups.AnonymousGroup, "AnonymousGroup"},
+	}
+
+	for _, a := range assertions {
+		assert.NotNil(t, a.group, a.name+" should not be nil")
+	}
+}
+
+func TestRegisterController_Routes(t *testing.T) {
+	router := gin.Default()
+	groups := controllers.NewRouterGroups(router)
+	ctr := controllers.NewControllerV2[models.TestModel]()
+	basePath := "/testmodels"
+
+	controllers.RegisterController(groups.AuthGroup, basePath, ctr)
+
+	// Check if all routes are registered
+	routes := router.Routes()
+
+	var methodPaths []string
+	for _, route := range routes {
+		methodPaths = append(methodPaths, route.Method+" - "+route.Path)
+	}
+
+	expectedRoutes := []gin.RouteInfo{
+		{Method: "GET", Path: basePath},
+		{Method: "GET", Path: basePath + "/:id"},
+		{Method: "POST", Path: basePath},
+		{Method: "PUT", Path: basePath + "/:id"},
+		{Method: "PATCH", Path: basePath},
+		{Method: "DELETE", Path: basePath + "/:id"},
+		{Method: "DELETE", Path: basePath},
+	}
+
+	assert.Equal(t, len(expectedRoutes), len(routes))
+	for _, route := range expectedRoutes {
+		assert.Contains(t, methodPaths, route.Method+" - "+route.Path)
+	}
+}
+
+func TestInitRoutes_ProjectsRoute(t *testing.T) {
+	router := gin.Default()
+
+	controllers.InitRoutes(router)
+
+	// Check if the projects route is registered
+	routes := router.Routes()
+
+	var methodPaths []string
+	for _, route := range routes {
+		methodPaths = append(methodPaths, route.Method+" - "+route.Path)
+	}
+
+	expectedRoutes := []gin.RouteInfo{
+		{Method: "GET", Path: "/projects"},
+		{Method: "GET", Path: "/projects/:id"},
+		{Method: "POST", Path: "/projects"},
+		{Method: "PUT", Path: "/projects/:id"},
+		{Method: "PATCH", Path: "/projects"},
+		{Method: "DELETE", Path: "/projects/:id"},
+		{Method: "DELETE", Path: "/projects"},
+	}
+
+	for _, route := range expectedRoutes {
+		assert.Contains(t, methodPaths, route.Method+" - "+route.Path)
+	}
+}
+
+func TestMain(m *testing.M) {
+	gin.SetMode(gin.TestMode)
+	m.Run()
+}

core/controllers/schedule.go

@@ -0,0 +1,221 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/container"
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"net/http"
+)
+
+var ScheduleController *scheduleController
+
+func getScheduleActions() []Action {
+	scheduleCtx := newScheduleContext()
+	return []Action{
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/enable",
+			HandlerFunc: scheduleCtx.enable,
+		},
+		{
+			Method:      http.MethodPost,
+			Path:        "/:id/disable",
+			HandlerFunc: scheduleCtx.disable,
+		},
+	}
+}
+
+type scheduleController struct {
+	ListActionControllerDelegate
+	d   ListActionControllerDelegate
+	ctx *scheduleContext
+}
+
+func (ctr *scheduleController) Post(c *gin.Context) {
+	var s models.Schedule
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if s.Enabled {
+		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *scheduleController) Put(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s models.Schedule
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if s.GetId() != oid {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+	if err := delegate.NewModelDelegate(&s).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if s.Enabled {
+		if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *scheduleController) Delete(c *gin.Context) {
+	id := c.Param("id")
+	oid, err := primitive.ObjectIDFromHex(id)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	s, err := ctr.ctx.modelSvc.GetScheduleById(oid)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+}
+
+func (ctr *scheduleController) DeleteList(c *gin.Context) {
+	payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c)
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	for _, id := range payload.Ids {
+		s, err := ctr.ctx.modelSvc.GetScheduleById(id)
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := ctr.ctx.scheduleSvc.Disable(s); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+	if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{
+		"_id": bson.M{
+			"$in": payload.Ids,
+		},
+	}); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) enable(c *gin.Context) {
+	s, err := ctx._getSchedule(c)
+	if err != nil {
+		return
+	}
+	if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) disable(c *gin.Context) {
+	s, err := ctx._getSchedule(c)
+	if err != nil {
+		return
+	}
+	if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	HandleSuccess(c)
+}
+
+func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	s, err = ctx.modelSvc.GetScheduleById(id)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	return s, nil
+}
+
+type scheduleContext struct {
+	modelSvc    service.ModelService
+	scheduleSvc interfaces.ScheduleService
+}
+
+func newScheduleContext() *scheduleContext {
+	// context
+	ctx := &scheduleContext{}
+
+	// dependency injection
+	if err := container.GetContainer().Invoke(func(
+		modelSvc service.ModelService,
+		scheduleSvc interfaces.ScheduleService,
+	) {
+		ctx.modelSvc = modelSvc
+		ctx.scheduleSvc = scheduleSvc
+	}); err != nil {
+		panic(err)
+	}
+
+	return ctx
+}
+
+func newScheduleController() *scheduleController {
+	actions := getScheduleActions()
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
+	d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions)
+	ctx := newScheduleContext()
+
+	return &scheduleController{
+		ListActionControllerDelegate: *ctr,
+		d:                            *d,
+		ctx:                          ctx,
+	}
+}

core/controllers/schedule_v2.go

@@ -0,0 +1,130 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/errors"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/crawlab-team/crawlab/core/schedule"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+)
+
+func PostSchedule(c *gin.Context) {
+	var s models.ScheduleV2
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	modelSvc := service.NewModelServiceV2[models.ScheduleV2]()
+
+	s.SetCreated(u.Id)
+	s.SetUpdated(u.Id)
+	id, err := modelSvc.InsertOne(s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+	s.Id = id
+
+	if s.Enabled {
+		scheduleSvc, err := schedule.GetScheduleServiceV2()
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		if err := scheduleSvc.Enable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PutScheduleById(c *gin.Context) {
+	id, err := primitive.ObjectIDFromHex(c.Param("id"))
+	if err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	var s models.ScheduleV2
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorBadRequest(c, err)
+		return
+	}
+	if s.Id != id {
+		HandleErrorBadRequest(c, errors.ErrorHttpBadRequest)
+		return
+	}
+
+	modelSvc := service.NewModelServiceV2[models.ScheduleV2]()
+	err = modelSvc.ReplaceById(id, s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	scheduleSvc, err := schedule.GetScheduleServiceV2()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	if s.Enabled {
+		if err := scheduleSvc.Enable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	} else {
+		if err := scheduleSvc.Disable(s, u.Id); err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PostScheduleEnable(c *gin.Context) {
+	postScheduleEnableDisableFunc(true)(c)
+}
+
+func PostScheduleDisable(c *gin.Context) {
+	postScheduleEnableDisableFunc(false)(c)
+}
+
+func postScheduleEnableDisableFunc(isEnable bool) func(c *gin.Context) {
+	return func(c *gin.Context) {
+		id, err := primitive.ObjectIDFromHex(c.Param("id"))
+		if err != nil {
+			HandleErrorBadRequest(c, err)
+			return
+		}
+		svc, err := schedule.GetScheduleServiceV2()
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		s, err := service.NewModelServiceV2[models.ScheduleV2]().GetById(id)
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		u := GetUserFromContextV2(c)
+		if isEnable {
+			err = svc.Enable(*s, u.Id)
+		} else {
+			err = svc.Disable(*s, u.Id)
+		}
+		if err != nil {
+			HandleErrorInternalServerError(c, err)
+			return
+		}
+		HandleSuccess(c)
+	}
+}

core/controllers/setting.go

@@ -0,0 +1,84 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/models/delegate"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+)
+
+var SettingController *settingController
+
+type settingController struct {
+	ListControllerDelegate
+}
+
+func (ctr *settingController) Get(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// model service
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// setting
+	s, err := modelSvc.GetSettingByKey(key, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func (ctr *settingController) Put(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// settings
+	var s models.Setting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// model service
+	modelSvc, err := service.NewService()
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// setting
+	_s, err := modelSvc.GetSettingByKey(key, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	// save
+	_s.Value = s.Value
+	if err := delegate.NewModelDelegate(_s).Save(); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}
+
+func newSettingController() *settingController {
+	modelSvc, err := service.GetService()
+	if err != nil {
+		panic(err)
+	}
+
+	ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting))
+
+	return &settingController{
+		ListControllerDelegate: *ctr,
+	}
+}

core/controllers/setting_v2.go

@@ -0,0 +1,56 @@
+package controllers
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/crawlab-team/crawlab/core/models/service"
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func GetSetting(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// setting
+	s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccessWithData(c, s)
+}
+
+func PutSetting(c *gin.Context) {
+	// key
+	key := c.Param("id")
+
+	// settings
+	var s models.Setting
+	if err := c.ShouldBindJSON(&s); err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	modelSvc := service.NewModelServiceV2[models.SettingV2]()
+
+	// setting
+	_s, err := modelSvc.GetOne(bson.M{"key": key}, nil)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	u := GetUserFromContextV2(c)
+
+	// save
+	_s.Value = s.Value
+	_s.SetUpdated(u.Id)
+	err = modelSvc.ReplaceOne(bson.M{"key": key}, *_s)
+	if err != nil {
+		HandleErrorInternalServerError(c, err)
+		return
+	}
+
+	HandleSuccess(c)
+}