feat: added modules

core/apps/api.go

@@ -0,0 +1,126 @@
+package apps
+
+import (
+	"context"
+	"errors"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/crawlab-team/crawlab/core/routes"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net"
+	"net/http"
+	"time"
+)
+
+func init() {
+	// set gin mode
+	if viper.GetString("gin.mode") == "" {
+		gin.SetMode(gin.ReleaseMode)
+	} else {
+		gin.SetMode(viper.GetString("gin.mode"))
+	}
+}
+
+type Api struct {
+	// dependencies
+	interfaces.WithConfigPath
+
+	// internals
+	app   *gin.Engine
+	ln    net.Listener
+	srv   *http.Server
+	ready bool
+}
+
+func (app *Api) Init() {
+	// initialize controllers
+	_ = initModule("controllers", controllers.InitControllers)
+
+	// initialize middlewares
+	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
+
+	// initialize routes
+	_ = app.initModuleWithApp("routes", routes.InitRoutes)
+}
+
+func (app *Api) Start() {
+	// address
+	host := viper.GetString("server.host")
+	port := viper.GetString("server.port")
+	address := net.JoinHostPort(host, port)
+
+	// http server
+	app.srv = &http.Server{
+		Handler: app.app,
+		Addr:    address,
+	}
+
+	// listen
+	var err error
+	app.ln, err = net.Listen("tcp", address)
+	if err != nil {
+		panic(err)
+	}
+	app.ready = true
+
+	// serve
+	if err := http.Serve(app.ln, app.app); err != nil {
+		if !errors.Is(err, http.ErrServerClosed) {
+			log.Error("run server error:" + err.Error())
+		} else {
+			log.Info("server graceful down")
+		}
+	}
+}
+
+func (app *Api) Wait() {
+	DefaultWait()
+}
+
+func (app *Api) Stop() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := app.srv.Shutdown(ctx); err != nil {
+		log.Error("run server error:" + err.Error())
+	}
+}
+
+func (app *Api) GetGinEngine() *gin.Engine {
+	return app.app
+}
+
+func (app *Api) GetHttpServer() *http.Server {
+	return app.srv
+}
+
+func (app *Api) Ready() (ok bool) {
+	return app.ready
+}
+
+func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
+	return initModule(name, func() error {
+		return fn(app.app)
+	})
+}
+
+func NewApi() *Api {
+	api := &Api{
+		app: gin.New(),
+	}
+	api.Init()
+	return api
+}
+
+var api *Api
+
+func GetApi() *Api {
+	if api != nil {
+		return api
+	}
+	api = NewApi()
+	return api
+}

core/apps/api_v2.go

@@ -0,0 +1,122 @@
+package apps
+
+import (
+	"context"
+	"errors"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/middlewares"
+	"github.com/gin-gonic/gin"
+	"github.com/spf13/viper"
+	"net"
+	"net/http"
+	"time"
+)
+
+func init() {
+	// set gin mode
+	if viper.GetString("gin.mode") == "" {
+		gin.SetMode(gin.ReleaseMode)
+	} else {
+		gin.SetMode(viper.GetString("gin.mode"))
+	}
+}
+
+type ApiV2 struct {
+	// dependencies
+	interfaces.WithConfigPath
+
+	// internals
+	app   *gin.Engine
+	ln    net.Listener
+	srv   *http.Server
+	ready bool
+}
+
+func (app *ApiV2) Init() {
+	// initialize middlewares
+	_ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares)
+
+	// initialize routes
+	_ = app.initModuleWithApp("routes", controllers.InitRoutes)
+}
+
+func (app *ApiV2) Start() {
+	// address
+	host := viper.GetString("server.host")
+	port := viper.GetString("server.port")
+	address := net.JoinHostPort(host, port)
+
+	// http server
+	app.srv = &http.Server{
+		Handler: app.app,
+		Addr:    address,
+	}
+
+	// listen
+	var err error
+	app.ln, err = net.Listen("tcp", address)
+	if err != nil {
+		panic(err)
+	}
+	app.ready = true
+
+	// serve
+	if err := http.Serve(app.ln, app.app); err != nil {
+		if !errors.Is(err, http.ErrServerClosed) {
+			log.Error("run server error:" + err.Error())
+		} else {
+			log.Info("server graceful down")
+		}
+	}
+}
+
+func (app *ApiV2) Wait() {
+	DefaultWait()
+}
+
+func (app *ApiV2) Stop() {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if err := app.srv.Shutdown(ctx); err != nil {
+		log.Error("run server error:" + err.Error())
+	}
+}
+
+func (app *ApiV2) GetGinEngine() *gin.Engine {
+	return app.app
+}
+
+func (app *ApiV2) GetHttpServer() *http.Server {
+	return app.srv
+}
+
+func (app *ApiV2) Ready() (ok bool) {
+	return app.ready
+}
+
+func (app *ApiV2) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) {
+	return initModule(name, func() error {
+		return fn(app.app)
+	})
+}
+
+func NewApiV2() *ApiV2 {
+	api := &ApiV2{
+		app: gin.New(),
+	}
+	api.Init()
+	return api
+}
+
+var apiV2 *ApiV2
+
+func GetApiV2() *ApiV2 {
+	if apiV2 != nil {
+		return apiV2
+	}
+	apiV2 = NewApiV2()
+	return apiV2
+}

core/apps/docker.go

@@ -0,0 +1,199 @@
+package apps
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/sys_exec"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+type Docker struct {
+	// parent
+	parent ServerApp
+
+	// dependencies
+	interfaces.WithConfigPath
+
+	// seaweedfs log
+	fsLogFilePath string
+	fsLogFile     *os.File
+	fsReady       bool
+}
+
+func (app *Docker) Init() {
+	var err error
+	app.fsLogFile, err = os.OpenFile(app.fsLogFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.FileMode(0777))
+	if err != nil {
+		trace.PrintError(err)
+	}
+
+	// replace paths
+	if err := app.replacePaths(); err != nil {
+		panic(err)
+	}
+
+	// start nginx
+	go app.startNginx()
+
+	// start seaweedfs
+	go app.startSeaweedFs()
+}
+
+func (app *Docker) Start() {
+	// import demo
+	//if utils.IsDemo() && utils.InitializedDemo() {
+	//	go app.importDemo()
+	//}
+}
+
+func (app *Docker) Wait() {
+	DefaultWait()
+}
+
+func (app *Docker) Stop() {
+}
+
+func (app *Docker) GetParent() (parent ServerApp) {
+	return app.parent
+}
+
+func (app *Docker) SetParent(parent ServerApp) {
+	app.parent = parent
+}
+
+func (app *Docker) Ready() (ok bool) {
+	return app.fsReady &&
+		app.parent.GetApi().Ready()
+}
+
+func (app *Docker) replacePaths() (err error) {
+	// read
+	indexHtmlPath := "/app/dist/index.html"
+	indexHtmlBytes, err := os.ReadFile(indexHtmlPath)
+	if err != nil {
+		return trace.TraceError(err)
+	}
+	indexHtml := string(indexHtmlBytes)
+
+	// replace paths
+	baseUrl := viper.GetString("base.url")
+	if baseUrl != "" {
+		indexHtml = app._replacePath(indexHtml, "js", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "css", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "<link rel=\"stylesheet\" href=\"", baseUrl)
+		indexHtml = app._replacePath(indexHtml, "window.VUE_APP_API_BASE_URL = '", baseUrl)
+	}
+
+	// replace path of baidu tongji
+	initBaiduTongji := viper.GetString("string")
+	if initBaiduTongji != "" {
+		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_BAIDU_TONGJI = ''", fmt.Sprintf("window.VUE_APP_INIT_BAIDU_TONGJI = '%s'", initBaiduTongji))
+	}
+
+	// replace path of umeng
+	initUmeng := viper.GetString("string")
+	if initUmeng != "" {
+		indexHtml = strings.ReplaceAll(indexHtml, "window.VUE_APP_INIT_UMENG = ''", fmt.Sprintf("window.VUE_APP_INIT_UMENG = '%s'", initUmeng))
+	}
+
+	// write
+	if err := os.WriteFile(indexHtmlPath, []byte(indexHtml), os.FileMode(0766)); err != nil {
+		return trace.TraceError(err)
+	}
+
+	return nil
+}
+
+func (app *Docker) _replacePath(text, path, baseUrl string) (res string) {
+	text = strings.ReplaceAll(text, path, fmt.Sprintf("%s/%s", baseUrl, path))
+	return text
+}
+
+func (app *Docker) startNginx() {
+	cmd := exec.Command("service", "nginx", "start")
+	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
+		for scanner.Scan() {
+			line := fmt.Sprintf("[nginx] %s\n", scanner.Text())
+			_, _ = os.Stdout.WriteString(line)
+		}
+	})
+	if err := cmd.Run(); err != nil {
+		trace.PrintError(err)
+	}
+}
+
+func (app *Docker) startSeaweedFs() {
+	seaweedFsDataPath := "/data/seaweedfs"
+	if !utils.Exists(seaweedFsDataPath) {
+		_ = os.MkdirAll(seaweedFsDataPath, os.FileMode(0777))
+	}
+	cmd := exec.Command("weed", "server",
+		"-dir", "/data",
+		"-master.dir", seaweedFsDataPath,
+		"-volume.dir.idx", seaweedFsDataPath,
+		"-ip", "localhost",
+		"-volume.port", "9999",
+		"-volume.minFreeSpace", "1GiB",
+		"-filer",
+	)
+	sys_exec.ConfigureCmdLogging(cmd, func(scanner *bufio.Scanner) {
+		for scanner.Scan() {
+			line := fmt.Sprintf("[seaweedfs] %s\n", scanner.Text())
+			_, _ = app.fsLogFile.WriteString(line)
+		}
+	})
+	go func() {
+		if err := cmd.Run(); err != nil {
+			trace.PrintError(err)
+		}
+	}()
+	for {
+		_, err := req.Get("http://localhost:8888")
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		app.fsReady = true
+		return
+	}
+}
+
+func (app *Docker) importDemo() {
+	for {
+		if app.Ready() {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	_ = utils.ImportDemo()
+}
+
+func NewDocker(svr ServerApp) *Docker {
+	dck := &Docker{
+		parent:        svr,
+		fsLogFilePath: "/var/log/weed.log",
+	}
+
+	dck.Init()
+
+	return dck
+}
+
+var dck *Docker
+
+func GetDocker(svr ServerApp) *Docker {
+	if dck != nil {
+		return dck
+	}
+	dck = NewDocker(svr)
+	return dck
+}

core/apps/interfaces.go

@@ -0,0 +1,39 @@
+package apps
+
+import (
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+type App interface {
+	Init()
+	Start()
+	Wait()
+	Stop()
+}
+
+type ApiApp interface {
+	App
+	GetGinEngine() (engine *gin.Engine)
+	GetHttpServer() (svr *http.Server)
+	Ready() (ok bool)
+}
+
+type NodeApp interface {
+	App
+	interfaces.WithConfigPath
+}
+
+type ServerApp interface {
+	NodeApp
+	GetApi() (api ApiApp)
+	GetNodeService() (masterSvc interfaces.NodeService)
+}
+
+type DockerApp interface {
+	App
+	GetParent() (parent NodeApp)
+	SetParent(parent NodeApp)
+	Ready() (ok bool)
+}

core/apps/server.go

@@ -0,0 +1,149 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/controllers"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/node/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/spf13/viper"
+	"net/http"
+	_ "net/http/pprof"
+)
+
+func init() {
+	injectModules()
+}
+
+type Server struct {
+	// settings
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+	nodeSvc interfaces.NodeService
+
+	// modules
+	api *Api
+	dck *Docker
+
+	// internals
+	quit chan int
+}
+
+func (app *Server) SetGrpcAddress(address interfaces.Address) {
+	app.grpcAddress = address
+}
+
+func (app *Server) GetApi() (api ApiApp) {
+	return app.api
+}
+
+func (app *Server) GetNodeService() (svc interfaces.NodeService) {
+	return app.nodeSvc
+}
+
+func (app *Server) Init() {
+	// log node info
+	app.logNodeInfo()
+
+	if utils.IsMaster() {
+
+		// initialize controllers
+		if err := controllers.InitControllers(); err != nil {
+			panic(err)
+		}
+	}
+
+	// pprof
+	app.initPprof()
+}
+
+func (app *Server) Start() {
+	if utils.IsMaster() {
+		// start docker app
+		if utils.IsDocker() {
+			go app.dck.Start()
+		}
+
+		// start api
+		go app.api.Start()
+	}
+
+	// start node service
+	go app.nodeSvc.Start()
+}
+
+func (app *Server) Wait() {
+	<-app.quit
+}
+
+func (app *Server) Stop() {
+	app.api.Stop()
+	app.quit <- 1
+}
+
+func (app *Server) logNodeInfo() {
+	log.Infof("current node type: %s", utils.GetNodeType())
+	if utils.IsDocker() {
+		log.Infof("running in docker container")
+	}
+}
+
+func (app *Server) initPprof() {
+	if viper.GetBool("pprof") {
+		go func() {
+			fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		}()
+	}
+}
+
+func NewServer() (app NodeApp) {
+	// server
+	svr := &Server{
+		WithConfigPath: config.NewConfigPathService(),
+		quit:           make(chan int, 1),
+	}
+
+	// service options
+	var svcOpts []service.Option
+	if svr.grpcAddress != nil {
+		svcOpts = append(svcOpts, service.WithAddress(svr.grpcAddress))
+	}
+
+	// master modules
+	if utils.IsMaster() {
+		// api
+		svr.api = GetApi()
+
+		// docker
+		if utils.IsDocker() {
+			svr.dck = GetDocker(svr)
+		}
+	}
+
+	// node service
+	var err error
+	if utils.IsMaster() {
+		svr.nodeSvc, err = service.NewMasterService(svcOpts...)
+	} else {
+		svr.nodeSvc, err = service.NewWorkerService(svcOpts...)
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	return svr
+}
+
+var server NodeApp
+
+func GetServer() NodeApp {
+	if server != nil {
+		return server
+	}
+	server = NewServer()
+	return server
+}

core/apps/server_test.go

@@ -0,0 +1,29 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/imroc/req"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+	"os"
+	"testing"
+	"time"
+)
+
+func init() {
+	_ = os.Setenv("CRAWLAB_DEMO", "false")
+}
+
+func TestServer_Start(t *testing.T) {
+	svr := GetServer()
+
+	// start
+	go Start(svr)
+	time.Sleep(1 * time.Second)
+
+	res, err := req.Get(fmt.Sprintf("http://localhost:%s/system-info", viper.GetString("server.port")))
+	require.Nil(t, err)
+	resStr, err := res.ToString()
+	require.Nil(t, err)
+	require.Contains(t, resStr, "success")
+}

core/apps/server_v2.go

@@ -0,0 +1,126 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/interfaces"
+	"github.com/crawlab-team/crawlab/core/node/service"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/spf13/viper"
+	"net/http"
+	_ "net/http/pprof"
+)
+
+type ServerV2 struct {
+	// settings
+	grpcAddress interfaces.Address
+
+	// dependencies
+	interfaces.WithConfigPath
+
+	// modules
+	nodeSvc interfaces.NodeService
+	api     *ApiV2
+	dck     *Docker
+
+	// internals
+	quit chan int
+}
+
+func (app *ServerV2) Init() {
+	// log node info
+	app.logNodeInfo()
+
+	// pprof
+	app.initPprof()
+}
+
+func (app *ServerV2) Start() {
+	if utils.IsMaster() {
+		// start docker app
+		if utils.IsDocker() {
+			go app.dck.Start()
+		}
+
+		// start api
+		go app.api.Start()
+	}
+
+	// start node service
+	go app.nodeSvc.Start()
+}
+
+func (app *ServerV2) Wait() {
+	<-app.quit
+}
+
+func (app *ServerV2) Stop() {
+	app.api.Stop()
+	app.quit <- 1
+}
+
+func (app *ServerV2) GetApi() ApiApp {
+	return app.api
+}
+
+func (app *ServerV2) GetNodeService() interfaces.NodeService {
+	return app.nodeSvc
+}
+
+func (app *ServerV2) logNodeInfo() {
+	log.Infof("current node type: %s", utils.GetNodeType())
+	if utils.IsDocker() {
+		log.Infof("running in docker container")
+	}
+}
+
+func (app *ServerV2) initPprof() {
+	if viper.GetBool("pprof") {
+		go func() {
+			fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil))
+		}()
+	}
+}
+
+func NewServerV2() (app NodeApp) {
+	// server
+	svr := &ServerV2{
+		WithConfigPath: config.NewConfigPathService(),
+		quit:           make(chan int, 1),
+	}
+
+	// master modules
+	if utils.IsMaster() {
+		// api
+		svr.api = GetApiV2()
+
+		// docker
+		if utils.IsDocker() {
+			svr.dck = GetDocker(svr)
+		}
+	}
+
+	// node service
+	var err error
+	if utils.IsMaster() {
+		svr.nodeSvc, err = service.NewMasterServiceV2()
+	} else {
+		svr.nodeSvc, err = service.NewWorkerServiceV2()
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	return svr
+}
+
+var serverV2 NodeApp
+
+func GetServerV2() NodeApp {
+	if serverV2 != nil {
+		return serverV2
+	}
+	serverV2 = NewServerV2()
+	return serverV2
+}

core/apps/utils.go

@@ -0,0 +1,92 @@
+package apps
+
+import (
+	"fmt"
+	"github.com/apex/log"
+	"github.com/crawlab-team/crawlab/core/color"
+	"github.com/crawlab-team/crawlab/core/config"
+	"github.com/crawlab-team/crawlab/core/container"
+	grpcclient "github.com/crawlab-team/crawlab/core/grpc/client"
+	grpcserver "github.com/crawlab-team/crawlab/core/grpc/server"
+	modelsclient "github.com/crawlab-team/crawlab/core/models/client"
+	modelsservice "github.com/crawlab-team/crawlab/core/models/service"
+	nodeconfig "github.com/crawlab-team/crawlab/core/node/config"
+	"github.com/crawlab-team/crawlab/core/schedule"
+	"github.com/crawlab-team/crawlab/core/spider/admin"
+	"github.com/crawlab-team/crawlab/core/stats"
+	"github.com/crawlab-team/crawlab/core/task/handler"
+	"github.com/crawlab-team/crawlab/core/task/scheduler"
+	taskstats "github.com/crawlab-team/crawlab/core/task/stats"
+	"github.com/crawlab-team/crawlab/core/user"
+	"github.com/crawlab-team/crawlab/core/utils"
+	"github.com/crawlab-team/go-trace"
+)
+
+func Start(app App) {
+	start(app)
+}
+
+func start(app App) {
+	app.Init()
+	go app.Start()
+	app.Wait()
+	app.Stop()
+}
+
+func DefaultWait() {
+	utils.DefaultWait()
+}
+
+func initModule(name string, fn func() error) (err error) {
+	if err := fn(); err != nil {
+		log.Error(fmt.Sprintf("init %s error: %s", name, err.Error()))
+		_ = trace.TraceError(err)
+		panic(err)
+	}
+	log.Info(fmt.Sprintf("initialized %s successfully", name))
+	return nil
+}
+
+func initApp(name string, app App) {
+	_ = initModule(name, func() error {
+		app.Init()
+		return nil
+	})
+}
+
+var injectors = []interface{}{
+	modelsservice.GetService,
+	modelsclient.NewServiceDelegate,
+	modelsclient.NewNodeServiceDelegate,
+	modelsclient.NewSpiderServiceDelegate,
+	modelsclient.NewTaskServiceDelegate,
+	modelsclient.NewTaskStatServiceDelegate,
+	modelsclient.NewEnvironmentServiceDelegate,
+	grpcclient.NewClient,
+	grpcclient.NewPool,
+	grpcserver.GetServer,
+	grpcserver.NewModelDelegateServer,
+	grpcserver.NewModelBaseServiceServer,
+	grpcserver.NewNodeServer,
+	grpcserver.NewTaskServer,
+	grpcserver.NewMessageServer,
+	config.NewConfigPathService,
+	user.GetUserService,
+	schedule.GetScheduleService,
+	admin.GetSpiderAdminService,
+	stats.GetStatsService,
+	nodeconfig.NewNodeConfigService,
+	taskstats.GetTaskStatsService,
+	color.NewService,
+	scheduler.GetTaskSchedulerService,
+	handler.GetTaskHandlerService,
+}
+
+func injectModules() {
+	c := container.GetContainer()
+	for _, injector := range injectors {
+		if err := c.Provide(injector); err != nil {
+			panic(err)
+		}
+	}
+}