feat: migrated git from spider

core/controllers/router_v2.go

@@ -58,7 +58,6 @@ func InitRoutes(app *gin.Engine) (err error) {
 	RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
 	RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]())
 	RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
-	RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]())
 	RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
 	RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]())
 	RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]())
@@ -172,31 +171,7 @@ func InitRoutes(app *gin.Engine) (err error) {
 			Path:        "/:id/run",
 			HandlerFunc: PostSpiderRun,
 		},
-		Action{
+
-			Method:      http.MethodGet,
-			Path:        "/:id/git",
-			HandlerFunc: GetSpiderGit,
-		},
-		Action{
-			Method:      http.MethodGet,
-			Path:        "/:id/git/remote-refs",
-			HandlerFunc: GetSpiderGitRemoteRefs,
-		},
-		Action{
-			Method:      http.MethodPost,
-			Path:        "/:id/git/checkout",
-			HandlerFunc: PostSpiderGitCheckout,
-		},
-		Action{
-			Method:      http.MethodPost,
-			Path:        "/:id/git/pull",
-			HandlerFunc: PostSpiderGitPull,
-		},
-		Action{
-			Method:      http.MethodPost,
-			Path:        "/:id/git/commit",
-			HandlerFunc: PostSpiderGitCommit,
-		},
 		Action{
 			Method:      http.MethodGet,
 			Path:        "/:id/data-source",

core/controllers/spider_v2.go

@@ -5,19 +5,13 @@ import (
 	"fmt"
 	log2 "github.com/apex/log"
 	"github.com/crawlab-team/crawlab/core/constants"
-	"github.com/crawlab-team/crawlab/core/entity"
 	"github.com/crawlab-team/crawlab/core/fs"
 	"github.com/crawlab-team/crawlab/core/interfaces"
 	"github.com/crawlab-team/crawlab/core/models/models"
 	"github.com/crawlab-team/crawlab/core/models/service"
 	"github.com/crawlab-team/crawlab/core/spider/admin"
-	"github.com/crawlab-team/crawlab/core/utils"
 	"github.com/crawlab-team/crawlab/db/mongo"
-	"github.com/crawlab-team/crawlab/trace"
-	"github.com/crawlab-team/crawlab/vcs"
 	"github.com/gin-gonic/gin"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
 	"github.com/spf13/viper"
 	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -26,7 +20,6 @@ import (
 	"math"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 )
 
@@ -756,266 +749,6 @@ func PostSpiderRun(c *gin.Context) {
 	HandleSuccessWithData(c, taskIds)
 }
 
-func GetSpiderGit(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// git client
-	gitClient, err := getSpiderGitClient(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// return null if git client is empty
-	if gitClient == nil {
-		HandleSuccess(c)
-		return
-	}
-
-	// current branch
-	currentBranch, err := gitClient.GetCurrentBranch()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// branches
-	branches, err := gitClient.GetBranches()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	if branches == nil || len(branches) == 0 && currentBranch != "" {
-		branches = []vcs.GitRef{{Name: currentBranch}}
-	}
-
-	// changes
-	changes, err := gitClient.GetStatus()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// logs
-	logs, err := gitClient.GetLogsWithRefs()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// ignore
-	ignore, err := getSpiderGitIgnore(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// git
-	_git, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
-	if err != nil {
-		if err.Error() != mongo2.ErrNoDocuments.Error() {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-
-	// response
-	res := bson.M{
-		"current_branch": currentBranch,
-		"branches":       branches,
-		"changes":        changes,
-		"logs":           logs,
-		"ignore":         ignore,
-		"git":            _git,
-	}
-
-	HandleSuccessWithData(c, res)
-}
-
-func GetSpiderGitRemoteRefs(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// remote name
-	remoteName := c.Query("remote")
-	if remoteName == "" {
-		remoteName = vcs.GitRemoteNameOrigin
-	}
-
-	// git client
-	gitClient, err := getSpiderGitClient(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// return null if git client is empty
-	if gitClient == nil {
-		HandleSuccess(c)
-		return
-	}
-
-	// refs
-	refs, err := gitClient.GetRemoteRefs(remoteName)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccessWithData(c, refs)
-}
-
-func PostSpiderGitCheckout(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// payload
-	var payload struct {
-		Paths         []string `json:"paths"`
-		CommitMessage string   `json:"commit_message"`
-		Branch        string   `json:"branch"`
-		Tag           string   `json:"tag"`
-	}
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// git client
-	gitClient, err := getSpiderGitClient(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// return null if git client is empty
-	if gitClient == nil {
-		HandleSuccess(c)
-		return
-	}
-
-	// branch to pull
-	var branch string
-	if payload.Branch == "" {
-		// by default current branch
-		branch, err = gitClient.GetCurrentBranch()
-		if err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	} else {
-		// payload branch
-		branch = payload.Branch
-	}
-
-	// checkout
-	if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccess(c)
-}
-
-func PostSpiderGitPull(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// payload
-	var payload struct {
-		Paths         []string `json:"paths"`
-		CommitMessage string   `json:"commit_message"`
-		Branch        string   `json:"branch"`
-		Tag           string   `json:"tag"`
-	}
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// git
-	g, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// attempt to sync git
-	adminSvc, err := admin.GetSpiderAdminServiceV2()
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-	_ = adminSvc.SyncGitOne(g)
-
-	HandleSuccess(c)
-}
-
-func PostSpiderGitCommit(c *gin.Context) {
-	id, err := primitive.ObjectIDFromHex(c.Param("id"))
-	if err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// payload
-	var payload entity.GitPayload
-	if err := c.ShouldBindJSON(&payload); err != nil {
-		HandleErrorBadRequest(c, err)
-		return
-	}
-
-	// git client
-	gitClient, err := getSpiderGitClient(id)
-	if err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// return null if git client is empty
-	if gitClient == nil {
-		HandleSuccess(c)
-		return
-	}
-
-	// add
-	for _, p := range payload.Paths {
-		if err := gitClient.Add(p); err != nil {
-			HandleErrorInternalServerError(c, err)
-			return
-		}
-	}
-
-	// commit
-	if err := gitClient.Commit(payload.CommitMessage); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	// push
-	if err := gitClient.Push(
-		vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin),
-	); err != nil {
-		HandleErrorInternalServerError(c, err)
-		return
-	}
-
-	HandleSuccess(c)
-}
-
 func GetSpiderDataSource(c *gin.Context) {
 	// spider id
 	id, err := primitive.ObjectIDFromHex(c.Param("id"))
@@ -1107,168 +840,6 @@ func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 {
 	return fsSvc
 }
 
-func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) {
-	// git
-	g, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
-	if err != nil {
-		if !errors.Is(err, mongo2.ErrNoDocuments) {
-			return nil, trace.TraceError(err)
-		}
-		return nil, nil
-	}
-
-	// git client
-	workspacePath := viper.GetString("workspace")
-	client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex())))
-	if err != nil {
-		return nil, err
-	}
-
-	// set auth
-	utils.InitGitClientAuthV2(g, client)
-
-	// remote name
-	remoteName := vcs.GitRemoteNameOrigin
-
-	// update remote
-	r, err := client.GetRemote(remoteName)
-	if errors.Is(err, git.ErrRemoteNotFound) {
-		// remote not exists, create
-		if _, err := client.CreateRemote(&config.RemoteConfig{
-			Name: remoteName,
-			URLs: []string{g.Url},
-		}); err != nil {
-			return nil, trace.TraceError(err)
-		}
-	} else if err == nil {
-		// remote exists, update if different
-		if g.Url != r.Config().URLs[0] {
-			if err := client.DeleteRemote(remoteName); err != nil {
-				return nil, trace.TraceError(err)
-			}
-			if _, err := client.CreateRemote(&config.RemoteConfig{
-				Name: remoteName,
-				URLs: []string{g.Url},
-			}); err != nil {
-				return nil, trace.TraceError(err)
-			}
-		}
-		client.SetRemoteUrl(g.Url)
-	} else {
-		// error
-		return nil, trace.TraceError(err)
-	}
-
-	// check if head reference exists
-	_, err = client.GetRepository().Head()
-	if err == nil {
-		return client, nil
-	}
-
-	// align master/main branch
-	alignSpiderGitBranch(client)
-
-	return client, nil
-}
-
-func alignSpiderGitBranch(gitClient *vcs.GitClient) {
-	// current branch
-	currentBranch, err := gitClient.GetCurrentBranch()
-	if err != nil {
-		trace.PrintError(err)
-		return
-	}
-
-	// skip if current branch is not master
-	if currentBranch != vcs.GitBranchNameMaster {
-		return
-	}
-
-	// remote refs
-	refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin)
-	if err != nil {
-		trace.PrintError(err)
-		return
-	}
-
-	// main branch
-	defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs)
-	if err != nil || defaultRemoteBranch == "" {
-		return
-	}
-
-	// move branch
-	if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil {
-		trace.PrintError(err)
-	}
-}
-
-func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) {
-	// remote branch name
-	for _, r := range refs {
-		if r.Type != vcs.GitRefTypeBranch {
-			continue
-		}
-
-		if r.Name == vcs.GitBranchNameMain {
-			defaultRemoteBranchName = r.Name
-			break
-		}
-
-		if r.Name == vcs.GitBranchNameMaster {
-			defaultRemoteBranchName = r.Name
-			continue
-		}
-
-		if defaultRemoteBranchName == "" {
-			defaultRemoteBranchName = r.Name
-			continue
-		}
-	}
-
-	return defaultRemoteBranchName, nil
-}
-
-func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) {
-	workspacePath := viper.GetString("workspace")
-	filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore")
-	if !utils.Exists(filePath) {
-		return nil, nil
-	}
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		return nil, trace.TraceError(err)
-	}
-	ignore = strings.Split(string(data), "\n")
-	return ignore, nil
-}
-
-func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) {
-	if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil {
-		return trace.TraceError(err)
-	}
-
-	// pull
-	return spiderGitPull(gitClient, remote, branch)
-}
-
-func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) {
-	// pull
-	if err := gitClient.Pull(
-		vcs.WithRemoteNamePull(remote),
-		vcs.WithBranchNamePull(branch),
-	); err != nil {
-		return trace.TraceError(err)
-	}
-
-	// reset
-	if err := gitClient.Reset(); err != nil {
-		return trace.TraceError(err)
-	}
-
-	return nil
-}
-
 func upsertSpiderDataCollection(s *models.SpiderV2) (err error) {
 	modelSvc := service.NewModelServiceV2[models.DataCollectionV2]()
 	if s.ColId.IsZero() {