mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
feat: migrated git from spider
This commit is contained in:
Marvin Zhang
@@ -58,7 +58,6 @@ func InitRoutes(app *gin.Engine) (err error) {
|
||||
RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]())
|
||||
RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]())
|
||||
RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]())
|
||||
RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]())
|
||||
RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]())
|
||||
RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]())
|
||||
RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]())
|
||||
@@ -172,31 +171,7 @@ func InitRoutes(app *gin.Engine) (err error) {
|
||||
Path: "/:id/run",
|
||||
HandlerFunc: PostSpiderRun,
|
||||
},
|
||||
Action{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/git",
|
||||
HandlerFunc: GetSpiderGit,
|
||||
},
|
||||
Action{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/git/remote-refs",
|
||||
HandlerFunc: GetSpiderGitRemoteRefs,
|
||||
},
|
||||
Action{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/git/checkout",
|
||||
HandlerFunc: PostSpiderGitCheckout,
|
||||
},
|
||||
Action{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/git/pull",
|
||||
HandlerFunc: PostSpiderGitPull,
|
||||
},
|
||||
Action{
|
||||
Method: http.MethodPost,
|
||||
Path: "/:id/git/commit",
|
||||
HandlerFunc: PostSpiderGitCommit,
|
||||
},
|
||||
|
||||
Action{
|
||||
Method: http.MethodGet,
|
||||
Path: "/:id/data-source",
|
||||
|
||||
@@ -5,19 +5,13 @@ import (
|
||||
"fmt"
|
||||
log2 "github.com/apex/log"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/fs"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/models/models"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/core/spider/admin"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/crawlab-team/crawlab/vcs"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/spf13/viper"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
@@ -26,7 +20,6 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@@ -756,266 +749,6 @@ func PostSpiderRun(c *gin.Context) {
|
||||
HandleSuccessWithData(c, taskIds)
|
||||
}
|
||||
|
||||
func GetSpiderGit(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// git client
|
||||
gitClient, err := getSpiderGitClient(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// return null if git client is empty
|
||||
if gitClient == nil {
|
||||
HandleSuccess(c)
|
||||
return
|
||||
}
|
||||
|
||||
// current branch
|
||||
currentBranch, err := gitClient.GetCurrentBranch()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// branches
|
||||
branches, err := gitClient.GetBranches()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
if branches == nil || len(branches) == 0 && currentBranch != "" {
|
||||
branches = []vcs.GitRef{{Name: currentBranch}}
|
||||
}
|
||||
|
||||
// changes
|
||||
changes, err := gitClient.GetStatus()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// logs
|
||||
logs, err := gitClient.GetLogsWithRefs()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// ignore
|
||||
ignore, err := getSpiderGitIgnore(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// git
|
||||
_git, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// response
|
||||
res := bson.M{
|
||||
"current_branch": currentBranch,
|
||||
"branches": branches,
|
||||
"changes": changes,
|
||||
"logs": logs,
|
||||
"ignore": ignore,
|
||||
"git": _git,
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, res)
|
||||
}
|
||||
|
||||
func GetSpiderGitRemoteRefs(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// remote name
|
||||
remoteName := c.Query("remote")
|
||||
if remoteName == "" {
|
||||
remoteName = vcs.GitRemoteNameOrigin
|
||||
}
|
||||
|
||||
// git client
|
||||
gitClient, err := getSpiderGitClient(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// return null if git client is empty
|
||||
if gitClient == nil {
|
||||
HandleSuccess(c)
|
||||
return
|
||||
}
|
||||
|
||||
// refs
|
||||
refs, err := gitClient.GetRemoteRefs(remoteName)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccessWithData(c, refs)
|
||||
}
|
||||
|
||||
func PostSpiderGitCheckout(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// payload
|
||||
var payload struct {
|
||||
Paths []string `json:"paths"`
|
||||
CommitMessage string `json:"commit_message"`
|
||||
Branch string `json:"branch"`
|
||||
Tag string `json:"tag"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// git client
|
||||
gitClient, err := getSpiderGitClient(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// return null if git client is empty
|
||||
if gitClient == nil {
|
||||
HandleSuccess(c)
|
||||
return
|
||||
}
|
||||
|
||||
// branch to pull
|
||||
var branch string
|
||||
if payload.Branch == "" {
|
||||
// by default current branch
|
||||
branch, err = gitClient.GetCurrentBranch()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// payload branch
|
||||
branch = payload.Branch
|
||||
}
|
||||
|
||||
// checkout
|
||||
if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func PostSpiderGitPull(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// payload
|
||||
var payload struct {
|
||||
Paths []string `json:"paths"`
|
||||
CommitMessage string `json:"commit_message"`
|
||||
Branch string `json:"branch"`
|
||||
Tag string `json:"tag"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// git
|
||||
g, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// attempt to sync git
|
||||
adminSvc, err := admin.GetSpiderAdminServiceV2()
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
_ = adminSvc.SyncGitOne(g)
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func PostSpiderGitCommit(c *gin.Context) {
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
if err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// payload
|
||||
var payload entity.GitPayload
|
||||
if err := c.ShouldBindJSON(&payload); err != nil {
|
||||
HandleErrorBadRequest(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// git client
|
||||
gitClient, err := getSpiderGitClient(id)
|
||||
if err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// return null if git client is empty
|
||||
if gitClient == nil {
|
||||
HandleSuccess(c)
|
||||
return
|
||||
}
|
||||
|
||||
// add
|
||||
for _, p := range payload.Paths {
|
||||
if err := gitClient.Add(p); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// commit
|
||||
if err := gitClient.Commit(payload.CommitMessage); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
// push
|
||||
if err := gitClient.Push(
|
||||
vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin),
|
||||
); err != nil {
|
||||
HandleErrorInternalServerError(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
HandleSuccess(c)
|
||||
}
|
||||
|
||||
func GetSpiderDataSource(c *gin.Context) {
|
||||
// spider id
|
||||
id, err := primitive.ObjectIDFromHex(c.Param("id"))
|
||||
@@ -1107,168 +840,6 @@ func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 {
|
||||
return fsSvc
|
||||
}
|
||||
|
||||
func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) {
|
||||
// git
|
||||
g, err := service.NewModelServiceV2[models.GitV2]().GetById(id)
|
||||
if err != nil {
|
||||
if !errors.Is(err, mongo2.ErrNoDocuments) {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// git client
|
||||
workspacePath := viper.GetString("workspace")
|
||||
client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex())))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// set auth
|
||||
utils.InitGitClientAuthV2(g, client)
|
||||
|
||||
// remote name
|
||||
remoteName := vcs.GitRemoteNameOrigin
|
||||
|
||||
// update remote
|
||||
r, err := client.GetRemote(remoteName)
|
||||
if errors.Is(err, git.ErrRemoteNotFound) {
|
||||
// remote not exists, create
|
||||
if _, err := client.CreateRemote(&config.RemoteConfig{
|
||||
Name: remoteName,
|
||||
URLs: []string{g.Url},
|
||||
}); err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
} else if err == nil {
|
||||
// remote exists, update if different
|
||||
if g.Url != r.Config().URLs[0] {
|
||||
if err := client.DeleteRemote(remoteName); err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
if _, err := client.CreateRemote(&config.RemoteConfig{
|
||||
Name: remoteName,
|
||||
URLs: []string{g.Url},
|
||||
}); err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
}
|
||||
client.SetRemoteUrl(g.Url)
|
||||
} else {
|
||||
// error
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
|
||||
// check if head reference exists
|
||||
_, err = client.GetRepository().Head()
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// align master/main branch
|
||||
alignSpiderGitBranch(client)
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func alignSpiderGitBranch(gitClient *vcs.GitClient) {
|
||||
// current branch
|
||||
currentBranch, err := gitClient.GetCurrentBranch()
|
||||
if err != nil {
|
||||
trace.PrintError(err)
|
||||
return
|
||||
}
|
||||
|
||||
// skip if current branch is not master
|
||||
if currentBranch != vcs.GitBranchNameMaster {
|
||||
return
|
||||
}
|
||||
|
||||
// remote refs
|
||||
refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin)
|
||||
if err != nil {
|
||||
trace.PrintError(err)
|
||||
return
|
||||
}
|
||||
|
||||
// main branch
|
||||
defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs)
|
||||
if err != nil || defaultRemoteBranch == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// move branch
|
||||
if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil {
|
||||
trace.PrintError(err)
|
||||
}
|
||||
}
|
||||
|
||||
func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) {
|
||||
// remote branch name
|
||||
for _, r := range refs {
|
||||
if r.Type != vcs.GitRefTypeBranch {
|
||||
continue
|
||||
}
|
||||
|
||||
if r.Name == vcs.GitBranchNameMain {
|
||||
defaultRemoteBranchName = r.Name
|
||||
break
|
||||
}
|
||||
|
||||
if r.Name == vcs.GitBranchNameMaster {
|
||||
defaultRemoteBranchName = r.Name
|
||||
continue
|
||||
}
|
||||
|
||||
if defaultRemoteBranchName == "" {
|
||||
defaultRemoteBranchName = r.Name
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return defaultRemoteBranchName, nil
|
||||
}
|
||||
|
||||
func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) {
|
||||
workspacePath := viper.GetString("workspace")
|
||||
filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore")
|
||||
if !utils.Exists(filePath) {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, trace.TraceError(err)
|
||||
}
|
||||
ignore = strings.Split(string(data), "\n")
|
||||
return ignore, nil
|
||||
}
|
||||
|
||||
func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) {
|
||||
if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
// pull
|
||||
return spiderGitPull(gitClient, remote, branch)
|
||||
}
|
||||
|
||||
func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) {
|
||||
// pull
|
||||
if err := gitClient.Pull(
|
||||
vcs.WithRemoteNamePull(remote),
|
||||
vcs.WithBranchNamePull(branch),
|
||||
); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
// reset
|
||||
if err := gitClient.Reset(); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func upsertSpiderDataCollection(s *models.SpiderV2) (err error) {
|
||||
modelSvc := service.NewModelServiceV2[models.DataCollectionV2]()
|
||||
if s.ColId.IsZero() {
|
||||
|
||||
Reference in New Issue
Block a user