Files
crawlab/vcs/git.go
2024-06-14 16:37:48 +08:00

1020 lines
21 KiB
Go

package vcs
import (
"github.com/apex/log"
"github.com/crawlab-team/crawlab/trace"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5/storage/memory"
"golang.org/x/crypto/ssh"
"io/ioutil"
"os"
"path"
"regexp"
"sort"
"strings"
)
var headRefRegexp, _ = regexp.Compile("^ref: (.*)")
type GitClient struct {
// settings
path string
remoteUrl string
isMem bool
authType GitAuthType
username string
password string
privateKey string
privateKeyPath string
defaultBranch string
// internals
r *git.Repository
}
func (c *GitClient) Init() (err error) {
initType := c.getInitType()
switch initType {
case GitInitTypeFs:
err = c.initFs()
case GitInitTypeMem:
err = c.initMem()
}
if err != nil {
return err
}
// if remote url is not empty and no remote exists
// create default remote and pull from remote url
remotes, err := c.r.Remotes()
if err != nil {
return err
}
if c.remoteUrl != "" && len(remotes) == 0 {
// attempt to get default remote
if _, err := c.r.Remote(GitRemoteNameOrigin); err != nil {
if err != git.ErrRemoteNotFound {
return trace.TraceError(err)
}
err = nil
// create default remote
if err := c.createRemote(GitRemoteNameOrigin, c.remoteUrl); err != nil {
return err
}
//// pull
//opts := []GitPullOption{
// WithRemoteNamePull(GitRemoteNameOrigin),
//}
//if err := c.Pull(opts...); err != nil {
// return err
//}
}
}
return nil
}
func (c *GitClient) Dispose() (err error) {
switch c.getInitType() {
case GitInitTypeFs:
if err := os.RemoveAll(c.path); err != nil {
return trace.TraceError(err)
}
case GitInitTypeMem:
GitMemStorages.Delete(c.path)
GitMemFileSystem.Delete(c.path)
}
return nil
}
func (c *GitClient) Checkout(opts ...GitCheckoutOption) (err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
// apply options
o := &git.CheckoutOptions{}
for _, opt := range opts {
opt(o)
}
// checkout to the branch
if err := wt.Checkout(o); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) Commit(msg string, opts ...GitCommitOption) (err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
// apply options
o := &git.CommitOptions{}
for _, opt := range opts {
opt(o)
}
// commit
if _, err := wt.Commit(msg, o); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) Pull(opts ...GitPullOption) (err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
// auth
auth, err := c.getGitAuth()
if err != nil {
return err
}
if auth != nil {
opts = append(opts, WithAuthPull(auth))
}
// apply options
o := &git.PullOptions{}
for _, opt := range opts {
opt(o)
}
// pull
if err := wt.Pull(o); err != nil {
if err == transport.ErrEmptyRemoteRepository {
return nil
}
if err == transport.ErrEmptyUploadPackRequest {
return nil
}
if err == git.NoErrAlreadyUpToDate {
return nil
}
if err == git.ErrNonFastForwardUpdate {
return nil
}
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) Push(opts ...GitPushOption) (err error) {
// auth
auth, err := c.getGitAuth()
if err != nil {
return err
}
if auth != nil {
opts = append(opts, WithAuthPush(auth))
}
// apply options
o := &git.PushOptions{}
for _, opt := range opts {
opt(o)
}
// push
if err := c.r.Push(o); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) Reset(opts ...GitResetOption) (err error) {
// apply options
o := &git.ResetOptions{
Mode: git.HardReset,
}
for _, opt := range opts {
opt(o)
}
// worktree
wt, err := c.r.Worktree()
if err != nil {
return err
}
// reset
if err := wt.Reset(o); err != nil {
return err
}
// clean
if err := wt.Clean(&git.CleanOptions{Dir: true}); err != nil {
return err
}
return nil
}
func (c *GitClient) CreateBranch(branch, remote string, ref *plumbing.Reference) (err error) {
return c.createBranch(branch, remote, ref)
}
func (c *GitClient) CheckoutBranchFromRef(branch string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) {
return c.CheckoutBranchWithRemote(branch, "", ref, opts...)
}
func (c *GitClient) CheckoutBranchWithRemoteFromRef(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) {
return c.CheckoutBranchWithRemote(branch, remote, ref, opts...)
}
func (c *GitClient) CheckoutBranch(branch string, opts ...GitCheckoutOption) (err error) {
return c.CheckoutBranchWithRemote(branch, "", nil, opts...)
}
func (c *GitClient) CheckoutBranchWithRemote(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) {
if remote == "" {
remote = GitRemoteNameOrigin
}
// remote
if _, err := c.r.Remote(remote); err != nil {
return trace.TraceError(err)
}
// check if the branch exists
b, err := c.r.Branch(branch)
if err != nil {
if err == git.ErrBranchNotFound {
// create a new branch if it does not exist
if err := c.createBranch(branch, remote, ref); err != nil {
return err
}
b, err = c.r.Branch(branch)
if err != nil {
return trace.TraceError(err)
}
} else {
return trace.TraceError(err)
}
}
// set branch remote
if remote != "" {
b.Remote = remote
}
// add to options
opts = append(opts, WithBranch(branch))
return c.Checkout(opts...)
}
func (c *GitClient) CheckoutHash(hash string, opts ...GitCheckoutOption) (err error) {
// add to options
opts = append(opts, WithHash(hash))
return c.Checkout(opts...)
}
func (c *GitClient) MoveBranch(from, to string) (err error) {
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
if err := wt.Checkout(&git.CheckoutOptions{
Create: true,
Branch: plumbing.NewBranchReferenceName(to),
}); err != nil {
return trace.TraceError(err)
}
fromRef, err := c.r.Reference(plumbing.NewBranchReferenceName(from), false)
if err != nil {
return trace.TraceError(err)
}
if err := c.r.Storer.RemoveReference(fromRef.Name()); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) CommitAll(msg string, opts ...GitCommitOption) (err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
// add all files
if _, err := wt.Add("."); err != nil {
return trace.TraceError(err)
}
return c.Commit(msg, opts...)
}
func (c *GitClient) GetLogs() (logs []GitLog, err error) {
iter, err := c.r.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, trace.TraceError(err)
}
if err := iter.ForEach(func(commit *object.Commit) error {
log := GitLog{
Hash: commit.Hash.String(),
Msg: commit.Message,
AuthorName: commit.Author.Name,
AuthorEmail: commit.Author.Email,
Timestamp: commit.Author.When,
}
logs = append(logs, log)
return nil
}); err != nil {
return nil, trace.TraceError(err)
}
return
}
func (c *GitClient) GetLogsWithRefs() (logs []GitLog, err error) {
// logs without tags
logs, err = c.GetLogs()
if err != nil {
return nil, err
}
// branches
branches, err := c.GetBranches()
if err != nil {
return nil, err
}
// tags
tags, err := c.GetTags()
if err != nil {
return nil, err
}
// refs
refs := append(branches, tags...)
// refs map
refsMap := map[string][]GitRef{}
for _, ref := range refs {
_, ok := refsMap[ref.Hash]
if !ok {
refsMap[ref.Hash] = []GitRef{}
}
refsMap[ref.Hash] = append(refsMap[ref.Hash], ref)
}
// iterate logs
for i, l := range logs {
refs, ok := refsMap[l.Hash]
if ok {
logs[i].Refs = refs
}
}
return logs, nil
}
func (c *GitClient) GetRepository() (r *git.Repository) {
return c.r
}
func (c *GitClient) GetPath() (path string) {
return c.path
}
func (c *GitClient) SetPath(path string) {
c.path = path
}
func (c *GitClient) GetRemoteUrl() (path string) {
return c.remoteUrl
}
func (c *GitClient) SetRemoteUrl(url string) {
c.remoteUrl = url
}
func (c *GitClient) GetIsMem() (isMem bool) {
return c.isMem
}
func (c *GitClient) SetIsMem(isMem bool) {
c.isMem = isMem
}
func (c *GitClient) GetAuthType() (authType GitAuthType) {
return c.authType
}
func (c *GitClient) SetAuthType(authType GitAuthType) {
c.authType = authType
}
func (c *GitClient) GetUsername() (username string) {
return c.username
}
func (c *GitClient) SetUsername(username string) {
c.username = username
}
func (c *GitClient) GetPassword() (password string) {
return c.password
}
func (c *GitClient) SetPassword(password string) {
c.password = password
}
func (c *GitClient) GetPrivateKey() (key string) {
return c.privateKey
}
func (c *GitClient) SetPrivateKey(key string) {
c.privateKey = key
}
func (c *GitClient) GetPrivateKeyPath() (path string) {
return c.privateKeyPath
}
func (c *GitClient) SetPrivateKeyPath(path string) {
c.privateKeyPath = path
}
func (c *GitClient) GetCurrentBranch() (branch string, err error) {
// attempt to get branch from .git/HEAD
headRefStr, err := c.getHeadRef()
if err != nil {
return "", err
}
// if .git/HEAD points to refs/heads/master, return branch as master
if headRefStr == plumbing.Master.String() {
return GitBranchNameMaster, nil
}
// attempt to get head ref
headRef, err := c.r.Head()
if err != nil {
return "", trace.TraceError(err)
}
if !headRef.Name().IsBranch() {
return "", trace.TraceError(ErrUnableToGetCurrentBranch)
}
return headRef.Name().Short(), nil
}
func (c *GitClient) GetCurrentBranchRef() (ref *GitRef, err error) {
currentBranch, err := c.GetCurrentBranch()
if err != nil {
return nil, err
}
branches, err := c.GetBranches()
if err != nil {
return nil, err
}
for _, branch := range branches {
if branch.Name == currentBranch {
return &branch, nil
}
}
return nil, trace.TraceError(ErrUnableToGetCurrentBranch)
}
func (c *GitClient) GetBranches() (branches []GitRef, err error) {
iter, err := c.r.Branches()
if err != nil {
return nil, trace.TraceError(err)
}
_ = iter.ForEach(func(r *plumbing.Reference) error {
branches = append(branches, GitRef{
Type: GitRefTypeBranch,
Name: r.Name().Short(),
Hash: r.Hash().String(),
})
return nil
})
return branches, nil
}
func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err error) {
// remote
r, err := c.r.Remote(remoteName)
if err != nil {
if err == git.ErrRemoteNotFound {
return nil, nil
}
return nil, trace.TraceError(err)
}
// auth
auth, err := c.getGitAuth()
if err != nil {
return nil, err
}
// refs
refs, err := r.List(&git.ListOptions{Auth: auth})
if err != nil {
if err != transport.ErrEmptyRemoteRepository {
return nil, trace.TraceError(err)
}
return nil, nil
}
// iterate refs
for _, ref := range refs {
// ref type
var refType string
if strings.HasPrefix(ref.Name().String(), "refs/heads") {
refType = GitRefTypeBranch
} else if strings.HasPrefix(ref.Name().String(), "refs/tags") {
refType = GitRefTypeTag
} else {
continue
}
// add to branches
gitRefs = append(gitRefs, GitRef{
Type: refType,
Name: ref.Name().Short(),
FullName: ref.Name().String(),
Hash: ref.Hash().String(),
})
}
// logs without tags
logs, err := c.GetLogs()
if err != nil {
return nil, err
}
// logs map
logsMap := map[string]GitLog{}
for _, l := range logs {
logsMap[l.Hash] = l
}
// iterate git refs
for i, gitRef := range gitRefs {
l, ok := logsMap[gitRef.Hash]
if !ok {
continue
}
gitRefs[i].Timestamp = l.Timestamp
}
// sort git refs
sort.Slice(gitRefs, func(i, j int) bool {
return gitRefs[i].Timestamp.Unix() > gitRefs[j].Timestamp.Unix()
})
return gitRefs, nil
}
func (c *GitClient) GetTags() (tags []GitRef, err error) {
iter, err := c.r.Tags()
if err != nil {
return nil, trace.TraceError(err)
}
_ = iter.ForEach(func(r *plumbing.Reference) error {
tags = append(tags, GitRef{
Type: GitRefTypeTag,
Name: r.Name().Short(),
Hash: r.Hash().String(),
})
return nil
})
return tags, nil
}
func (c *GitClient) GetStatus() (statusList []GitFileStatus, err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return nil, trace.TraceError(err)
}
// status
status, err := wt.Status()
if err != nil {
log.Warnf("failed to get worktree status: %v", err)
}
// file status list
var list []GitFileStatus
for filePath, fileStatus := range status {
// file name
fileName := path.Base(filePath)
// file status
s := GitFileStatus{
Path: filePath,
Name: fileName,
IsDir: false,
Staging: c.getStatusString(fileStatus.Staging),
Worktree: c.getStatusString(fileStatus.Worktree),
Extra: fileStatus.Extra,
}
// add to list
list = append(list, s)
}
// sort list ascending
sort.Slice(list, func(i, j int) bool {
return list[i].Path < list[j].Path
})
return list, nil
}
func (c *GitClient) Add(filePath string) (err error) {
// worktree
wt, err := c.r.Worktree()
if err != nil {
return trace.TraceError(err)
}
if _, err := wt.Add(filePath); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) GetRemote(name string) (r *git.Remote, err error) {
return c.r.Remote(name)
}
func (c *GitClient) CreateRemote(cfg *config.RemoteConfig) (r *git.Remote, err error) {
return c.r.CreateRemote(cfg)
}
func (c *GitClient) DeleteRemote(name string) (err error) {
return c.r.DeleteRemote(name)
}
func (c *GitClient) IsRemoteChanged() (ok bool, err error) {
return c.isRemoteChanged()
}
func (c *GitClient) initMem() (err error) {
// validate options
if !c.isMem || c.path == "" {
return trace.TraceError(ErrInvalidOptions)
}
// get storage and worktree
storage, wt := c.getMemStorageAndMemFs(c.path)
// attempt to init
c.r, err = git.Init(storage, wt)
if err != nil {
if err == git.ErrRepositoryAlreadyExists {
// if already exists, attempt to open
c.r, err = git.Open(storage, wt)
if err != nil {
return trace.TraceError(err)
}
} else {
return trace.TraceError(err)
}
}
return nil
}
func (c *GitClient) initFs() (err error) {
// validate options
if c.path == "" {
return trace.TraceError(ErrInvalidOptions)
}
// create directory if not exists
_, err = os.Stat(c.path)
if err != nil {
if err := os.MkdirAll(c.path, os.ModePerm); err != nil {
return trace.TraceError(err)
}
err = nil
}
// try to open repo
c.r, err = git.PlainOpen(c.path)
if err == git.ErrRepositoryNotExists {
// repo not exists, init
c.r, err = git.PlainInit(c.path, false)
if err != nil {
return trace.TraceError(err)
}
} else if err != nil {
// error
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) clone() (err error) {
// validate
if c.remoteUrl == "" {
return trace.TraceError(ErrUnableToCloneWithEmptyRemoteUrl)
}
// auth
auth, err := c.getGitAuth()
if err != nil {
return err
}
// options
o := &git.CloneOptions{
URL: c.remoteUrl,
Auth: auth,
}
// clone
if _, err := git.PlainClone(c.path, false, o); err != nil {
return trace.TraceError(err)
}
return nil
}
func (c *GitClient) getInitType() (res GitInitType) {
if c.isMem {
return GitInitTypeMem
} else {
return GitInitTypeFs
}
}
func (c *GitClient) createRemote(remoteName string, url string) (err error) {
_, err = c.r.CreateRemote(&config.RemoteConfig{
Name: remoteName,
URLs: []string{url},
})
if err != nil {
return trace.TraceError(err)
}
return
}
func (c *GitClient) getMemStorageAndMemFs(key string) (storage *memory.Storage, fs billy.Filesystem) {
// storage
storageItem, ok := GitMemStorages.Load(key)
if !ok {
storage = memory.NewStorage()
GitMemStorages.Store(key, storage)
} else {
switch storageItem.(type) {
case *memory.Storage:
storage = storageItem.(*memory.Storage)
default:
storage = memory.NewStorage()
GitMemStorages.Store(key, storage)
}
}
// file system
fsItem, ok := GitMemFileSystem.Load(key)
if !ok {
fs = memfs.New()
GitMemFileSystem.Store(key, fs)
} else {
switch fsItem.(type) {
case billy.Filesystem:
fs = fsItem.(billy.Filesystem)
default:
fs = memfs.New()
GitMemFileSystem.Store(key, fs)
}
}
return storage, fs
}
func (c *GitClient) getGitAuth() (auth transport.AuthMethod, err error) {
switch c.authType {
case GitAuthTypeNone:
return nil, nil
case GitAuthTypeHTTP:
if c.username == "" && c.password == "" {
return nil, nil
}
auth = &http.BasicAuth{
Username: c.username,
Password: c.password,
}
return auth, nil
case GitAuthTypeSSH:
var privateKeyData []byte
if c.privateKey != "" {
// private key content
privateKeyData = []byte(c.privateKey)
} else if c.privateKeyPath != "" {
// read from private key file
privateKeyData, err = ioutil.ReadFile(c.privateKeyPath)
if err != nil {
return nil, trace.TraceError(err)
}
} else {
// no private key
return nil, nil
}
signer, err := ssh.ParsePrivateKey(privateKeyData)
if err != nil {
return nil, trace.TraceError(err)
}
auth = &gitssh.PublicKeys{
User: c.username,
Signer: signer,
HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
},
}
return auth, nil
default:
return nil, trace.TraceError(ErrInvalidAuthType)
}
}
func (c *GitClient) getHeadRef() (ref string, err error) {
wt, err := c.r.Worktree()
if err != nil {
return "", trace.TraceError(err)
}
fh, err := wt.Filesystem.Open(path.Join(".git", "HEAD"))
if err != nil {
return "", trace.TraceError(err)
}
data, err := ioutil.ReadAll(fh)
if err != nil {
return "", trace.TraceError(err)
}
m := headRefRegexp.FindStringSubmatch(string(data))
if len(m) < 2 {
return "", trace.TraceError(ErrInvalidHeadRef)
}
return m[1], nil
}
func (c *GitClient) getStatusString(statusCode git.StatusCode) (code string) {
return string(statusCode)
//switch statusCode {
//}
//Unmodified StatusCode = ' '
//Untracked StatusCode = '?'
//Modified StatusCode = 'M'
//Added StatusCode = 'A'
//Deleted StatusCode = 'D'
//Renamed StatusCode = 'R'
//Copied StatusCode = 'C'
//UpdatedButUnmerged StatusCode = 'U'
}
func (c *GitClient) getDirPaths(filePath string) (paths []string) {
pathItems := strings.Split(filePath, "/")
var items []string
for i, pathItem := range pathItems {
if i == len(pathItems)-1 {
continue
}
items = append(items, pathItem)
dirPath := strings.Join(items, "/")
paths = append(paths, dirPath)
}
return paths
}
func (c *GitClient) createBranch(branch, remote string, ref *plumbing.Reference) (err error) {
// create a new branch if it does not exist
cfg := config.Branch{
Name: branch,
Remote: remote,
}
if err := c.r.CreateBranch(&cfg); err != nil {
return err
}
// if ref is nil
if ref == nil {
// try to set to remote ref of branch first
ref, err = c.getBranchHashRef(branch, remote)
// if no matched remote branch, set to HEAD
if err == ErrNoMatchedRemoteBranch {
ref, err = c.r.Head()
if err != nil {
return trace.TraceError(err)
}
}
// error
if err != nil {
return trace.TraceError(err)
}
}
// branch reference name
branchRefName := plumbing.NewBranchReferenceName(branch)
// branch reference
branchRef := plumbing.NewHashReference(branchRefName, ref.Hash())
// set HEAD to branch reference
if err := c.r.Storer.SetReference(branchRef); err != nil {
return err
}
return nil
}
func (c *GitClient) getBranchHashRef(branch, remote string) (hashRef *plumbing.Reference, err error) {
refs, err := c.GetRemoteRefs(remote)
if err != nil {
return nil, err
}
var branchRef *GitRef
for _, r := range refs {
if r.Name == branch {
branchRef = &r
break
}
}
if branchRef == nil {
return nil, ErrNoMatchedRemoteBranch
}
branchHashRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName(branch), plumbing.NewHash(branchRef.Hash))
return branchHashRef, nil
}
func (c *GitClient) isRemoteChanged() (ok bool, err error) {
b, err := c.GetCurrentBranchRef()
if err != nil {
return false, err
}
refs, err := c.GetRemoteRefs(GitRemoteNameOrigin)
if err != nil {
return false, err
}
for _, r := range refs {
if r.Name == b.Name {
return r.Hash != b.Hash, nil
}
}
return false, nil
}
func NewGitClient(opts ...GitOption) (c *GitClient, err error) {
// client
c = &GitClient{
isMem: false,
authType: GitAuthTypeNone,
username: "git",
privateKeyPath: getDefaultPublicKeyPath(),
}
// apply options
for _, opt := range opts {
opt(c)
}
// init
if err := c.Init(); err != nil {
return c, err
}
return
}