package vcs import ( "github.com/apex/log" "github.com/crawlab-team/crawlab/trace" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage/memory" "golang.org/x/crypto/ssh" "io/ioutil" "os" "path" "regexp" "sort" "strings" ) var headRefRegexp, _ = regexp.Compile("^ref: (.*)") type GitClient struct { // settings path string remoteUrl string isMem bool authType GitAuthType username string password string privateKey string privateKeyPath string defaultBranch string // internals r *git.Repository } func (c *GitClient) Init() (err error) { initType := c.getInitType() switch initType { case GitInitTypeFs: err = c.initFs() case GitInitTypeMem: err = c.initMem() } if err != nil { return err } // if remote url is not empty and no remote exists // create default remote and pull from remote url remotes, err := c.r.Remotes() if err != nil { return err } if c.remoteUrl != "" && len(remotes) == 0 { // attempt to get default remote if _, err := c.r.Remote(GitRemoteNameOrigin); err != nil { if err != git.ErrRemoteNotFound { return trace.TraceError(err) } err = nil // create default remote if err := c.createRemote(GitRemoteNameOrigin, c.remoteUrl); err != nil { return err } //// pull //opts := []GitPullOption{ // WithRemoteNamePull(GitRemoteNameOrigin), //} //if err := c.Pull(opts...); err != nil { // return err //} } } return nil } func (c *GitClient) Dispose() (err error) { switch c.getInitType() { case GitInitTypeFs: if err := os.RemoveAll(c.path); err != nil { return trace.TraceError(err) } case GitInitTypeMem: GitMemStorages.Delete(c.path) GitMemFileSystem.Delete(c.path) } return nil } func (c *GitClient) Checkout(opts ...GitCheckoutOption) (err error) { // worktree wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } // apply options o := &git.CheckoutOptions{} for _, opt := range opts { opt(o) } // checkout to the branch if err := wt.Checkout(o); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) Commit(msg string, opts ...GitCommitOption) (err error) { // worktree wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } // apply options o := &git.CommitOptions{} for _, opt := range opts { opt(o) } // commit if _, err := wt.Commit(msg, o); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) Pull(opts ...GitPullOption) (err error) { // worktree wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } // auth auth, err := c.getGitAuth() if err != nil { return err } if auth != nil { opts = append(opts, WithAuthPull(auth)) } // apply options o := &git.PullOptions{} for _, opt := range opts { opt(o) } // pull if err := wt.Pull(o); err != nil { if err == transport.ErrEmptyRemoteRepository { return nil } if err == transport.ErrEmptyUploadPackRequest { return nil } if err == git.NoErrAlreadyUpToDate { return nil } if err == git.ErrNonFastForwardUpdate { return nil } return trace.TraceError(err) } return nil } func (c *GitClient) Push(opts ...GitPushOption) (err error) { // auth auth, err := c.getGitAuth() if err != nil { return err } if auth != nil { opts = append(opts, WithAuthPush(auth)) } // apply options o := &git.PushOptions{} for _, opt := range opts { opt(o) } // push if err := c.r.Push(o); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) Reset(opts ...GitResetOption) (err error) { // apply options o := &git.ResetOptions{ Mode: git.HardReset, } for _, opt := range opts { opt(o) } // worktree wt, err := c.r.Worktree() if err != nil { return err } // reset if err := wt.Reset(o); err != nil { return err } // clean if err := wt.Clean(&git.CleanOptions{Dir: true}); err != nil { return err } return nil } func (c *GitClient) CreateBranch(branch, remote string, ref *plumbing.Reference) (err error) { return c.createBranch(branch, remote, ref) } func (c *GitClient) CheckoutBranchFromRef(branch string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { return c.CheckoutBranchWithRemote(branch, "", ref, opts...) } func (c *GitClient) CheckoutBranchWithRemoteFromRef(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { return c.CheckoutBranchWithRemote(branch, remote, ref, opts...) } func (c *GitClient) CheckoutBranch(branch string, opts ...GitCheckoutOption) (err error) { return c.CheckoutBranchWithRemote(branch, "", nil, opts...) } func (c *GitClient) CheckoutBranchWithRemote(branch, remote string, ref *plumbing.Reference, opts ...GitCheckoutOption) (err error) { if remote == "" { remote = GitRemoteNameOrigin } // remote if _, err := c.r.Remote(remote); err != nil { return trace.TraceError(err) } // check if the branch exists b, err := c.r.Branch(branch) if err != nil { if err == git.ErrBranchNotFound { // create a new branch if it does not exist if err := c.createBranch(branch, remote, ref); err != nil { return err } b, err = c.r.Branch(branch) if err != nil { return trace.TraceError(err) } } else { return trace.TraceError(err) } } // set branch remote if remote != "" { b.Remote = remote } // add to options opts = append(opts, WithBranch(branch)) return c.Checkout(opts...) } func (c *GitClient) CheckoutHash(hash string, opts ...GitCheckoutOption) (err error) { // add to options opts = append(opts, WithHash(hash)) return c.Checkout(opts...) } func (c *GitClient) MoveBranch(from, to string) (err error) { wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } if err := wt.Checkout(&git.CheckoutOptions{ Create: true, Branch: plumbing.NewBranchReferenceName(to), }); err != nil { return trace.TraceError(err) } fromRef, err := c.r.Reference(plumbing.NewBranchReferenceName(from), false) if err != nil { return trace.TraceError(err) } if err := c.r.Storer.RemoveReference(fromRef.Name()); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) CommitAll(msg string, opts ...GitCommitOption) (err error) { // worktree wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } // add all files if _, err := wt.Add("."); err != nil { return trace.TraceError(err) } return c.Commit(msg, opts...) } func (c *GitClient) GetLogs() (logs []GitLog, err error) { iter, err := c.r.Log(&git.LogOptions{ All: true, }) if err != nil { return nil, trace.TraceError(err) } if err := iter.ForEach(func(commit *object.Commit) error { log := GitLog{ Hash: commit.Hash.String(), Msg: commit.Message, AuthorName: commit.Author.Name, AuthorEmail: commit.Author.Email, Timestamp: commit.Author.When, } logs = append(logs, log) return nil }); err != nil { return nil, trace.TraceError(err) } return } func (c *GitClient) GetLogsWithRefs() (logs []GitLog, err error) { // logs without tags logs, err = c.GetLogs() if err != nil { return nil, err } // branches branches, err := c.GetBranches() if err != nil { return nil, err } // tags tags, err := c.GetTags() if err != nil { return nil, err } // refs refs := append(branches, tags...) // refs map refsMap := map[string][]GitRef{} for _, ref := range refs { _, ok := refsMap[ref.Hash] if !ok { refsMap[ref.Hash] = []GitRef{} } refsMap[ref.Hash] = append(refsMap[ref.Hash], ref) } // iterate logs for i, l := range logs { refs, ok := refsMap[l.Hash] if ok { logs[i].Refs = refs } } return logs, nil } func (c *GitClient) GetRepository() (r *git.Repository) { return c.r } func (c *GitClient) GetPath() (path string) { return c.path } func (c *GitClient) SetPath(path string) { c.path = path } func (c *GitClient) GetRemoteUrl() (path string) { return c.remoteUrl } func (c *GitClient) SetRemoteUrl(url string) { c.remoteUrl = url } func (c *GitClient) GetIsMem() (isMem bool) { return c.isMem } func (c *GitClient) SetIsMem(isMem bool) { c.isMem = isMem } func (c *GitClient) GetAuthType() (authType GitAuthType) { return c.authType } func (c *GitClient) SetAuthType(authType GitAuthType) { c.authType = authType } func (c *GitClient) GetUsername() (username string) { return c.username } func (c *GitClient) SetUsername(username string) { c.username = username } func (c *GitClient) GetPassword() (password string) { return c.password } func (c *GitClient) SetPassword(password string) { c.password = password } func (c *GitClient) GetPrivateKey() (key string) { return c.privateKey } func (c *GitClient) SetPrivateKey(key string) { c.privateKey = key } func (c *GitClient) GetPrivateKeyPath() (path string) { return c.privateKeyPath } func (c *GitClient) SetPrivateKeyPath(path string) { c.privateKeyPath = path } func (c *GitClient) GetCurrentBranch() (branch string, err error) { // attempt to get branch from .git/HEAD headRefStr, err := c.getHeadRef() if err != nil { return "", err } // if .git/HEAD points to refs/heads/master, return branch as master if headRefStr == plumbing.Master.String() { return GitBranchNameMaster, nil } // attempt to get head ref headRef, err := c.r.Head() if err != nil { return "", trace.TraceError(err) } if !headRef.Name().IsBranch() { return "", trace.TraceError(ErrUnableToGetCurrentBranch) } return headRef.Name().Short(), nil } func (c *GitClient) GetCurrentBranchRef() (ref *GitRef, err error) { currentBranch, err := c.GetCurrentBranch() if err != nil { return nil, err } branches, err := c.GetBranches() if err != nil { return nil, err } for _, branch := range branches { if branch.Name == currentBranch { return &branch, nil } } return nil, trace.TraceError(ErrUnableToGetCurrentBranch) } func (c *GitClient) GetBranches() (branches []GitRef, err error) { iter, err := c.r.Branches() if err != nil { return nil, trace.TraceError(err) } _ = iter.ForEach(func(r *plumbing.Reference) error { branches = append(branches, GitRef{ Type: GitRefTypeBranch, Name: r.Name().Short(), Hash: r.Hash().String(), }) return nil }) return branches, nil } func (c *GitClient) GetRemoteRefs(remoteName string) (gitRefs []GitRef, err error) { // remote r, err := c.r.Remote(remoteName) if err != nil { if err == git.ErrRemoteNotFound { return nil, nil } return nil, trace.TraceError(err) } // auth auth, err := c.getGitAuth() if err != nil { return nil, err } // refs refs, err := r.List(&git.ListOptions{Auth: auth}) if err != nil { if err != transport.ErrEmptyRemoteRepository { return nil, trace.TraceError(err) } return nil, nil } // iterate refs for _, ref := range refs { // ref type var refType string if strings.HasPrefix(ref.Name().String(), "refs/heads") { refType = GitRefTypeBranch } else if strings.HasPrefix(ref.Name().String(), "refs/tags") { refType = GitRefTypeTag } else { continue } // add to branches gitRefs = append(gitRefs, GitRef{ Type: refType, Name: ref.Name().Short(), FullName: ref.Name().String(), Hash: ref.Hash().String(), }) } // logs without tags logs, err := c.GetLogs() if err != nil { return nil, err } // logs map logsMap := map[string]GitLog{} for _, l := range logs { logsMap[l.Hash] = l } // iterate git refs for i, gitRef := range gitRefs { l, ok := logsMap[gitRef.Hash] if !ok { continue } gitRefs[i].Timestamp = l.Timestamp } // sort git refs sort.Slice(gitRefs, func(i, j int) bool { return gitRefs[i].Timestamp.Unix() > gitRefs[j].Timestamp.Unix() }) return gitRefs, nil } func (c *GitClient) GetTags() (tags []GitRef, err error) { iter, err := c.r.Tags() if err != nil { return nil, trace.TraceError(err) } _ = iter.ForEach(func(r *plumbing.Reference) error { tags = append(tags, GitRef{ Type: GitRefTypeTag, Name: r.Name().Short(), Hash: r.Hash().String(), }) return nil }) return tags, nil } func (c *GitClient) GetStatus() (statusList []GitFileStatus, err error) { // worktree wt, err := c.r.Worktree() if err != nil { return nil, trace.TraceError(err) } // status status, err := wt.Status() if err != nil { log.Warnf("failed to get worktree status: %v", err) } // file status list var list []GitFileStatus for filePath, fileStatus := range status { // file name fileName := path.Base(filePath) // file status s := GitFileStatus{ Path: filePath, Name: fileName, IsDir: false, Staging: c.getStatusString(fileStatus.Staging), Worktree: c.getStatusString(fileStatus.Worktree), Extra: fileStatus.Extra, } // add to list list = append(list, s) } // sort list ascending sort.Slice(list, func(i, j int) bool { return list[i].Path < list[j].Path }) return list, nil } func (c *GitClient) Add(filePath string) (err error) { // worktree wt, err := c.r.Worktree() if err != nil { return trace.TraceError(err) } if _, err := wt.Add(filePath); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) GetRemote(name string) (r *git.Remote, err error) { return c.r.Remote(name) } func (c *GitClient) CreateRemote(cfg *config.RemoteConfig) (r *git.Remote, err error) { return c.r.CreateRemote(cfg) } func (c *GitClient) DeleteRemote(name string) (err error) { return c.r.DeleteRemote(name) } func (c *GitClient) IsRemoteChanged() (ok bool, err error) { return c.isRemoteChanged() } func (c *GitClient) initMem() (err error) { // validate options if !c.isMem || c.path == "" { return trace.TraceError(ErrInvalidOptions) } // get storage and worktree storage, wt := c.getMemStorageAndMemFs(c.path) // attempt to init c.r, err = git.Init(storage, wt) if err != nil { if err == git.ErrRepositoryAlreadyExists { // if already exists, attempt to open c.r, err = git.Open(storage, wt) if err != nil { return trace.TraceError(err) } } else { return trace.TraceError(err) } } return nil } func (c *GitClient) initFs() (err error) { // validate options if c.path == "" { return trace.TraceError(ErrInvalidOptions) } // create directory if not exists _, err = os.Stat(c.path) if err != nil { if err := os.MkdirAll(c.path, os.ModePerm); err != nil { return trace.TraceError(err) } err = nil } // try to open repo c.r, err = git.PlainOpen(c.path) if err == git.ErrRepositoryNotExists { // repo not exists, init c.r, err = git.PlainInit(c.path, false) if err != nil { return trace.TraceError(err) } } else if err != nil { // error return trace.TraceError(err) } return nil } func (c *GitClient) clone() (err error) { // validate if c.remoteUrl == "" { return trace.TraceError(ErrUnableToCloneWithEmptyRemoteUrl) } // auth auth, err := c.getGitAuth() if err != nil { return err } // options o := &git.CloneOptions{ URL: c.remoteUrl, Auth: auth, } // clone if _, err := git.PlainClone(c.path, false, o); err != nil { return trace.TraceError(err) } return nil } func (c *GitClient) getInitType() (res GitInitType) { if c.isMem { return GitInitTypeMem } else { return GitInitTypeFs } } func (c *GitClient) createRemote(remoteName string, url string) (err error) { _, err = c.r.CreateRemote(&config.RemoteConfig{ Name: remoteName, URLs: []string{url}, }) if err != nil { return trace.TraceError(err) } return } func (c *GitClient) getMemStorageAndMemFs(key string) (storage *memory.Storage, fs billy.Filesystem) { // storage storageItem, ok := GitMemStorages.Load(key) if !ok { storage = memory.NewStorage() GitMemStorages.Store(key, storage) } else { switch storageItem.(type) { case *memory.Storage: storage = storageItem.(*memory.Storage) default: storage = memory.NewStorage() GitMemStorages.Store(key, storage) } } // file system fsItem, ok := GitMemFileSystem.Load(key) if !ok { fs = memfs.New() GitMemFileSystem.Store(key, fs) } else { switch fsItem.(type) { case billy.Filesystem: fs = fsItem.(billy.Filesystem) default: fs = memfs.New() GitMemFileSystem.Store(key, fs) } } return storage, fs } func (c *GitClient) getGitAuth() (auth transport.AuthMethod, err error) { switch c.authType { case GitAuthTypeNone: return nil, nil case GitAuthTypeHTTP: if c.username == "" && c.password == "" { return nil, nil } auth = &http.BasicAuth{ Username: c.username, Password: c.password, } return auth, nil case GitAuthTypeSSH: var privateKeyData []byte if c.privateKey != "" { // private key content privateKeyData = []byte(c.privateKey) } else if c.privateKeyPath != "" { // read from private key file privateKeyData, err = ioutil.ReadFile(c.privateKeyPath) if err != nil { return nil, trace.TraceError(err) } } else { // no private key return nil, nil } signer, err := ssh.ParsePrivateKey(privateKeyData) if err != nil { return nil, trace.TraceError(err) } auth = &gitssh.PublicKeys{ User: c.username, Signer: signer, HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ HostKeyCallback: ssh.InsecureIgnoreHostKey(), }, } return auth, nil default: return nil, trace.TraceError(ErrInvalidAuthType) } } func (c *GitClient) getHeadRef() (ref string, err error) { wt, err := c.r.Worktree() if err != nil { return "", trace.TraceError(err) } fh, err := wt.Filesystem.Open(path.Join(".git", "HEAD")) if err != nil { return "", trace.TraceError(err) } data, err := ioutil.ReadAll(fh) if err != nil { return "", trace.TraceError(err) } m := headRefRegexp.FindStringSubmatch(string(data)) if len(m) < 2 { return "", trace.TraceError(ErrInvalidHeadRef) } return m[1], nil } func (c *GitClient) getStatusString(statusCode git.StatusCode) (code string) { return string(statusCode) //switch statusCode { //} //Unmodified StatusCode = ' ' //Untracked StatusCode = '?' //Modified StatusCode = 'M' //Added StatusCode = 'A' //Deleted StatusCode = 'D' //Renamed StatusCode = 'R' //Copied StatusCode = 'C' //UpdatedButUnmerged StatusCode = 'U' } func (c *GitClient) getDirPaths(filePath string) (paths []string) { pathItems := strings.Split(filePath, "/") var items []string for i, pathItem := range pathItems { if i == len(pathItems)-1 { continue } items = append(items, pathItem) dirPath := strings.Join(items, "/") paths = append(paths, dirPath) } return paths } func (c *GitClient) createBranch(branch, remote string, ref *plumbing.Reference) (err error) { // create a new branch if it does not exist cfg := config.Branch{ Name: branch, Remote: remote, } if err := c.r.CreateBranch(&cfg); err != nil { return err } // if ref is nil if ref == nil { // try to set to remote ref of branch first ref, err = c.getBranchHashRef(branch, remote) // if no matched remote branch, set to HEAD if err == ErrNoMatchedRemoteBranch { ref, err = c.r.Head() if err != nil { return trace.TraceError(err) } } // error if err != nil { return trace.TraceError(err) } } // branch reference name branchRefName := plumbing.NewBranchReferenceName(branch) // branch reference branchRef := plumbing.NewHashReference(branchRefName, ref.Hash()) // set HEAD to branch reference if err := c.r.Storer.SetReference(branchRef); err != nil { return err } return nil } func (c *GitClient) getBranchHashRef(branch, remote string) (hashRef *plumbing.Reference, err error) { refs, err := c.GetRemoteRefs(remote) if err != nil { return nil, err } var branchRef *GitRef for _, r := range refs { if r.Name == branch { branchRef = &r break } } if branchRef == nil { return nil, ErrNoMatchedRemoteBranch } branchHashRef := plumbing.NewHashReference(plumbing.NewBranchReferenceName(branch), plumbing.NewHash(branchRef.Hash)) return branchHashRef, nil } func (c *GitClient) isRemoteChanged() (ok bool, err error) { b, err := c.GetCurrentBranchRef() if err != nil { return false, err } refs, err := c.GetRemoteRefs(GitRemoteNameOrigin) if err != nil { return false, err } for _, r := range refs { if r.Name == b.Name { return r.Hash != b.Hash, nil } } return false, nil } func NewGitClient(opts ...GitOption) (c *GitClient, err error) { // client c = &GitClient{ isMem: false, authType: GitAuthTypeNone, username: "git", privateKeyPath: getDefaultPublicKeyPath(), } // apply options for _, opt := range opts { opt(c) } // init if err := c.Init(); err != nil { return c, err } return }