diff --git a/.dockerignore b/.dockerignore index 4207a951..7ca01557 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,5 @@ logs *.log dist/ -**/node_modules/ \ No newline at end of file +**/node_modules/ +**/tmp/ diff --git a/backend/services/git.go b/backend/services/git.go index 14fcb52d..1bb8b3b9 100644 --- a/backend/services/git.go +++ b/backend/services/git.go @@ -30,6 +30,7 @@ type GitCronScheduler struct { } func SaveSpiderGitSyncError(s model.Spider, errMsg string) { + s, _ = model.GetSpider(s.Id) s.GitSyncError = errMsg if err := s.Save(); err != nil { log.Errorf(err.Error()) @@ -140,8 +141,23 @@ func SyncSpiderGit(s model.Spider) (err error) { var auth ssh.AuthMethod if !strings.HasPrefix(s.GitUrl, "http") { // 为 SSH - u, _ := url.Parse(s.GitUrl) - auth, _ = ssh.NewPublicKeysFromFile(u.User.String(), path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "") + regex := regexp.MustCompile("^(?:ssh://?)?([0-9a-zA-Z_]+)@") + res := regex.FindStringSubmatch(s.GitUrl) + username := s.GitUsername + if username == "" { + if len(res) > 1 { + username = res[1] + } else { + username = "git" + } + } + auth, err = ssh.NewPublicKeysFromFile(username, path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "") + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + SaveSpiderGitSyncError(s, err.Error()) + return err + } } // 获取 repo @@ -166,6 +182,9 @@ func SyncSpiderGit(s model.Spider) (err error) { Auth: auth, }); err != nil { if err.Error() == "already up-to-date" { + // 检查是否为 Scrapy + sync := spider_handler.SpiderSync{Spider: s} + sync.CheckIsScrapy() // 如果没有错误,则保存空字符串 SaveSpiderGitSyncError(s, "") return nil @@ -293,12 +312,9 @@ func GetGitSshPublicKey() string { if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) || !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) || !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")) { - cmd := exec.Command("ssh-keygen -q -t rsa -N \"\" -f $HOME/.ssh/id_rsa") - if err := cmd.Start(); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return "" - } + log.Errorf("no ssh public key") + debug.PrintStack() + return "" } content, err := ioutil.ReadFile(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")) if err != nil { diff --git a/docker_init.sh b/docker_init.sh index 6f139c92..b65188b4 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -29,5 +29,8 @@ then /bin/sh /app/backend/scripts/install-nodejs.sh fi +# generate ssh +ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa + # start backend crawlab \ No newline at end of file