Merge pull request #573 from crawlab-team/develop

Develop

.dockerignore

@@ -2,4 +2,5 @@
 logs
 *.log
 dist/
-**/node_modules/
+**/node_modules/
+**/tmp/

backend/services/git.go

@@ -30,6 +30,7 @@ type GitCronScheduler struct {
 }
 
 func SaveSpiderGitSyncError(s model.Spider, errMsg string) {
+	s, _ = model.GetSpider(s.Id)
 	s.GitSyncError = errMsg
 	if err := s.Save(); err != nil {
 		log.Errorf(err.Error())
@@ -140,8 +141,23 @@ func SyncSpiderGit(s model.Spider) (err error) {
 	var auth ssh.AuthMethod
 	if !strings.HasPrefix(s.GitUrl, "http") {
 		// 为 SSH
-		u, _ := url.Parse(s.GitUrl)
-		auth, _ = ssh.NewPublicKeysFromFile(u.User.String(), path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "")
+		regex := regexp.MustCompile("^(?:ssh://?)?([0-9a-zA-Z_]+)@")
+		res := regex.FindStringSubmatch(s.GitUrl)
+		username := s.GitUsername
+		if username == "" {
+			if len(res) > 1 {
+				username = res[1]
+			} else {
+				username = "git"
+			}
+		}
+		auth, err = ssh.NewPublicKeysFromFile(username, path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "")
+		if err != nil {
+			log.Error(err.Error())
+			debug.PrintStack()
+			SaveSpiderGitSyncError(s, err.Error())
+			return err
+		}
 	}
 
 	// 获取 repo
@@ -166,6 +182,9 @@ func SyncSpiderGit(s model.Spider) (err error) {
 		Auth:       auth,
 	}); err != nil {
 		if err.Error() == "already up-to-date" {
+			// 检查是否为 Scrapy
+			sync := spider_handler.SpiderSync{Spider: s}
+			sync.CheckIsScrapy()
 			// 如果没有错误，则保存空字符串
 			SaveSpiderGitSyncError(s, "")
 			return nil
@@ -293,12 +312,9 @@ func GetGitSshPublicKey() string {
 	if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")) {
-		cmd := exec.Command("ssh-keygen -q -t rsa -N \"\" -f $HOME/.ssh/id_rsa")
-		if err := cmd.Start(); err != nil {
-			log.Errorf(err.Error())
-			debug.PrintStack()
-			return ""
-		}
+		log.Errorf("no ssh public key")
+		debug.PrintStack()
+		return ""
 	}
 	content, err := ioutil.ReadFile(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub"))
 	if err != nil {

docker_init.sh

@@ -29,5 +29,8 @@ then
 	/bin/sh /app/backend/scripts/install-nodejs.sh
 fi
 
+# generate ssh
+ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa
+
 # start backend
 crawlab