From fb9b6e16197339f106096c7d78e33090ae8a6df9 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 18 Feb 2020 13:42:31 +0800
Subject: [PATCH 1/3] =?UTF-8?q?=E6=9B=B4=E6=94=B9ssh=E9=80=BB=E8=BE=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/services/git.go | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/backend/services/git.go b/backend/services/git.go
index 14fcb52d..27b5420c 100644
--- a/backend/services/git.go
+++ b/backend/services/git.go
@@ -140,8 +140,23 @@ func SyncSpiderGit(s model.Spider) (err error) {
 	var auth ssh.AuthMethod
 	if !strings.HasPrefix(s.GitUrl, "http") {
 		// 为 SSH
-		u, _ := url.Parse(s.GitUrl)
-		auth, _ = ssh.NewPublicKeysFromFile(u.User.String(), path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "")
+		regex := regexp.MustCompile("^(?:ssh://?)?([0-9a-zA-Z_]+)@")
+		res := regex.FindStringSubmatch(s.GitUrl)
+		username := s.GitUsername
+		if username == "" {
+			if len(res) > 1 {
+				username = res[1]
+			} else {
+				username = "git"
+			}
+		}
+		auth, err = ssh.NewPublicKeysFromFile(username, path.Join(os.Getenv("HOME"), ".ssh", "id_rsa"), "")
+		if err != nil {
+			log.Error(err.Error())
+			debug.PrintStack()
+			SaveSpiderGitSyncError(s, err.Error())
+			return err
+		}
 	}
 
 	// 获取 repo
@@ -166,6 +181,9 @@ func SyncSpiderGit(s model.Spider) (err error) {
 		Auth:       auth,
 	}); err != nil {
 		if err.Error() == "already up-to-date" {
+			// 检查是否为 Scrapy
+			sync := spider_handler.SpiderSync{Spider: s}
+			sync.CheckIsScrapy()
 			// 如果没有错误，则保存空字符串
 			SaveSpiderGitSyncError(s, "")
 			return nil

From 357a2d26a8108dc0fc738a6f1740271d3737a48f Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 18 Feb 2020 14:57:57 +0800
Subject: [PATCH 2/3] fixed unable to update is_scrapy issue

---
 backend/services/git.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/services/git.go b/backend/services/git.go
index 27b5420c..f1d42f23 100644
--- a/backend/services/git.go
+++ b/backend/services/git.go
@@ -30,6 +30,7 @@ type GitCronScheduler struct {
 }
 
 func SaveSpiderGitSyncError(s model.Spider, errMsg string) {
+	s, _ = model.GetSpider(s.Id)
 	s.GitSyncError = errMsg
 	if err := s.Save(); err != nil {
 		log.Errorf(err.Error())
@@ -311,7 +312,7 @@ func GetGitSshPublicKey() string {
 	if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")) {
-		cmd := exec.Command("ssh-keygen -q -t rsa -N \"\" -f $HOME/.ssh/id_rsa")
+		cmd := exec.Command(fmt.Sprintf("ssh-keygen -q -t rsa -N \"\" -f %s/.ssh/id_rsa", os.Getenv("HOME")))
 		if err := cmd.Start(); err != nil {
 			log.Errorf(err.Error())
 			debug.PrintStack()

From 16263a9423bb33422bfe0eb762716aa6e9d68e4c Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Tue, 18 Feb 2020 15:20:57 +0800
Subject: [PATCH 3/3] fixed ssh-keygen issue

---
 .dockerignore           | 3 ++-
 backend/services/git.go | 9 +++------
 docker_init.sh          | 3 +++
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 4207a951..7ca01557 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,4 +2,5 @@
 logs
 *.log
 dist/
-**/node_modules/
\ No newline at end of file
+**/node_modules/
+**/tmp/
diff --git a/backend/services/git.go b/backend/services/git.go
index f1d42f23..1bb8b3b9 100644
--- a/backend/services/git.go
+++ b/backend/services/git.go
@@ -312,12 +312,9 @@ func GetGitSshPublicKey() string {
 	if !utils.Exists(path.Join(os.Getenv("HOME"), ".ssh")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa")) ||
 		!utils.Exists(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub")) {
-		cmd := exec.Command(fmt.Sprintf("ssh-keygen -q -t rsa -N \"\" -f %s/.ssh/id_rsa", os.Getenv("HOME")))
-		if err := cmd.Start(); err != nil {
-			log.Errorf(err.Error())
-			debug.PrintStack()
-			return ""
-		}
+		log.Errorf("no ssh public key")
+		debug.PrintStack()
+		return ""
 	}
 	content, err := ioutil.ReadFile(path.Join(os.Getenv("HOME"), ".ssh", "id_rsa.pub"))
 	if err != nil {
diff --git a/docker_init.sh b/docker_init.sh
index 6f139c92..b65188b4 100755
--- a/docker_init.sh
+++ b/docker_init.sh
@@ -29,5 +29,8 @@ then
 	/bin/sh /app/backend/scripts/install-nodejs.sh
 fi
 
+# generate ssh
+ssh-keygen -q -t rsa -N "" -f ${HOME}/.ssh/id_rsa
+
 # start backend
 crawlab
\ No newline at end of file