From 1ddac86fe1d0090894eace20134b3b7cc2b6fd9f Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Thu, 20 Feb 2020 17:33:45 +0800
Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=B7=BB=E5=8A=A0scrapy?=
 =?UTF-8?q?=E9=A1=B9=E7=9B=AE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 backend/routes/spider.go   | 18 ++++++++++++++++--
 backend/services/scrapy.go | 21 +++++++++++++++++++++
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/backend/routes/spider.go b/backend/routes/spider.go
index dc84f462..4d9b824a 100644
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -187,19 +187,33 @@ func PutSpider(c *gin.Context) {
 	// 将FileId置空
 	spider.FileId = bson.ObjectIdHex(constants.ObjectIdNull)
 
-	// 创建爬虫目录
+	// 爬虫目录
 	spiderDir := filepath.Join(viper.GetString("spider.path"), spider.Name)
+
+	// 赋值到爬虫实例
+	spider.Src = spiderDir
+
+	// 移除已有爬虫目录
 	if utils.Exists(spiderDir) {
 		if err := os.RemoveAll(spiderDir); err != nil {
 			HandleError(http.StatusInternalServerError, c, err)
 			return
 		}
 	}
+
+	// 生成爬虫目录
 	if err := os.MkdirAll(spiderDir, 0777); err != nil {
 		HandleError(http.StatusInternalServerError, c, err)
 		return
 	}
-	spider.Src = spiderDir
+
+	// 如果为 Scrapy 项目，生成 Scrapy 项目
+	if spider.IsScrapy {
+		if err := services.CreateScrapyProject(spider); err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+	}
 
 	// 添加爬虫到数据库
 	if err := spider.Add(); err != nil {
diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go
index 374df4ee..52c316c3 100644
--- a/backend/services/scrapy.go
+++ b/backend/services/scrapy.go
@@ -145,6 +145,27 @@ func CreateScrapySpider(s model.Spider, name string, domain string) (err error)
 	cmd.Stderr = &stderr
 	if err := cmd.Run(); err != nil {
 		log.Errorf(err.Error())
+		log.Errorf("stdout: " + stdout.String())
+		log.Errorf("stderr: " + stderr.String())
+		debug.PrintStack()
+		return err
+	}
+
+	return
+}
+
+func CreateScrapyProject(s model.Spider) (err error) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	cmd := exec.Command("scrapy", "startproject", s.Name, s.Src)
+	cmd.Dir = s.Src
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		log.Errorf(err.Error())
+		log.Errorf("stdout: " + stdout.String())
+		log.Errorf("stderr: " + stderr.String())
 		debug.PrintStack()
 		return err
 	}