添加Scrapy设置后端

backend/services/scrapy.go

@@ -3,7 +3,10 @@ package services
 import (
 	"bytes"
 	"crawlab/model"
+	"encoding/json"
+	"github.com/apex/log"
 	"os/exec"
+	"runtime/debug"
 	"strings"
 )
 
@@ -16,6 +19,8 @@ func GetScrapySpiderNames(s model.Spider) ([]string, error) {
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 	if err := cmd.Run(); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
 		return []string{}, err
 	}
 
@@ -30,3 +35,28 @@ func GetScrapySpiderNames(s model.Spider) ([]string, error) {
 
 	return res, nil
 }
+
+func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error) {
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+
+	cmd := exec.Command("crawlab", "settings")
+	cmd.Dir = s.Src
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		log.Errorf(err.Error())
+		log.Errorf(stderr.String())
+		debug.PrintStack()
+		return res, err
+	}
+
+	log.Infof(stdout.String())
+	if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return res, err
+	}
+
+	return res, nil
+}