From dccfb23901d5cf91b4df8dd2bd6b7f87707f2609 Mon Sep 17 00:00:00 2001
From: marvzhang <tikazyq@163.com>
Date: Mon, 25 Nov 2019 17:52:19 +0800
Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E5=82=A8=E5=AD=98=E9=80=BB?=
 =?UTF-8?q?=E8=BE=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../template/scrapy/config_spider/pipelines.py    |  8 ++++----
 backend/utils/file.go                             | 15 +++++++++++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/backend/template/scrapy/config_spider/pipelines.py b/backend/template/scrapy/config_spider/pipelines.py
index 830ab0b0..7a3286ff 100644
--- a/backend/template/scrapy/config_spider/pipelines.py
+++ b/backend/template/scrapy/config_spider/pipelines.py
@@ -9,14 +9,14 @@ import os
 from pymongo import MongoClient
 
 mongo = MongoClient(
-    host=os.environ.get('CRAWLAB_MONGO_HOST'),
+    host=os.environ.get('CRAWLAB_MONGO_HOST') or 'localhost',
     port=int(os.environ.get('CRAWLAB_MONGO_PORT') or 27017),
     username=os.environ.get('CRAWLAB_MONGO_USERNAME'),
     password=os.environ.get('CRAWLAB_MONGO_PASSWORD'),
-    authSource=os.environ.get('CRAWLAB_MONGO_AUTHSOURCE')
+    authSource=os.environ.get('CRAWLAB_MONGO_AUTHSOURCE') or 'admin'
 )
-db = mongo[os.environ.get('CRAWLAB_MONGO_DB')]
-col_name = os.environ.get('CRAWLAB_COLLECTION')
+db = mongo[os.environ.get('CRAWLAB_MONGO_DB') or 'test']
+col_name = os.environ.get('CRAWLAB_COLLECTION') or 'test'
 task_id = os.environ.get('CRAWLAB_TASK_ID')
 
 class ConfigSpiderPipeline(object):
diff --git a/backend/utils/file.go b/backend/utils/file.go
index 5e461038..2dacc9ed 100644
--- a/backend/utils/file.go
+++ b/backend/utils/file.go
@@ -199,8 +199,7 @@ func Compress(files []*os.File, dest string) error {
 	w := zip.NewWriter(d)
 	defer Close(w)
 	for _, file := range files {
-		err := _Compress(file, "", w)
-		if err != nil {
+		if err := _Compress(file, "", w); err != nil {
 			return err
 		}
 	}
@@ -255,6 +254,18 @@ func _Compress(file *os.File, prefix string, zw *zip.Writer) error {
 }
 
 func GetFilesFromDir(dirPath string) ([]*os.File, error) {
+	var res []*os.File
+	for _, fInfo := range ListDir(dirPath) {
+		f, err := os.Open(filepath.Join(dirPath, fInfo.Name()))
+		if err != nil {
+			return res, err
+		}
+		res = append(res, f)
+	}
+	return res, nil
+}
+
+func GetAllFilesFromDir(dirPath string) ([]*os.File, error) {
 	var res []*os.File
 	if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
 		if !IsDir(path) {