git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入储存逻辑

Browse Source
This commit is contained in:
marvzhang
2019-11-25 17:52:19 +08:00
parent 1c32c92c54
commit dccfb23901
2 changed files with 17 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/template/scrapy/config_spider/pipelines.py
View File

@@ -9,14 +9,14 @@ import os
from pymongo import MongoClient
mongo = MongoClient(
host=os.environ.get('CRAWLAB_MONGO_HOST'),
host=os.environ.get('CRAWLAB_MONGO_HOST') or 'localhost',
port=int(os.environ.get('CRAWLAB_MONGO_PORT') or 27017),
username=os.environ.get('CRAWLAB_MONGO_USERNAME'),
password=os.environ.get('CRAWLAB_MONGO_PASSWORD'),
authSource=os.environ.get('CRAWLAB_MONGO_AUTHSOURCE')
authSource=os.environ.get('CRAWLAB_MONGO_AUTHSOURCE') or 'admin'
)
db = mongo[os.environ.get('CRAWLAB_MONGO_DB')]
col_name = os.environ.get('CRAWLAB_COLLECTION')
db = mongo[os.environ.get('CRAWLAB_MONGO_DB') or 'test']
col_name = os.environ.get('CRAWLAB_COLLECTION') or 'test'
task_id = os.environ.get('CRAWLAB_TASK_ID')
class ConfigSpiderPipeline(object):

15
backend/utils/file.go
View File

@@ -199,8 +199,7 @@ func Compress(files []*os.File, dest string) error {
w := zip.NewWriter(d)
defer Close(w)
for _, file := range files {
err := _Compress(file, "", w)
if err != nil {
if err := _Compress(file, "", w); err != nil {
return err
}
}
@@ -255,6 +254,18 @@ func _Compress(file *os.File, prefix string, zw *zip.Writer) error {
}
func GetFilesFromDir(dirPath string) ([]*os.File, error) {
var res []*os.File
for _, fInfo := range ListDir(dirPath) {
f, err := os.Open(filepath.Join(dirPath, fInfo.Name()))
if err != nil {
return res, err
}
res = append(res, f)
}
return res, nil
}
func GetAllFilesFromDir(dirPath string) ([]*os.File, error) {
var res []*os.File
if err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if !IsDir(path) {