mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
384 lines
7.6 KiB
Go
384 lines
7.6 KiB
Go
package utils
|
||
|
||
import (
|
||
"archive/zip"
|
||
"crypto/md5"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"github.com/apex/log"
|
||
"github.com/crawlab-team/crawlab/core/constants"
|
||
"github.com/crawlab-team/crawlab/core/entity"
|
||
"io"
|
||
"io/fs"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"runtime/debug"
|
||
)
|
||
|
||
func OpenFile(fileName string) *os.File {
|
||
file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, os.ModePerm)
|
||
if err != nil {
|
||
log.Errorf("create file error: %s, file_name: %s", err.Error(), fileName)
|
||
debug.PrintStack()
|
||
return nil
|
||
}
|
||
return file
|
||
}
|
||
|
||
func Exists(path string) bool {
|
||
_, err := os.Stat(path) //os.Stat获取文件信息
|
||
if err != nil {
|
||
return os.IsExist(err)
|
||
}
|
||
return true
|
||
}
|
||
|
||
func IsDir(path string) bool {
|
||
s, err := os.Stat(path)
|
||
if err != nil {
|
||
return false
|
||
}
|
||
return s.IsDir()
|
||
}
|
||
|
||
// ListDir Add: 增加error类型作为第二返回值
|
||
// 在其他函数如 /task/log/file_driver.go中的 *FileLogDriver.cleanup()函数调用时
|
||
// 可以通过判断err是否为nil来判断是否有错误发生
|
||
func ListDir(path string) ([]fs.FileInfo, error) {
|
||
list, err := os.ReadDir(path)
|
||
if err != nil {
|
||
log.Errorf(err.Error())
|
||
debug.PrintStack()
|
||
return nil, err
|
||
}
|
||
|
||
var res []fs.FileInfo
|
||
for _, item := range list {
|
||
info, err := item.Info()
|
||
if err != nil {
|
||
log.Errorf(err.Error())
|
||
debug.PrintStack()
|
||
return nil, err
|
||
}
|
||
res = append(res, info)
|
||
}
|
||
return res, nil
|
||
}
|
||
|
||
func DeCompress(srcFile *os.File, dstPath string) error {
|
||
// 如果保存路径不存在，创建一个
|
||
if !Exists(dstPath) {
|
||
if err := os.MkdirAll(dstPath, os.ModePerm); err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
}
|
||
|
||
// 读取zip文件
|
||
zipFile, err := zip.OpenReader(srcFile.Name())
|
||
if err != nil {
|
||
log.Errorf("Unzip File Error：" + err.Error())
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
defer Close(zipFile)
|
||
|
||
// 遍历zip内所有文件和目录
|
||
for _, innerFile := range zipFile.File {
|
||
// 获取该文件数据
|
||
info := innerFile.FileInfo()
|
||
|
||
// 如果是目录，则创建一个
|
||
if info.IsDir() {
|
||
err = os.MkdirAll(filepath.Join(dstPath, innerFile.Name), os.ModeDir|os.ModePerm)
|
||
if err != nil {
|
||
log.Errorf("Unzip File Error : " + err.Error())
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
|
||
// 如果文件目录不存在，则创建一个
|
||
dirPath := filepath.Join(dstPath, filepath.Dir(innerFile.Name))
|
||
if !Exists(dirPath) {
|
||
if err = os.MkdirAll(dirPath, os.ModeDir|os.ModePerm); err != nil {
|
||
log.Errorf("Unzip File Error : " + err.Error())
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
}
|
||
|
||
// 打开该文件
|
||
srcFile, err := innerFile.Open()
|
||
if err != nil {
|
||
log.Errorf("Unzip File Error : " + err.Error())
|
||
debug.PrintStack()
|
||
continue
|
||
}
|
||
|
||
// 创建新文件
|
||
newFilePath := filepath.Join(dstPath, innerFile.Name)
|
||
newFile, err := os.OpenFile(newFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode())
|
||
if err != nil {
|
||
log.Errorf("Unzip File Error : " + err.Error())
|
||
debug.PrintStack()
|
||
continue
|
||
}
|
||
|
||
// 拷贝该文件到新文件中
|
||
if _, err := io.Copy(newFile, srcFile); err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
|
||
// 关闭该文件
|
||
if err := srcFile.Close(); err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
|
||
// 关闭新文件
|
||
if err := newFile.Close(); err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Compress 压缩文件
|
||
// files 文件数组，可以是不同dir下的文件或者文件夹
|
||
// dest 压缩文件存放地址
|
||
func Compress(files []*os.File, dest string) error {
|
||
d, _ := os.Create(dest)
|
||
defer Close(d)
|
||
w := zip.NewWriter(d)
|
||
defer Close(w)
|
||
for _, file := range files {
|
||
if err := _Compress(file, "", w); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func _Compress(file *os.File, prefix string, zw *zip.Writer) error {
|
||
info, err := file.Stat()
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
if info.IsDir() {
|
||
prefix = prefix + "/" + info.Name()
|
||
fileInfos, err := file.Readdir(-1)
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
for _, fi := range fileInfos {
|
||
f, err := os.Open(file.Name() + "/" + fi.Name())
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
err = _Compress(f, prefix, zw)
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
}
|
||
} else {
|
||
header, err := zip.FileInfoHeader(info)
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
header.Name = prefix + "/" + header.Name
|
||
writer, err := zw.CreateHeader(header)
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
_, err = io.Copy(writer, file)
|
||
Close(file)
|
||
if err != nil {
|
||
debug.PrintStack()
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func TrimFileData(data []byte) (res []byte) {
|
||
if string(data) == constants.EmptyFileData {
|
||
return res
|
||
}
|
||
return data
|
||
}
|
||
|
||
func ZipDirectory(dir, zipfile string) error {
|
||
zipFile, err := os.Create(zipfile)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer zipFile.Close()
|
||
|
||
zipWriter := zip.NewWriter(zipFile)
|
||
defer zipWriter.Close()
|
||
|
||
baseDir := filepath.Dir(dir)
|
||
|
||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
if info.IsDir() {
|
||
return nil
|
||
}
|
||
|
||
relPath, err := filepath.Rel(baseDir, path)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
zipFile, err := zipWriter.Create(relPath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
file, err := os.Open(path)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
|
||
_, err = io.Copy(zipFile, file)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
return nil
|
||
})
|
||
|
||
return err
|
||
}
|
||
|
||
// CopyFile File copies a single file from src to dst
|
||
func CopyFile(src, dst string) error {
|
||
var err error
|
||
var srcFd *os.File
|
||
var dstFd *os.File
|
||
var srcInfo os.FileInfo
|
||
|
||
if srcFd, err = os.Open(src); err != nil {
|
||
return err
|
||
}
|
||
defer srcFd.Close()
|
||
|
||
if dstFd, err = os.Create(dst); err != nil {
|
||
return err
|
||
}
|
||
defer dstFd.Close()
|
||
|
||
if _, err = io.Copy(dstFd, srcFd); err != nil {
|
||
return err
|
||
}
|
||
if srcInfo, err = os.Stat(src); err != nil {
|
||
return err
|
||
}
|
||
return os.Chmod(dst, srcInfo.Mode())
|
||
}
|
||
|
||
// CopyDir Dir copies a whole directory recursively
|
||
func CopyDir(src string, dst string) error {
|
||
var err error
|
||
var fds []os.DirEntry
|
||
var srcInfo os.FileInfo
|
||
|
||
if srcInfo, err = os.Stat(src); err != nil {
|
||
return err
|
||
}
|
||
|
||
if err = os.MkdirAll(dst, srcInfo.Mode()); err != nil {
|
||
return err
|
||
}
|
||
|
||
if fds, err = os.ReadDir(src); err != nil {
|
||
return err
|
||
}
|
||
for _, fd := range fds {
|
||
srcfp := path.Join(src, fd.Name())
|
||
dstfp := path.Join(dst, fd.Name())
|
||
|
||
if fd.IsDir() {
|
||
if err = CopyDir(srcfp, dstfp); err != nil {
|
||
fmt.Println(err)
|
||
}
|
||
} else {
|
||
if err = CopyFile(srcfp, dstfp); err != nil {
|
||
fmt.Println(err)
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func GetFileHash(filePath string) (res string, err error) {
|
||
file, err := os.Open(filePath)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
defer file.Close()
|
||
|
||
hash := md5.New()
|
||
if _, err := io.Copy(hash, file); err != nil {
|
||
return "", err
|
||
}
|
||
|
||
return hex.EncodeToString(hash.Sum(nil)), nil
|
||
}
|
||
|
||
func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) {
|
||
files := make(map[string]entity.FsFileInfo)
|
||
|
||
err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
var hash string
|
||
if !info.IsDir() {
|
||
hash, err = GetFileHash(path)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
relPath, err := filepath.Rel(dir, path)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
files[relPath] = entity.FsFileInfo{
|
||
Name: info.Name(),
|
||
Path: relPath,
|
||
FullPath: path,
|
||
Extension: filepath.Ext(path),
|
||
IsDir: info.IsDir(),
|
||
FileSize: info.Size(),
|
||
ModTime: info.ModTime(),
|
||
Mode: info.Mode(),
|
||
Hash: hash,
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
return files, nil
|
||
}
|