mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
feat: added modules
This commit is contained in:
Marvin Zhang
24
core/fs/default.go
Normal file
24
core/fs/default.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"github.com/apex/log"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/spf13/viper"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rootDir, err := homedir.Dir()
|
||||
if err != nil {
|
||||
log.Warnf("cannot find home directory: %v", err)
|
||||
return
|
||||
}
|
||||
DefaultWorkspacePath = filepath.Join(rootDir, "crawlab_workspace")
|
||||
|
||||
workspacePath := viper.GetString("workspace")
|
||||
if workspacePath == "" {
|
||||
viper.Set("workspace", DefaultWorkspacePath)
|
||||
}
|
||||
}
|
||||
|
||||
var DefaultWorkspacePath string
|
||||
167
core/fs/service_v2.go
Normal file
167
core/fs/service_v2.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type ServiceV2 struct {
|
||||
// settings
|
||||
rootPath string
|
||||
skipNames []string
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) List(path string) (files []interfaces.FsFileInfo, err error) {
|
||||
// Normalize the provided path
|
||||
normPath := filepath.Clean(path)
|
||||
if normPath == "." {
|
||||
normPath = ""
|
||||
}
|
||||
fullPath := filepath.Join(svc.rootPath, normPath)
|
||||
|
||||
// Temporary map to hold directory information and their children
|
||||
dirMap := make(map[string]*entity.FsFileInfo)
|
||||
|
||||
// Use filepath.Walk to recursively traverse directories
|
||||
err = filepath.Walk(fullPath, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(svc.rootPath, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fi := &entity.FsFileInfo{
|
||||
Name: info.Name(),
|
||||
Path: filepath.ToSlash(relPath),
|
||||
FullPath: p,
|
||||
Extension: filepath.Ext(p),
|
||||
IsDir: info.IsDir(),
|
||||
FileSize: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
Mode: info.Mode(),
|
||||
Children: nil,
|
||||
}
|
||||
|
||||
// Skip files/folders matching the pattern
|
||||
for _, name := range svc.skipNames {
|
||||
if fi.Name == name {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
dirMap[p] = fi
|
||||
}
|
||||
|
||||
if parentDir := filepath.Dir(p); parentDir != p && dirMap[parentDir] != nil {
|
||||
dirMap[parentDir].Children = append(dirMap[parentDir].Children, fi)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if rootInfo, ok := dirMap[fullPath]; ok {
|
||||
for _, info := range rootInfo.GetChildren() {
|
||||
files = append(files, info)
|
||||
}
|
||||
}
|
||||
|
||||
return files, err
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetFile(path string) (data []byte, err error) {
|
||||
return os.ReadFile(filepath.Join(svc.rootPath, path))
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetFileInfo(path string) (file interfaces.FsFileInfo, err error) {
|
||||
f, err := os.Stat(filepath.Join(svc.rootPath, path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &entity.FsFileInfo{
|
||||
Name: f.Name(),
|
||||
Path: path,
|
||||
FullPath: filepath.Join(svc.rootPath, path),
|
||||
Extension: filepath.Ext(path),
|
||||
IsDir: f.IsDir(),
|
||||
FileSize: f.Size(),
|
||||
ModTime: f.ModTime(),
|
||||
Mode: f.Mode(),
|
||||
Children: nil,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Save(path string, data []byte) (err error) {
|
||||
// Create directories if not exist
|
||||
dir := filepath.Dir(filepath.Join(svc.rootPath, path))
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Write file
|
||||
return os.WriteFile(filepath.Join(svc.rootPath, path), data, 0644)
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) CreateDir(path string) (err error) {
|
||||
return os.MkdirAll(filepath.Join(svc.rootPath, path), 0755)
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Rename(path, newPath string) (err error) {
|
||||
oldPath := filepath.Join(svc.rootPath, path)
|
||||
newFullPath := filepath.Join(svc.rootPath, newPath)
|
||||
return os.Rename(oldPath, newFullPath)
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Delete(path string) (err error) {
|
||||
fullPath := filepath.Join(svc.rootPath, path)
|
||||
return os.RemoveAll(fullPath)
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Copy(path, newPath string) (err error) {
|
||||
srcPath := filepath.Join(svc.rootPath, path)
|
||||
destPath := filepath.Join(svc.rootPath, newPath)
|
||||
|
||||
// Get source info
|
||||
srcInfo, err := os.Stat(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If source is file, copy it
|
||||
if !srcInfo.IsDir() {
|
||||
srcFile, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer srcFile.Close()
|
||||
|
||||
destFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
_, err = io.Copy(destFile, srcFile)
|
||||
|
||||
return err
|
||||
} else {
|
||||
// If source is directory, copy it recursively
|
||||
return utils.CopyDir(srcPath, destPath)
|
||||
}
|
||||
}
|
||||
|
||||
func NewFsServiceV2(path string) (svc interfaces.FsServiceV2) {
|
||||
return &ServiceV2{
|
||||
rootPath: path,
|
||||
skipNames: []string{".git"},
|
||||
}
|
||||
}
|
||||
238
core/fs/service_v2_test.go
Normal file
238
core/fs/service_v2_test.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestServiceV2_List(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
testDir := filepath.Join(rootDir, "dir")
|
||||
os.Mkdir(testDir, 0755)
|
||||
ioutil.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644)
|
||||
ioutil.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644)
|
||||
subDir := filepath.Join(testDir, "subdir")
|
||||
os.Mkdir(subDir, 0755)
|
||||
ioutil.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644)
|
||||
os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
files, err := svc.List("dir")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to list files: %v", err)
|
||||
}
|
||||
|
||||
// Assert correct number of items
|
||||
assert.Len(t, files, 4)
|
||||
// Use a map to verify presence and characteristics of files/directories to avoid order issues
|
||||
items := make(map[string]bool)
|
||||
for _, item := range files {
|
||||
items[item.GetName()] = item.GetIsDir()
|
||||
}
|
||||
|
||||
_, file1Exists := items["file1.txt"]
|
||||
_, file2Exists := items["file2.txt"]
|
||||
_, subdirExists := items["subdir"]
|
||||
_, emptyExists := items["empty"]
|
||||
|
||||
assert.True(t, file1Exists)
|
||||
assert.True(t, file2Exists)
|
||||
assert.True(t, subdirExists)
|
||||
assert.True(t, emptyExists) // Verify that the empty directory is included
|
||||
|
||||
if subdirExists && len(files[2].GetChildren()) > 0 {
|
||||
assert.Equal(t, "file3.txt", files[2].GetChildren()[0].GetName())
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceV2_GetFile(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
expectedContent := []byte("hello world")
|
||||
ioutil.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644)
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
content, err := svc.GetFile("file.txt")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to get file: %v", err)
|
||||
}
|
||||
assert.Equal(t, expectedContent, content)
|
||||
}
|
||||
|
||||
func TestServiceV2_Delete(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
filePath := filepath.Join(rootDir, "file.txt")
|
||||
ioutil.WriteFile(filePath, []byte("hello world"), 0644)
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Delete the file
|
||||
err = svc.Delete("file.txt")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to delete file: %v", err)
|
||||
}
|
||||
|
||||
// Verify deletion
|
||||
_, err = os.Stat(filePath)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
func TestServiceV2_CreateDir(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Create a new directory
|
||||
err = svc.CreateDir("newDir")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to create directory: %v", err)
|
||||
}
|
||||
|
||||
// Verify the directory was created
|
||||
_, err = os.Stat(filepath.Join(rootDir, "newDir"))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestServiceV2_Save(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Save a new file
|
||||
err = svc.Save("newFile.txt", []byte("Hello, world!"))
|
||||
if err != nil {
|
||||
t.Errorf("Failed to save file: %v", err)
|
||||
}
|
||||
|
||||
// Verify the file was saved
|
||||
data, err := ioutil.ReadFile(filepath.Join(rootDir, "newFile.txt"))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Hello, world!", string(data))
|
||||
}
|
||||
|
||||
func TestServiceV2_Rename(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Create a file to rename
|
||||
ioutil.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644)
|
||||
|
||||
// Rename the file
|
||||
err = svc.Rename("oldName.txt", "newName.txt")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to rename file: %v", err)
|
||||
}
|
||||
|
||||
// Verify the file was renamed
|
||||
_, err = os.Stat(filepath.Join(rootDir, "newName.txt"))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestServiceV2_RenameDir(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Create a directory to rename
|
||||
os.Mkdir(filepath.Join(rootDir, "oldName"), 0755)
|
||||
|
||||
// Rename the directory
|
||||
err = svc.Rename("oldName", "newName")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to rename directory: %v", err)
|
||||
}
|
||||
|
||||
// Verify the directory was renamed
|
||||
_, err = os.Stat(filepath.Join(rootDir, "newName"))
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestServiceV2_Copy(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Create a file to copy
|
||||
ioutil.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644)
|
||||
|
||||
// Copy the file
|
||||
err = svc.Copy("source.txt", "copy.txt")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to copy file: %v", err)
|
||||
}
|
||||
|
||||
// Verify the file was copied
|
||||
data, err := ioutil.ReadFile(filepath.Join(rootDir, "copy.txt"))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Hello, world!", string(data))
|
||||
}
|
||||
|
||||
func TestServiceV2_CopyDir(t *testing.T) {
|
||||
rootDir, err := ioutil.TempDir("", "fsTest")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootDir) // clean up
|
||||
|
||||
svc := NewFsServiceV2(rootDir)
|
||||
|
||||
// Create a directory to copy
|
||||
os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755)
|
||||
ioutil.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644)
|
||||
|
||||
// Copy the directory
|
||||
err = svc.Copy("sourceDir", "copyDir")
|
||||
if err != nil {
|
||||
t.Errorf("Failed to copy directory: %v", err)
|
||||
}
|
||||
|
||||
// Verify the directory was copied
|
||||
_, err = os.Stat(filepath.Join(rootDir, "copyDir"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify the file inside the directory was copied
|
||||
data, err := ioutil.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt"))
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "Hello, world!", string(data))
|
||||
}
|
||||
Reference in New Issue
Block a user