git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: optimized dependencies for spider

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-15 18:40:06 +08:00
parent a4b62ca221
commit 7c15c36201
3 changed files with 49 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/constants/dependency.go
View File

@@ -15,6 +15,6 @@ const (
)
const (
DependencyConfigRequirementsTxt = "requirements.txt"
DependencyConfigPackageJson = "package.json"
DependencyFileTypeRequirementsTxt = "requirements.txt"
DependencyFileTypePackageJson = "package.json"
)

34
core/controllers/spider.go
View File

@@ -544,7 +544,7 @@ func DeleteSpiderList(c *gin.Context) {
}
func GetSpiderListDir(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -553,7 +553,7 @@ func GetSpiderListDir(c *gin.Context) {
}
func GetSpiderFile(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -562,7 +562,7 @@ func GetSpiderFile(c *gin.Context) {
}
func GetSpiderFileInfo(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -571,7 +571,7 @@ func GetSpiderFileInfo(c *gin.Context) {
}
func PostSpiderSaveFile(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -580,7 +580,7 @@ func PostSpiderSaveFile(c *gin.Context) {
}
func PostSpiderSaveFiles(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -590,7 +590,7 @@ func PostSpiderSaveFiles(c *gin.Context) {
}
func PostSpiderSaveDir(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -599,7 +599,7 @@ func PostSpiderSaveDir(c *gin.Context) {
}
func PostSpiderRenameFile(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -608,7 +608,7 @@ func PostSpiderRenameFile(c *gin.Context) {
}
func DeleteSpiderFile(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -617,7 +617,7 @@ func DeleteSpiderFile(c *gin.Context) {
}
func PostSpiderCopyFile(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -626,7 +626,7 @@ func PostSpiderCopyFile(c *gin.Context) {
}
func PostSpiderExport(c *gin.Context) {
rootPath, err := getSpiderRootPath(c)
rootPath, err := getSpiderRootPathByContext(c)
if err != nil {
HandleErrorForbidden(c, err)
return
@@ -721,7 +721,7 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsService, err er
return getSpiderFsSvc(s)
}
func getSpiderRootPath(c *gin.Context) (rootPath string, err error) {
func getSpiderRootPathByContext(c *gin.Context) (rootPath string, err error) {
// spider id
id, err := primitive.ObjectIDFromHex(c.Param("id"))
if err != nil {
@@ -734,15 +734,5 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) {
return "", err
}
// check git permission
if !utils.IsPro() && !s.GitId.IsZero() {
return "", errors.New("git is not allowed in the community version")
}
// if git id is zero, return spider id as root path
if s.GitId.IsZero() {
return id.Hex(), nil
}
return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil
return utils.GetSpiderRootPath(s)
}

35
core/utils/spider.go Normal file
View File

@@ -0,0 +1,35 @@
package utils
import (
"errors"
"github.com/crawlab-team/crawlab/core/models/models"
"github.com/spf13/viper"
"path/filepath"
)
func GetSpiderRootPath(s *models.Spider) (rootPath string, err error) {
// check git permission
if !IsPro() && !s.GitId.IsZero() {
return "", errors.New("git is not allowed in the community version")
}
// if git id is zero, return spider id as root path
if s.GitId.IsZero() {
return s.Id.Hex(), nil
}
return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil
}
func GetSpiderFullRootPath(s *models.Spider) (rootPath string, err error) {
// workspace path
workspacePath := viper.GetString("workspace")
// get spider root path
rootPath, err = GetSpiderRootPath(s)
if err != nil {
return "", err
}
return filepath.Join(workspacePath, rootPath), nil
}