From 459d94ad968aed1cb986e471771dbc1464182956 Mon Sep 17 00:00:00 2001
From: Marvin Zhang <tikazyq@163.com>
Date: Fri, 15 Nov 2024 18:40:06 +0800
Subject: [PATCH] feat: optimized dependencies for spider

---
 core/constants/dependency.go |  4 ++--
 core/controllers/spider.go   | 34 ++++++++++++----------------------
 core/utils/spider.go         | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 49 insertions(+), 24 deletions(-)
 create mode 100644 core/utils/spider.go

diff --git a/core/constants/dependency.go b/core/constants/dependency.go
index b5820092..a4025c39 100644
--- a/core/constants/dependency.go
+++ b/core/constants/dependency.go
@@ -15,6 +15,6 @@ const (
 )
 
 const (
-	DependencyConfigRequirementsTxt = "requirements.txt"
-	DependencyConfigPackageJson     = "package.json"
+	DependencyFileTypeRequirementsTxt = "requirements.txt"
+	DependencyFileTypePackageJson     = "package.json"
 )
diff --git a/core/controllers/spider.go b/core/controllers/spider.go
index e93a26c7..51c7f43b 100644
--- a/core/controllers/spider.go
+++ b/core/controllers/spider.go
@@ -544,7 +544,7 @@ func DeleteSpiderList(c *gin.Context) {
 }
 
 func GetSpiderListDir(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -553,7 +553,7 @@ func GetSpiderListDir(c *gin.Context) {
 }
 
 func GetSpiderFile(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -562,7 +562,7 @@ func GetSpiderFile(c *gin.Context) {
 }
 
 func GetSpiderFileInfo(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -571,7 +571,7 @@ func GetSpiderFileInfo(c *gin.Context) {
 }
 
 func PostSpiderSaveFile(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -580,7 +580,7 @@ func PostSpiderSaveFile(c *gin.Context) {
 }
 
 func PostSpiderSaveFiles(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -590,7 +590,7 @@ func PostSpiderSaveFiles(c *gin.Context) {
 }
 
 func PostSpiderSaveDir(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -599,7 +599,7 @@ func PostSpiderSaveDir(c *gin.Context) {
 }
 
 func PostSpiderRenameFile(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -608,7 +608,7 @@ func PostSpiderRenameFile(c *gin.Context) {
 }
 
 func DeleteSpiderFile(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -617,7 +617,7 @@ func DeleteSpiderFile(c *gin.Context) {
 }
 
 func PostSpiderCopyFile(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -626,7 +626,7 @@ func PostSpiderCopyFile(c *gin.Context) {
 }
 
 func PostSpiderExport(c *gin.Context) {
-	rootPath, err := getSpiderRootPath(c)
+	rootPath, err := getSpiderRootPathByContext(c)
 	if err != nil {
 		HandleErrorForbidden(c, err)
 		return
@@ -721,7 +721,7 @@ func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsService, err er
 	return getSpiderFsSvc(s)
 }
 
-func getSpiderRootPath(c *gin.Context) (rootPath string, err error) {
+func getSpiderRootPathByContext(c *gin.Context) (rootPath string, err error) {
 	// spider id
 	id, err := primitive.ObjectIDFromHex(c.Param("id"))
 	if err != nil {
@@ -734,15 +734,5 @@ func getSpiderRootPath(c *gin.Context) (rootPath string, err error) {
 		return "", err
 	}
 
-	// check git permission
-	if !utils.IsPro() && !s.GitId.IsZero() {
-		return "", errors.New("git is not allowed in the community version")
-	}
-
-	// if git id is zero, return spider id as root path
-	if s.GitId.IsZero() {
-		return id.Hex(), nil
-	}
-
-	return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil
+	return utils.GetSpiderRootPath(s)
 }
diff --git a/core/utils/spider.go b/core/utils/spider.go
new file mode 100644
index 00000000..7cc4e00e
--- /dev/null
+++ b/core/utils/spider.go
@@ -0,0 +1,35 @@
+package utils
+
+import (
+	"errors"
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/spf13/viper"
+	"path/filepath"
+)
+
+func GetSpiderRootPath(s *models.Spider) (rootPath string, err error) {
+	// check git permission
+	if !IsPro() && !s.GitId.IsZero() {
+		return "", errors.New("git is not allowed in the community version")
+	}
+
+	// if git id is zero, return spider id as root path
+	if s.GitId.IsZero() {
+		return s.Id.Hex(), nil
+	}
+
+	return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil
+}
+
+func GetSpiderFullRootPath(s *models.Spider) (rootPath string, err error) {
+	// workspace path
+	workspacePath := viper.GetString("workspace")
+
+	// get spider root path
+	rootPath, err = GetSpiderRootPath(s)
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(workspacePath, rootPath), nil
+}