diff --git a/backend/entity/system.go b/backend/entity/system.go
index 2738b55a..bb95216d 100644
--- a/backend/entity/system.go
+++ b/backend/entity/system.go
@@ -22,6 +22,8 @@ type Lang struct {
LockPath string `json:"lock_path"`
InstallScript string `json:"install_script"`
InstallStatus string `json:"install_status"`
+ DepFileName string `json:"dep_file_name"`
+ InstallDepArgs string `json:"install_dep_cmd"`
}
type Dependency struct {
@@ -30,3 +32,7 @@ type Dependency struct {
Description string `json:"description"`
Installed bool `json:"installed"`
}
+
+type PackageJson struct {
+ Dependencies map[string]string `json:"dependencies"`
+}
diff --git a/backend/services/spider.go b/backend/services/spider.go
index 2a8a3d48..7d14c287 100644
--- a/backend/services/spider.go
+++ b/backend/services/spider.go
@@ -218,6 +218,9 @@ func PublishSpider(spider model.Spider) {
Spider: spider,
}
+ // 安装依赖
+ go spiderSync.InstallDeps()
+
//目录不存在，则直接下载
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
if !utils.Exists(path) {
diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go
index f8af323b..5c1adf3a 100644
--- a/backend/services/spider_handler/spider.go
+++ b/backend/services/spider_handler/spider.go
@@ -16,6 +16,8 @@ import (
"path"
"path/filepath"
"runtime/debug"
+ "strings"
+ "sync"
)
const (
@@ -184,6 +186,57 @@ func (s *SpiderSync) Download() {
_ = database.RedisClient.HDel("spider", key)
}
+// locks for dependency installation
+var installLockMap sync.Map
+
+// install dependencies
func (s *SpiderSync) InstallDeps() {
- //s.Spider.Src
+ langs := utils.GetLangList()
+ for _, l := range langs {
+ // no dep file name is found, skip
+ if l.DepFileName == "" {
+ continue
+ }
+
+ // being locked, i.e. installation is running, skip
+ key := s.Spider.Name + "|" + l.Name
+ _, locked := installLockMap.Load(key)
+ if locked {
+ continue
+ }
+
+ // no dep file found, skip
+ if !utils.Exists(path.Join(s.Spider.Src, l.DepFileName)) {
+ continue
+ }
+
+ // lock
+ installLockMap.Store(key, true)
+
+ // command to install dependencies
+ cmd := exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs, " ")...)
+
+ // working directory
+ cmd.Dir = s.Spider.Src
+
+ // compatibility with node.js
+ if l.ExecutableName == constants.Nodejs {
+ deps, err := utils.GetPackageJsonDeps(path.Join(s.Spider.Src, l.DepFileName))
+ if err != nil {
+ continue
+ }
+ cmd = exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs+" "+strings.Join(deps, " "), " ")...)
+ }
+
+ // start executing command
+ output, err := cmd.Output()
+ if err != nil {
+ log.Errorf("install dep error: " + err.Error())
+ log.Errorf(string(output))
+ debug.PrintStack()
+ }
+
+ // unlock
+ installLockMap.Delete(key)
+ }
}
diff --git a/backend/utils/system.go b/backend/utils/system.go
index f8f917be..5721aadf 100644
--- a/backend/utils/system.go
+++ b/backend/utils/system.go
@@ -1,6 +1,12 @@
package utils
-import "crawlab/entity"
+import (
+ "crawlab/entity"
+ "encoding/json"
+ "github.com/apex/log"
+ "io/ioutil"
+ "runtime/debug"
+)
func GetLangList() []entity.Lang {
list := []entity.Lang{
@@ -10,6 +16,8 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"},
DepExecutablePath: "/usr/local/bin/pip",
LockPath: "/tmp/install-python.lock",
+ DepFileName: "requirements.txt",
+ InstallDepArgs: "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt",
},
{
Name: "Node.js",
@@ -18,6 +26,8 @@ func GetLangList() []entity.Lang {
DepExecutablePath: "/usr/local/bin/npm",
LockPath: "/tmp/install-nodejs.lock",
InstallScript: "install-nodejs.sh",
+ DepFileName: "package.json",
+ InstallDepArgs: "install -g --registry=https://registry.npm.taobao.org",
},
{
Name: "Java",
@@ -60,3 +70,24 @@ func GetLangFromLangNamePlain(name string) entity.Lang {
}
return entity.Lang{}
}
+
+func GetPackageJsonDeps(filepath string) (deps []string, err error) {
+ data, err := ioutil.ReadFile(filepath)
+ if err != nil {
+ log.Errorf("get package.json deps error: " + err.Error())
+ debug.PrintStack()
+ return deps, err
+ }
+ var packageJson entity.PackageJson
+ if err := json.Unmarshal(data, &packageJson); err != nil {
+ log.Errorf("get package.json deps error: " + err.Error())
+ debug.PrintStack()
+ return deps, err
+ }
+
+ for d, v := range packageJson.Dependencies {
+ deps = append(deps, d+"@"+v)
+ }
+
+ return deps, nil
+}
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 08628202..d9a82d01 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -490,6 +490,8 @@ export default {
'By default: ': '默认: ',
'Max Error Logs Display': '最大异常日志展示',
'Log Errors': '日志错误',
+ 'No Expire': '不过期',
+ 'Log Expire Duration': '日志过期时间',
// 挑战
'Challenge': '挑战',
@@ -499,6 +501,22 @@ export default {
'Not Achieved': '未达成',
'Start Challenge': '开始挑战',
+ // 时间
+ 'Second': '秒',
+ 'Seconds': '秒',
+ 'Minute': '分',
+ 'Minutes': '分',
+ 'Hour': '小时',
+ 'Hours': '小时',
+ 'Day': '天',
+ 'Days': '天',
+ 'Week': '周',
+ 'Weeks': '周',
+ 'Month': '月',
+ 'Months': '月',
+ 'Year': '年',
+ 'Years': '年',
+
// 全局
'Related Documentation': '相关文档',
'Click to view related Documentation': '点击查看相关文档',
diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue
index 3f1f3bc9..eb1ed344 100644
--- a/frontend/src/views/setting/Setting.vue
+++ b/frontend/src/views/setting/Setting.vue
@@ -141,6 +141,23 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+