From b8713dfb675d072177af6e1cad05af2463cd98fd Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sun, 19 Apr 2020 16:56:07 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E8=87=AA=E5=8A=A8=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/entity/system.go | 6 +++ backend/services/spider.go | 3 ++ backend/services/spider_handler/spider.go | 55 ++++++++++++++++++++++- backend/utils/system.go | 33 +++++++++++++- frontend/src/i18n/zh.js | 18 ++++++++ frontend/src/views/setting/Setting.vue | 17 +++++++ 6 files changed, 130 insertions(+), 2 deletions(-) diff --git a/backend/entity/system.go b/backend/entity/system.go index 2738b55a..bb95216d 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -22,6 +22,8 @@ type Lang struct { LockPath string `json:"lock_path"` InstallScript string `json:"install_script"` InstallStatus string `json:"install_status"` + DepFileName string `json:"dep_file_name"` + InstallDepArgs string `json:"install_dep_cmd"` } type Dependency struct { @@ -30,3 +32,7 @@ type Dependency struct { Description string `json:"description"` Installed bool `json:"installed"` } + +type PackageJson struct { + Dependencies map[string]string `json:"dependencies"` +} diff --git a/backend/services/spider.go b/backend/services/spider.go index 2a8a3d48..7d14c287 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -218,6 +218,9 @@ func PublishSpider(spider model.Spider) { Spider: spider, } + // 安装依赖 + go spiderSync.InstallDeps() + //目录不存在，则直接下载 path := filepath.Join(viper.GetString("spider.path"), spider.Name) if !utils.Exists(path) { diff --git a/backend/services/spider_handler/spider.go b/backend/services/spider_handler/spider.go index f8af323b..5c1adf3a 100644 --- a/backend/services/spider_handler/spider.go +++ b/backend/services/spider_handler/spider.go @@ -16,6 +16,8 @@ import ( "path" "path/filepath" "runtime/debug" + "strings" + "sync" ) const ( @@ -184,6 +186,57 @@ func (s *SpiderSync) Download() { _ = database.RedisClient.HDel("spider", key) } +// locks for dependency installation +var installLockMap sync.Map + +// install dependencies func (s *SpiderSync) InstallDeps() { - //s.Spider.Src + langs := utils.GetLangList() + for _, l := range langs { + // no dep file name is found, skip + if l.DepFileName == "" { + continue + } + + // being locked, i.e. installation is running, skip + key := s.Spider.Name + "|" + l.Name + _, locked := installLockMap.Load(key) + if locked { + continue + } + + // no dep file found, skip + if !utils.Exists(path.Join(s.Spider.Src, l.DepFileName)) { + continue + } + + // lock + installLockMap.Store(key, true) + + // command to install dependencies + cmd := exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs, " ")...) + + // working directory + cmd.Dir = s.Spider.Src + + // compatibility with node.js + if l.ExecutableName == constants.Nodejs { + deps, err := utils.GetPackageJsonDeps(path.Join(s.Spider.Src, l.DepFileName)) + if err != nil { + continue + } + cmd = exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs+" "+strings.Join(deps, " "), " ")...) + } + + // start executing command + output, err := cmd.Output() + if err != nil { + log.Errorf("install dep error: " + err.Error()) + log.Errorf(string(output)) + debug.PrintStack() + } + + // unlock + installLockMap.Delete(key) + } } diff --git a/backend/utils/system.go b/backend/utils/system.go index f8f917be..5721aadf 100644 --- a/backend/utils/system.go +++ b/backend/utils/system.go @@ -1,6 +1,12 @@ package utils -import "crawlab/entity" +import ( + "crawlab/entity" + "encoding/json" + "github.com/apex/log" + "io/ioutil" + "runtime/debug" +) func GetLangList() []entity.Lang { list := []entity.Lang{ @@ -10,6 +16,8 @@ func GetLangList() []entity.Lang { ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip", LockPath: "/tmp/install-python.lock", + DepFileName: "requirements.txt", + InstallDepArgs: "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt", }, { Name: "Node.js", @@ -18,6 +26,8 @@ func GetLangList() []entity.Lang { DepExecutablePath: "/usr/local/bin/npm", LockPath: "/tmp/install-nodejs.lock", InstallScript: "install-nodejs.sh", + DepFileName: "package.json", + InstallDepArgs: "install -g --registry=https://registry.npm.taobao.org", }, { Name: "Java", @@ -60,3 +70,24 @@ func GetLangFromLangNamePlain(name string) entity.Lang { } return entity.Lang{} } + +func GetPackageJsonDeps(filepath string) (deps []string, err error) { + data, err := ioutil.ReadFile(filepath) + if err != nil { + log.Errorf("get package.json deps error: " + err.Error()) + debug.PrintStack() + return deps, err + } + var packageJson entity.PackageJson + if err := json.Unmarshal(data, &packageJson); err != nil { + log.Errorf("get package.json deps error: " + err.Error()) + debug.PrintStack() + return deps, err + } + + for d, v := range packageJson.Dependencies { + deps = append(deps, d+"@"+v) + } + + return deps, nil +} diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 08628202..d9a82d01 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -490,6 +490,8 @@ export default { 'By default: ': '默认: ', 'Max Error Logs Display': '最大异常日志展示', 'Log Errors': '日志错误', + 'No Expire': '不过期', + 'Log Expire Duration': '日志过期时间', // 挑战 'Challenge': '挑战', @@ -499,6 +501,22 @@ export default { 'Not Achieved': '未达成', 'Start Challenge': '开始挑战', + // 时间 + 'Second': '秒', + 'Seconds': '秒', + 'Minute': '分', + 'Minutes': '分', + 'Hour': '小时', + 'Hours': '小时', + 'Day': '天', + 'Days': '天', + 'Week': '周', + 'Weeks': '周', + 'Month': '月', + 'Months': '月', + 'Year': '年', + 'Years': '年', + // 全局 'Related Documentation': '相关文档', 'Click to view related Documentation': '点击查看相关文档', diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index 3f1f3bc9..eb1ed344 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -141,6 +141,23 @@ + + + + + + + + + + + + + +