git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-22 17:31:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入自动安装依赖

Browse Source
This commit is contained in:
marvzhang
2020-04-19 16:56:07 +08:00
parent 468ee7bda0
commit b8713dfb67
6 changed files with 130 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
backend/entity/system.go
View File

@@ -22,6 +22,8 @@ type Lang struct {
LockPath string `json:"lock_path"`
InstallScript string `json:"install_script"`
InstallStatus string `json:"install_status"`
DepFileName string `json:"dep_file_name"`
InstallDepArgs string `json:"install_dep_cmd"`
}
type Dependency struct {
@@ -30,3 +32,7 @@ type Dependency struct {
Description string `json:"description"`
Installed bool `json:"installed"`
}
type PackageJson struct {
Dependencies map[string]string `json:"dependencies"`
}

3
backend/services/spider.go
View File

@@ -218,6 +218,9 @@ func PublishSpider(spider model.Spider) {
Spider: spider,
}
// 安装依赖
go spiderSync.InstallDeps()
//目录不存在，则直接下载
path := filepath.Join(viper.GetString("spider.path"), spider.Name)
if !utils.Exists(path) {

55
backend/services/spider_handler/spider.go
View File

@@ -16,6 +16,8 @@ import (
"path"
"path/filepath"
"runtime/debug"
"strings"
"sync"
)
const (
@@ -184,6 +186,57 @@ func (s *SpiderSync) Download() {
_ = database.RedisClient.HDel("spider", key)
}
// locks for dependency installation
var installLockMap sync.Map
// install dependencies
func (s *SpiderSync) InstallDeps() {
//s.Spider.Src
langs := utils.GetLangList()
for _, l := range langs {
// no dep file name is found, skip
if l.DepFileName == "" {
continue
}
// being locked, i.e. installation is running, skip
key := s.Spider.Name + "|" + l.Name
_, locked := installLockMap.Load(key)
if locked {
continue
}
// no dep file found, skip
if !utils.Exists(path.Join(s.Spider.Src, l.DepFileName)) {
continue
}
// lock
installLockMap.Store(key, true)
// command to install dependencies
cmd := exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs, " ")...)
// working directory
cmd.Dir = s.Spider.Src
// compatibility with node.js
if l.ExecutableName == constants.Nodejs {
deps, err := utils.GetPackageJsonDeps(path.Join(s.Spider.Src, l.DepFileName))
if err != nil {
continue
}
cmd = exec.Command(l.DepExecutablePath, strings.Split(l.InstallDepArgs+" "+strings.Join(deps, " "), " ")...)
}
// start executing command
output, err := cmd.Output()
if err != nil {
log.Errorf("install dep error: " + err.Error())
log.Errorf(string(output))
debug.PrintStack()
}
// unlock
installLockMap.Delete(key)
}
}

33
backend/utils/system.go
View File

@@ -1,6 +1,12 @@
package utils
import "crawlab/entity"
import (
"crawlab/entity"
"encoding/json"
"github.com/apex/log"
"io/ioutil"
"runtime/debug"
)
func GetLangList() []entity.Lang {
list := []entity.Lang{
@@ -10,6 +16,8 @@ func GetLangList() []entity.Lang {
ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"},
DepExecutablePath: "/usr/local/bin/pip",
LockPath: "/tmp/install-python.lock",
DepFileName: "requirements.txt",
InstallDepArgs: "install -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements.txt",
},
{
Name: "Node.js",
@@ -18,6 +26,8 @@ func GetLangList() []entity.Lang {
DepExecutablePath: "/usr/local/bin/npm",
LockPath: "/tmp/install-nodejs.lock",
InstallScript: "install-nodejs.sh",
DepFileName: "package.json",
InstallDepArgs: "install -g --registry=https://registry.npm.taobao.org",
},
{
Name: "Java",
@@ -60,3 +70,24 @@ func GetLangFromLangNamePlain(name string) entity.Lang {
}
return entity.Lang{}
}
func GetPackageJsonDeps(filepath string) (deps []string, err error) {
data, err := ioutil.ReadFile(filepath)
if err != nil {
log.Errorf("get package.json deps error: " + err.Error())
debug.PrintStack()
return deps, err
}
var packageJson entity.PackageJson
if err := json.Unmarshal(data, &packageJson); err != nil {
log.Errorf("get package.json deps error: " + err.Error())
debug.PrintStack()
return deps, err
}
for d, v := range packageJson.Dependencies {
deps = append(deps, d+"@"+v)
}
return deps, nil
}

18
frontend/src/i18n/zh.js
View File

@@ -490,6 +490,8 @@ export default {
'By default: ': '默认: ',
'Max Error Logs Display': '最大异常日志展示',
'Log Errors': '日志错误',
'No Expire': '不过期',
'Log Expire Duration': '日志过期时间',
// 挑战
'Challenge': '挑战',
@@ -499,6 +501,22 @@ export default {
'Not Achieved': '未达成',
'Start Challenge': '开始挑战',
// 时间
'Second': '秒',
'Seconds': '秒',
'Minute': '分',
'Minutes': '分',
'Hour': '小时',
'Hours': '小时',
'Day': '天',
'Days': '天',
'Week': '周',
'Weeks': '周',
'Month': '月',
'Months': '月',
'Year': '年',
'Years': '年',
// 全局
'Related Documentation': '相关文档',
'Click to view related Documentation': '点击查看相关文档',

17
frontend/src/views/setting/Setting.vue
View File

@@ -141,6 +141,23 @@
<el-option :value="10000" label="10000"/>
</el-select>
</el-form-item>
<el-form-item :label="$t('Log Expire Duration')" prop="setting.log_expire_duration">
<el-select
v-model="userInfo.setting.log_expire_duration"
clearable
>
<el-option :value="0" :label="$t('No Expire')"/>
<el-option :value="3600" :label="'1 ' + $t('Hour')"/>
<el-option :value="3600 * 6" :label="'6 ' + $t('Hours')"/>
<el-option :value="3600 * 12" :label="'12 ' + $t('Hours')"/>
<el-option :value="3600 * 24" :label="'1 ' + $t('Day')"/>
<el-option :value="3600 * 24 * 7" :label="'7 ' + $t('Days')"/>
<el-option :value="3600 * 24 * 14" :label="'14 ' + $t('Days')"/>
<el-option :value="3600 * 24 * 30" :label="'30 ' + $t('Days')"/>
<el-option :value="3600 * 24 * 30 * 3" :label="'90 ' + $t('Days')"/>
<el-option :value="3600 * 24 * 30 * 6" :label="'180 ' + $t('Days')"/>
</el-select>
</el-form-item>
<el-form-item>
<div style="text-align: right">
<el-button type="success" size="small" @click="saveUserInfo">