diff --git a/backend/constants/rpc.go b/backend/constants/rpc.go index 6eebf0d5..a650bcda 100644 --- a/backend/constants/rpc.go +++ b/backend/constants/rpc.go @@ -6,4 +6,5 @@ const ( RpcUninstallDep = "uninstall_dep" RpcGetDepList = "get_dep_list" RpcGetInstalledDepList = "get_installed_dep_list" + RpcGetLang = "get_lang" ) diff --git a/backend/constants/system.go b/backend/constants/system.go index bec8b8c5..03491222 100644 --- a/backend/constants/system.go +++ b/backend/constants/system.go @@ -11,3 +11,10 @@ const ( Nodejs = "node" Java = "java" ) + +const ( + InstallStatusNotInstalled = "not-installed" + InstallStatusInstalling = "installing" + InstallStatusInstallingOther = "installing-other" + InstallStatusInstalled = "installed" +) diff --git a/backend/entity/node.go b/backend/entity/node.go index cf52fafb..ebe2047f 100644 --- a/backend/entity/node.go +++ b/backend/entity/node.go @@ -20,6 +20,9 @@ type NodeMessage struct { // 爬虫相关 SpiderId string `json:"spider_id"` //爬虫ID + // 语言相关 + Lang Lang `json:"lang"` + // 错误相关 Error string `json:"error"` } diff --git a/backend/entity/system.go b/backend/entity/system.go index 4c15cc54..8e174b0e 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -19,7 +19,8 @@ type Lang struct { ExecutableName string `json:"executable_name"` ExecutablePaths []string `json:"executable_paths"` DepExecutablePath string `json:"dep_executable_path"` - Installed bool `json:"installed"` + LockPath string `json:"lock_path"` + InstallStatus string `json:"install_status"` } type Dependency struct { diff --git a/backend/scripts/install-java.sh b/backend/scripts/install-java.sh index d928ae02..7cb8972c 100755 --- a/backend/scripts/install-java.sh +++ b/backend/scripts/install-java.sh @@ -1,5 +1,8 @@ #!/bin/bash +# lock global +touch /tmp/install.lock + # lock touch /tmp/install-java.lock @@ -11,3 +14,6 @@ ln -s /usr/bin/java /usr/local/bin/java # unlock rm /tmp/install-java.lock + +# unlock global +rm /tmp/install.lock diff --git a/backend/scripts/install-nodejs.sh b/backend/scripts/install-nodejs.sh index 8a054b69..cf01c7f8 100644 --- a/backend/scripts/install-nodejs.sh +++ b/backend/scripts/install-nodejs.sh @@ -1,5 +1,8 @@ #!/bin/bash +# lock global +touch /tmp/install.lock + # lock touch /tmp/install-nodejs.lock @@ -39,3 +42,6 @@ npm install puppeteer-chromium-resolver crawlab-sdk -g --unsafe-perm=true --regi # unlock rm /tmp/install-nodejs.lock + +# unlock global +rm /tmp/install.lock diff --git a/backend/scripts/install.sh b/backend/scripts/install.sh index 3579da75..cf6bf243 100644 --- a/backend/scripts/install.sh +++ b/backend/scripts/install.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash # install node.js if [ "${CRAWLAB_SERVER_LANG_NODE}" = "Y" ]; diff --git a/backend/services/rpc.go b/backend/services/rpc.go index 66c04369..3afaf7cb 100644 --- a/backend/services/rpc.go +++ b/backend/services/rpc.go @@ -21,6 +21,8 @@ type RpcMessage struct { Result string `json:"result"` } +// ========安装语言======== + func RpcServerInstallLang(msg RpcMessage) RpcMessage { lang := GetRpcParam("lang", msg.Params) if lang == constants.Nodejs { @@ -34,6 +36,7 @@ func RpcClientInstallLang(nodeId string, lang string) (output string, err error) params := map[string]string{} params["lang"] = lang + // 发起 RPC 请求并阻塞，获取服务端数据 data, err := RpcClientFunc(nodeId, constants.RpcInstallLang, params, 600)() if err != nil { return @@ -44,6 +47,41 @@ func RpcClientInstallLang(nodeId string, lang string) (output string, err error) return } +// ========./安装语言======== + +// ========获取语言======== + +func RpcServerGetLang(msg RpcMessage) RpcMessage { + langName := GetRpcParam("lang", msg.Params) + lang := GetLangFromLangNamePlain(langName) + + // 序列化 + resultStr, _ := json.Marshal(lang) + msg.Result = string(resultStr) + return msg +} + +func RpcClientGetLang(nodeId string, langName string) (lang entity.Lang, err error) { + params := map[string]string{} + params["lang"] = langName + + data, err := RpcClientFunc(nodeId, constants.RpcGetLang, params, 10)() + if err != nil { + return + } + + // 反序列化结果 + if err := json.Unmarshal([]byte(data), &lang); err != nil { + return lang, err + } + + return +} + +// ========./获取语言======== + +// ========安装依赖======== + func RpcServerInstallDep(msg RpcMessage) RpcMessage { lang := GetRpcParam("lang", msg.Params) depName := GetRpcParam("dep_name", msg.Params) @@ -69,6 +107,10 @@ func RpcClientInstallDep(nodeId string, lang string, depName string) (output str return } +// ========./安装依赖======== + +// ========卸载依赖======== + func RpcServerUninstallDep(msg RpcMessage) RpcMessage { lang := GetRpcParam("lang", msg.Params) depName := GetRpcParam("dep_name", msg.Params) @@ -94,6 +136,10 @@ func RpcClientUninstallDep(nodeId string, lang string, depName string) (output s return } +// ========./卸载依赖======== + +// ========获取已安装依赖列表======== + func RpcServerGetInstalledDepList(nodeId string, msg RpcMessage) RpcMessage { lang := GetRpcParam("lang", msg.Params) if lang == constants.Python { @@ -125,6 +171,9 @@ func RpcClientGetInstalledDepList(nodeId string, lang string) (list []entity.Dep return } +// ========./获取已安装依赖列表======== + +// RPC 客户端函数 func RpcClientFunc(nodeId string, method string, params map[string]string, timeout int) func() (string, error) { return func() (result string, err error) { // 请求ID @@ -159,10 +208,12 @@ func RpcClientFunc(nodeId string, method string, params map[string]string, timeo } } +// 获取 RPC 参数 func GetRpcParam(key string, params map[string]string) string { return params[key] } +// Object 转化为 String func ObjectToString(params interface{}) string { bytes, _ := json.Marshal(params) return utils.BytesToString(bytes) @@ -174,6 +225,7 @@ func StopRpcService() { IsRpcStopped = true } +// 初始化 RPC 服务 func InitRpcService() error { go func() { for { @@ -213,6 +265,8 @@ func InitRpcService() error { replyMsg = RpcServerInstallLang(msg) } else if msg.Method == constants.RpcGetInstalledDepList { replyMsg = RpcServerGetInstalledDepList(node.Id.Hex(), msg) + } else if msg.Method == constants.RpcGetLang { + replyMsg = RpcServerGetLang(msg) } else { continue } diff --git a/backend/services/system.go b/backend/services/system.go index 285cad5a..dff4d029 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -62,19 +62,48 @@ func GetSystemInfo(nodeId string) (sysInfo entity.SystemInfo, err error) { return } +func getLangList() []entity.Lang { + list := []entity.Lang{ + { + Name: "Python", + ExecutableName: "python", + ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, + DepExecutablePath: "/usr/local/bin/pip", + LockPath: "/tmp/install-python.lock", + }, + { + Name: "Node.js", + ExecutableName: "node", + ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, + DepExecutablePath: "/usr/local/bin/npm", + LockPath: "/tmp/install-nodejs.lock", + }, + { + Name: "Java", + ExecutableName: "java", + ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}, + LockPath: "/tmp/install-java.lock", + }, + } + return list +} + // 获取语言列表 func GetLangList(nodeId string) []entity.Lang { - list := []entity.Lang{ - {Name: "Python", ExecutableName: "python", ExecutablePaths: []string{"/usr/bin/python", "/usr/local/bin/python"}, DepExecutablePath: "/usr/local/bin/pip"}, - {Name: "Node.js", ExecutableName: "node", ExecutablePaths: []string{"/usr/bin/node", "/usr/local/bin/node"}, DepExecutablePath: "/usr/local/bin/npm"}, - //{Name: "Java", ExecutableName: "java", ExecutablePaths: []string{"/usr/bin/java", "/usr/local/bin/java"}}, - } + list := getLangList() for i, lang := range list { - list[i].Installed = IsInstalledLang(nodeId, lang) + status, _ := GetLangInstallStatus(nodeId, lang) + list[i].InstallStatus = status } return list } +// 获取语言列表 +func GetLangListPlain() []entity.Lang { + list := getLangList() + return list +} + // 根据语言名获取语言实例 func GetLangFromLangName(nodeId string, name string) entity.Lang { langList := GetLangList(nodeId) @@ -86,6 +115,63 @@ func GetLangFromLangName(nodeId string, name string) entity.Lang { return entity.Lang{} } +// 根据语言名获取语言实例，不包含状态 +func GetLangFromLangNamePlain(name string) entity.Lang { + langList := GetLangListPlain() + for _, lang := range langList { + if lang.ExecutableName == name { + return lang + } + } + return entity.Lang{} +} + +func GetLangInstallStatus(nodeId string, lang entity.Lang) (string, error) { + if IsMasterNode(nodeId) { + lang := GetLangLocal(lang) + return lang.InstallStatus, nil + } else { + lang, err := GetLangRemote(nodeId, lang) + if err != nil { + return "", err + } + return lang.InstallStatus, nil + } +} + +func GetLangLocal(lang entity.Lang) entity.Lang{ + // 检查是否存在执行路径 + for _, p := range lang.ExecutablePaths { + if utils.Exists(p) { + lang.InstallStatus = constants.InstallStatusInstalled + return lang + } + } + + // 检查是否正在安装 + if utils.Exists(lang.LockPath) { + lang.InstallStatus = constants.InstallStatusInstalling + return lang + } + + // 检查其他语言是否在安装 + if utils.Exists("/tmp/install.lock") { + lang.InstallStatus = constants.InstallStatusInstallingOther + return lang + } + + lang.InstallStatus = constants.InstallStatusNotInstalled + return lang +} + +func GetLangRemote(nodeId string, lang entity.Lang) (entity.Lang, error) { + l, err := RpcClientGetLang(nodeId, lang.Name) + if err != nil { + return l, err + } + return l, nil +} + // 是否已安装该依赖 func IsInstalledLang(nodeId string, lang entity.Lang) bool { sysInfo, err := GetSystemInfo(nodeId) diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue index 68eb14c7..f2e7b393 100644 --- a/frontend/src/components/Node/NodeInstallation.vue +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -38,65 +38,98 @@ - -