From 32b1f313408189462576b241216fe063cbe1d30d Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 31 Dec 2019 17:05:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E4=BE=9D=E8=B5=96=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E5=89=8D=E7=AB=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/constants/system.go | 6 + backend/entity/system.go | 1 - backend/main.go | 19 ++- backend/routes/system.go | 84 +++++++++- backend/services/system.go | 144 +++++++++++++++--- .../src/components/Node/NodeInstallation.vue | 79 ++++++++-- frontend/src/i18n/zh.js | 3 + 7 files changed, 290 insertions(+), 46 deletions(-) diff --git a/backend/constants/system.go b/backend/constants/system.go index 59c39787..70d41063 100644 --- a/backend/constants/system.go +++ b/backend/constants/system.go @@ -5,3 +5,9 @@ const ( Linux = "linux" Darwin = "darwin" ) + +const ( + Python = "python" + NodeJS = "node" + Java = "java" +) diff --git a/backend/entity/system.go b/backend/entity/system.go index a4bcdec2..ac3e9dec 100644 --- a/backend/entity/system.go +++ b/backend/entity/system.go @@ -26,6 +26,5 @@ type Dependency struct { Name string `json:"name"` Version string `json:"version"` Description string `json:"description"` - Lang string `json:"lang"` Installed bool `json:"installed"` } diff --git a/backend/main.go b/backend/main.go index 019ce6a8..6a807331 100644 --- a/backend/main.go +++ b/backend/main.go @@ -130,14 +130,15 @@ func main() { { // 路由 // 节点 - authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 - authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 - authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 - authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 - authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 - authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 + authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 // 爬虫 authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 @@ -194,6 +195,8 @@ func main() { authGroup.GET("/me", routes.GetMe) // 获取自己账户 // release版本 authGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + // 系统 + authGroup.GET("/system/deps", routes.GetAllDepList) // 节点所有第三方依赖列表 } } diff --git a/backend/routes/system.go b/backend/routes/system.go index 5042c73a..bcd186f8 100644 --- a/backend/routes/system.go +++ b/backend/routes/system.go @@ -1,9 +1,13 @@ package routes import ( + "crawlab/constants" + "crawlab/entity" "crawlab/services" + "fmt" "github.com/gin-gonic/gin" "net/http" + "strings" ) func GetLangList(c *gin.Context) { @@ -19,14 +23,88 @@ func GetDepList(c *gin.Context) { nodeId := c.Param("id") lang := c.Query("lang") depName := c.Query("dep_name") - depList, err := services.GetDepList(nodeId, lang, depName) - if err != nil { - HandleError(http.StatusInternalServerError, c, err) + + var depList []entity.Dependency + if lang == constants.Python { + list, err := services.GetPythonDepList(nodeId, depName) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) return } + c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", Data: depList, }) } + +func GetInstalledDepList(c *gin.Context) { + nodeId := c.Param("id") + lang := c.Query("lang") + var depList []entity.Dependency + if lang == constants.Python { + list, err := services.GetPythonInstalledDepList(nodeId) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + depList = list + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: depList, + }) +} + +func GetAllDepList(c *gin.Context) { + lang := c.Query("lang") + depName := c.Query("dep_name") + + // 获取所有依赖列表 + var list []string + if lang == constants.Python { + _list, err := services.GetPythonDepListFromRedis() + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + list = _list + } else { + HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang)) + return + } + + // 过滤依赖列表 + var depList []string + for _, name := range list { + if strings.HasPrefix(strings.ToLower(name), strings.ToLower(depName)) { + depList = append(depList, name) + } + } + + // 只取前20 + var returnList []string + for i, name := range depList { + if i >= 10 { + break + } + returnList = append(returnList, name) + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: returnList, + }) +} diff --git a/backend/services/system.go b/backend/services/system.go index d0e45f66..045ecbff 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -8,9 +8,11 @@ import ( "crawlab/model" "crawlab/utils" "encoding/json" + "errors" "fmt" "github.com/apex/log" "github.com/imroc/req" + "os/exec" "regexp" "runtime/debug" "sort" @@ -84,7 +86,7 @@ func GetLangList(nodeId string) []entity.Lang { {Name: "Java", ExecutableName: "java", ExecutablePath: "/usr/local/bin/java"}, } for i, lang := range list { - list[i].Installed = isInstalledLang(nodeId, lang) + list[i].Installed = IsInstalledLang(nodeId, lang) } return list } @@ -99,20 +101,19 @@ func GetLangFromLangName(nodeId string, name string) entity.Lang { return entity.Lang{} } -func GetDepList(nodeId string, langExecutableName string, searchDepName string) ([]entity.Dependency, error) { - // TODO: support other languages - // 获取语言 - lang := GetLangFromLangName(nodeId, langExecutableName) +func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, error) { + var list []entity.Dependency - // 如果没有依赖列表，先获取 - if len(DepList) == 0 { - FetchDepList() + // 先从 Redis 获取 + depList, err := GetPythonDepListFromRedis() + if err != nil { + return list, err } // 过滤相似的依赖 var depNameList PythonDepNameDictSlice - for _, depName := range DepList { - if strings.Contains(strings.ToLower(depName), strings.ToLower(searchDepName)) { + for _, depName := range depList { + if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) { var weight int if strings.ToLower(depName) == strings.ToLower(searchDepName) { weight = 3 @@ -128,11 +129,17 @@ func GetDepList(nodeId string, langExecutableName string, searchDepName string) } } - var depList []entity.Dependency + // 获取已安装依赖 + installedDepList, err := GetPythonInstalledDepList(nodeId) + if err != nil { + return list, err + } + + // 从依赖源获取数据 var goSync sync.WaitGroup sort.Stable(depNameList) for i, depNameDict := range depNameList { - if i > 20 { + if i > 10 { break } goSync.Add(1) @@ -152,19 +159,40 @@ func GetDepList(nodeId string, langExecutableName string, searchDepName string) Name: depName, Version: data.Info.Version, Description: data.Info.Summary, - Lang: lang.ExecutableName, - Installed: false, } - depList = append(depList, dep) + dep.Installed = IsInstalledDep(installedDepList, dep) + list = append(list, dep) n.Done() }(depNameDict.Name, &goSync) } goSync.Wait() - return depList, nil + return list, nil } -func isInstalledLang(nodeId string, lang entity.Lang) bool { +func GetPythonDepListFromRedis() ([]string, error) { + var list []string + + // 从 Redis 获取字符串 + rawData, err := database.RedisClient.HGet("system", "deps:python") + if err != nil { + return list, err + } + + // 反序列化 + if err := json.Unmarshal([]byte(rawData), &list); err != nil { + return list, err + } + + // 如果为空，则从依赖源获取列表 + if len(list) == 0 { + UpdatePythonDepList() + } + + return list, nil +} + +func IsInstalledLang(nodeId string, lang entity.Lang) bool { sysInfo, err := GetSystemInfo(nodeId) if err != nil { return false @@ -177,21 +205,39 @@ func isInstalledLang(nodeId string, lang entity.Lang) bool { return false } -func FetchDepList() { +func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) bool { + for _, _dep := range installedDepList { + if strings.ToLower(_dep.Name) == strings.ToLower(dep.Name) { + return true + } + } + return false +} + +func FetchPythonDepList() ([]string, error) { + // 依赖URL url := "https://pypi.tuna.tsinghua.edu.cn/simple" + + // 输出列表 + var list []string + + // 请求URL res, err := req.Get(url) if err != nil { log.Error(err.Error()) debug.PrintStack() - return + return list, err } + + // 获取响应数据 text, err := res.ToString() if err != nil { log.Error(err.Error()) debug.PrintStack() - return + return list, err } - var list []string + + // 从响应数据中提取依赖名 regex := regexp.MustCompile("(.*)") for _, line := range strings.Split(text, "\n") { arr := regex.FindStringSubmatch(line) @@ -200,15 +246,65 @@ func FetchDepList() { } list = append(list, arr[1]) } - DepList = list + + // 赋值给列表 + return list, nil } -var DepList []string +func UpdatePythonDepList() { + // 从依赖源获取列表 + list, _ := FetchPythonDepList() + + // 序列化 + listBytes, err := json.Marshal(list) + if err != nil { + log.Error(err.Error()) + debug.PrintStack() + return + } + + // 设置Redis + if err := database.RedisClient.HSet("system", "deps:python", string(listBytes)); err != nil { + log.Error(err.Error()) + debug.PrintStack() + return + } +} + +func GetPythonInstalledDepList(nodeId string) ([]entity.Dependency, error){ + var list []entity.Dependency + + lang := GetLangFromLangName(nodeId, constants.Python) + if !IsInstalledLang(nodeId, lang) { + return list, errors.New("python is not installed") + } + cmd := exec.Command("pip", "freeze") + outputBytes, err := cmd.Output() + if err != nil { + debug.PrintStack() + return list, err + } + + for _, line := range strings.Split(string(outputBytes), "\n") { + arr := strings.Split(line, "==") + if len(arr) < 2 { + continue + } + dep := entity.Dependency{ + Name: strings.ToLower(arr[0]), + Version: arr[1], + Installed: true, + } + list = append(list, dep) + } + + return list, nil +} func InitDepsFetcher() error { c := cron.New(cron.WithSeconds()) c.Start() - if _, err := c.AddFunc("0 */5 * * * *", FetchDepList); err != nil { + if _, err := c.AddFunc("0 */5 * * * *", UpdatePythonDepList); err != nil { return err } return nil diff --git a/frontend/src/components/Node/NodeInstallation.vue b/frontend/src/components/Node/NodeInstallation.vue index 080b85b9..d9de222d 100644 --- a/frontend/src/components/Node/NodeInstallation.vue +++ b/frontend/src/components/Node/NodeInstallation.vue @@ -1,44 +1,63 @@