mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
优化依赖安装
This commit is contained in:
@@ -36,6 +36,7 @@ func (r *Redis) RPush(collection string, value interface{}) error {
|
||||
defer utils.Close(c)
|
||||
|
||||
if _, err := c.Do("RPUSH", collection, value); err != nil {
|
||||
log.Error(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
@@ -47,6 +48,7 @@ func (r *Redis) LPush(collection string, value interface{}) error {
|
||||
defer utils.Close(c)
|
||||
|
||||
if _, err := c.Do("RPUSH", collection, value); err != nil {
|
||||
log.Error(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
@@ -69,6 +71,7 @@ func (r *Redis) HSet(collection string, key string, value string) error {
|
||||
defer utils.Close(c)
|
||||
|
||||
if _, err := c.Do("HSET", collection, key, value); err != nil {
|
||||
log.Error(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
@@ -81,6 +84,8 @@ func (r *Redis) HGet(collection string, key string) (string, error) {
|
||||
|
||||
value, err2 := redis.String(c.Do("HGET", collection, key))
|
||||
if err2 != nil {
|
||||
log.Error(err2.Error())
|
||||
debug.PrintStack()
|
||||
return value, err2
|
||||
}
|
||||
return value, nil
|
||||
@@ -91,6 +96,8 @@ func (r *Redis) HDel(collection string, key string) error {
|
||||
defer utils.Close(c)
|
||||
|
||||
if _, err := c.Do("HDEL", collection, key); err != nil {
|
||||
log.Error(err.Error())
|
||||
debug.PrintStack()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -102,6 +109,8 @@ func (r *Redis) HKeys(collection string) ([]string, error) {
|
||||
|
||||
value, err2 := redis.Strings(c.Do("HKeys", collection))
|
||||
if err2 != nil {
|
||||
log.Error(err2.Error())
|
||||
debug.PrintStack()
|
||||
return []string{}, err2
|
||||
}
|
||||
return value, nil
|
||||
@@ -116,6 +125,8 @@ func (r *Redis) BRPop(collection string, timeout int) (string, error) {
|
||||
|
||||
values, err := redis.Strings(c.Do("BRPOP", collection, timeout))
|
||||
if err != nil {
|
||||
log.Error(err.Error())
|
||||
debug.PrintStack()
|
||||
return "", err
|
||||
}
|
||||
return values[1], nil
|
||||
|
||||
@@ -206,7 +206,8 @@ func main() {
|
||||
// release版本
|
||||
authGroup.GET("/version", routes.GetVersion) // 获取发布的版本
|
||||
// 系统
|
||||
authGroup.GET("/system/deps", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表
|
||||
authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ func GetInstalledDepList(c *gin.Context) {
|
||||
}
|
||||
|
||||
func GetAllDepList(c *gin.Context) {
|
||||
lang := c.Query("lang")
|
||||
lang := c.Param("lang")
|
||||
depName := c.Query("dep_name")
|
||||
|
||||
// 获取所有依赖列表
|
||||
@@ -197,3 +197,27 @@ func UninstallDep(c *gin.Context) {
|
||||
Message: "success",
|
||||
})
|
||||
}
|
||||
|
||||
func GetDepJson(c *gin.Context) {
|
||||
depName := c.Param("dep_name")
|
||||
lang := c.Param("lang")
|
||||
|
||||
var dep entity.Dependency
|
||||
if lang == constants.Python {
|
||||
_dep, err := services.FetchPythonDepInfo(depName)
|
||||
if err != nil {
|
||||
HandleError(http.StatusInternalServerError, c, err)
|
||||
}
|
||||
dep = _dep
|
||||
} else {
|
||||
HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
|
||||
return
|
||||
}
|
||||
|
||||
c.Header("Cache-Control", "max-age=86400")
|
||||
c.JSON(http.StatusOK, Response{
|
||||
Status: "ok",
|
||||
Message: "success",
|
||||
Data: dep,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -166,9 +166,9 @@ func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency,
|
||||
// 根据依赖名排序
|
||||
sort.Stable(depNameList)
|
||||
|
||||
// 遍历依赖名列表,取前10个
|
||||
// 遍历依赖名列表,取前20个
|
||||
for i, depNameDict := range depNameList {
|
||||
if i > 10 {
|
||||
if i > 20 {
|
||||
break
|
||||
}
|
||||
dep := entity.Dependency{
|
||||
@@ -179,7 +179,7 @@ func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency,
|
||||
}
|
||||
|
||||
// 从依赖源获取信息
|
||||
list, err = GetPythonDepListWithInfo(list)
|
||||
//list, err = GetPythonDepListWithInfo(list)
|
||||
|
||||
return list, nil
|
||||
}
|
||||
@@ -213,6 +213,28 @@ func GetPythonDepListWithInfo(depList []entity.Dependency) ([]entity.Dependency,
|
||||
return depList, nil
|
||||
}
|
||||
|
||||
func FetchPythonDepInfo(depName string) (entity.Dependency, error) {
|
||||
url := fmt.Sprintf("https://pypi.org/pypi/%s/json", depName)
|
||||
res, err := req.Get(url)
|
||||
if err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return entity.Dependency{}, err
|
||||
}
|
||||
var data PythonDepJsonData
|
||||
if err := res.ToJSON(&data); err != nil {
|
||||
log.Errorf(err.Error())
|
||||
debug.PrintStack()
|
||||
return entity.Dependency{}, err
|
||||
}
|
||||
dep := entity.Dependency{
|
||||
Name: depName,
|
||||
Version: data.Info.Version,
|
||||
Description: data.Info.Summary,
|
||||
}
|
||||
return dep, nil
|
||||
}
|
||||
|
||||
// 从Redis获取Python依赖列表
|
||||
func GetPythonDepListFromRedis() ([]string, error) {
|
||||
var list []string
|
||||
|
||||
Reference in New Issue
Block a user