加入依赖安装前端

backend/constants/system.go

@@ -5,3 +5,9 @@ const (
 	Linux   = "linux"
 	Darwin  = "darwin"
 )
+
+const (
+	Python = "python"
+	NodeJS = "node"
+	Java   = "java"
+)

backend/entity/system.go

@@ -26,6 +26,5 @@ type Dependency struct {
 	Name        string `json:"name"`
 	Version     string `json:"version"`
 	Description string `json:"description"`
-	Lang        string `json:"lang"`
 	Installed   bool   `json:"installed"`
 }

backend/main.go

@@ -130,14 +130,15 @@ func main() {
 		{
 			// 路由
 			// 节点
-			authGroup.GET("/nodes", routes.GetNodeList)               // 节点列表
-			authGroup.GET("/nodes/:id", routes.GetNode)               // 节点详情
-			authGroup.POST("/nodes/:id", routes.PostNode)             // 修改节点
-			authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表
-			authGroup.GET("/nodes/:id/system", routes.GetSystemInfo)  // 节点任务列表
-			authGroup.DELETE("/nodes/:id", routes.DeleteNode)         // 删除节点
-			authGroup.GET("/nodes/:id/langs", routes.GetLangList)     // 节点语言环境列表
-			authGroup.GET("/nodes/:id/deps", routes.GetDepList)       // 节点第三方依赖列表
+			authGroup.GET("/nodes", routes.GetNodeList)                            // 节点列表
+			authGroup.GET("/nodes/:id", routes.GetNode)                            // 节点详情
+			authGroup.POST("/nodes/:id", routes.PostNode)                          // 修改节点
+			authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList)              // 节点任务列表
+			authGroup.GET("/nodes/:id/system", routes.GetSystemInfo)               // 节点任务列表
+			authGroup.DELETE("/nodes/:id", routes.DeleteNode)                      // 删除节点
+			authGroup.GET("/nodes/:id/langs", routes.GetLangList)                  // 节点语言环境列表
+			authGroup.GET("/nodes/:id/deps", routes.GetDepList)                    // 节点第三方依赖列表
+			authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表
 			// 爬虫
 			authGroup.GET("/spiders", routes.GetSpiderList)                     // 爬虫列表
 			authGroup.GET("/spiders/:id", routes.GetSpider)                     // 爬虫详情
@@ -194,6 +195,8 @@ func main() {
 			authGroup.GET("/me", routes.GetMe)                // 获取自己账户
 			// release版本
 			authGroup.GET("/version", routes.GetVersion) // 获取发布的版本
+			// 系统
+			authGroup.GET("/system/deps", routes.GetAllDepList) // 节点所有第三方依赖列表
 		}
 
 	}

backend/routes/system.go

@@ -1,9 +1,13 @@
 package routes
 
 import (
+	"crawlab/constants"
+	"crawlab/entity"
 	"crawlab/services"
+	"fmt"
 	"github.com/gin-gonic/gin"
 	"net/http"
+	"strings"
 )
 
 func GetLangList(c *gin.Context) {
@@ -19,14 +23,88 @@ func GetDepList(c *gin.Context) {
 	nodeId := c.Param("id")
 	lang := c.Query("lang")
 	depName := c.Query("dep_name")
-	depList, err := services.GetDepList(nodeId, lang, depName)
-	if err != nil {
-		HandleError(http.StatusInternalServerError, c, err)
+
+	var depList []entity.Dependency
+	if lang == constants.Python {
+		list, err := services.GetPythonDepList(nodeId, depName)
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		depList = list
+	} else {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
 		return
 	}
+
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
 		Data:    depList,
 	})
 }
+
+func GetInstalledDepList(c *gin.Context) {
+	nodeId := c.Param("id")
+	lang := c.Query("lang")
+	var depList []entity.Dependency
+	if lang == constants.Python {
+		list, err := services.GetPythonInstalledDepList(nodeId)
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		depList = list
+	} else {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
+		return
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    depList,
+	})
+}
+
+func GetAllDepList(c *gin.Context) {
+	lang := c.Query("lang")
+	depName := c.Query("dep_name")
+
+	// 获取所有依赖列表
+	var list []string
+	if lang == constants.Python {
+		_list, err := services.GetPythonDepListFromRedis()
+		if err != nil {
+			HandleError(http.StatusInternalServerError, c, err)
+			return
+		}
+		list = _list
+	} else {
+		HandleErrorF(http.StatusBadRequest, c, fmt.Sprintf("%s is not implemented", lang))
+		return
+	}
+
+	// 过滤依赖列表
+	var depList []string
+	for _, name := range list {
+		if strings.HasPrefix(strings.ToLower(name), strings.ToLower(depName)) {
+			depList = append(depList, name)
+		}
+	}
+
+	// 只取前20
+	var returnList []string
+	for i, name := range depList {
+		if i >= 10 {
+			break
+		}
+		returnList = append(returnList, name)
+	}
+
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    returnList,
+	})
+}

backend/services/system.go

@@ -8,9 +8,11 @@ import (
 	"crawlab/model"
 	"crawlab/utils"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/apex/log"
 	"github.com/imroc/req"
+	"os/exec"
 	"regexp"
 	"runtime/debug"
 	"sort"
@@ -84,7 +86,7 @@ func GetLangList(nodeId string) []entity.Lang {
 		{Name: "Java", ExecutableName: "java", ExecutablePath: "/usr/local/bin/java"},
 	}
 	for i, lang := range list {
-		list[i].Installed = isInstalledLang(nodeId, lang)
+		list[i].Installed = IsInstalledLang(nodeId, lang)
 	}
 	return list
 }
@@ -99,20 +101,19 @@ func GetLangFromLangName(nodeId string, name string) entity.Lang {
 	return entity.Lang{}
 }
 
-func GetDepList(nodeId string, langExecutableName string, searchDepName string) ([]entity.Dependency, error) {
-	// TODO: support other languages
-	// 获取语言
-	lang := GetLangFromLangName(nodeId, langExecutableName)
+func GetPythonDepList(nodeId string, searchDepName string) ([]entity.Dependency, error) {
+	var list []entity.Dependency
 
-	// 如果没有依赖列表，先获取
-	if len(DepList) == 0 {
-		FetchDepList()
+	// 先从 Redis 获取
+	depList, err := GetPythonDepListFromRedis()
+	if err != nil {
+		return list, err
 	}
 
 	// 过滤相似的依赖
 	var depNameList PythonDepNameDictSlice
-	for _, depName := range DepList {
-		if strings.Contains(strings.ToLower(depName), strings.ToLower(searchDepName)) {
+	for _, depName := range depList {
+		if strings.HasPrefix(strings.ToLower(depName), strings.ToLower(searchDepName)) {
 			var weight int
 			if strings.ToLower(depName) == strings.ToLower(searchDepName) {
 				weight = 3
@@ -128,11 +129,17 @@ func GetDepList(nodeId string, langExecutableName string, searchDepName string)
 		}
 	}
 
-	var depList []entity.Dependency
+	// 获取已安装依赖
+	installedDepList, err := GetPythonInstalledDepList(nodeId)
+	if err != nil {
+		return list, err
+	}
+
+	// 从依赖源获取数据
 	var goSync sync.WaitGroup
 	sort.Stable(depNameList)
 	for i, depNameDict := range depNameList {
-		if i > 20 {
+		if i > 10 {
 			break
 		}
 		goSync.Add(1)
@@ -152,19 +159,40 @@ func GetDepList(nodeId string, langExecutableName string, searchDepName string)
 				Name:        depName,
 				Version:     data.Info.Version,
 				Description: data.Info.Summary,
-				Lang:        lang.ExecutableName,
-				Installed:   false,
 			}
-			depList = append(depList, dep)
+			dep.Installed = IsInstalledDep(installedDepList, dep)
+			list = append(list, dep)
 			n.Done()
 		}(depNameDict.Name, &goSync)
 	}
 	goSync.Wait()
 
-	return depList, nil
+	return list, nil
 }
 
-func isInstalledLang(nodeId string, lang entity.Lang) bool {
+func GetPythonDepListFromRedis() ([]string, error) {
+	var list []string
+
+	// 从 Redis 获取字符串
+	rawData, err := database.RedisClient.HGet("system", "deps:python")
+	if err != nil {
+		return list, err
+	}
+
+	// 反序列化
+	if err := json.Unmarshal([]byte(rawData), &list); err != nil {
+		return list, err
+	}
+
+	// 如果为空，则从依赖源获取列表
+	if len(list) == 0 {
+		UpdatePythonDepList()
+	}
+
+	return list, nil
+}
+
+func IsInstalledLang(nodeId string, lang entity.Lang) bool {
 	sysInfo, err := GetSystemInfo(nodeId)
 	if err != nil {
 		return false
@@ -177,21 +205,39 @@ func isInstalledLang(nodeId string, lang entity.Lang) bool {
 	return false
 }
 
-func FetchDepList() {
+func IsInstalledDep(installedDepList []entity.Dependency, dep entity.Dependency) bool {
+	for _, _dep := range installedDepList {
+		if strings.ToLower(_dep.Name) == strings.ToLower(dep.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func FetchPythonDepList() ([]string, error) {
+	// 依赖URL
 	url := "https://pypi.tuna.tsinghua.edu.cn/simple"
+
+	// 输出列表
+	var list []string
+
+	// 请求URL
 	res, err := req.Get(url)
 	if err != nil {
 		log.Error(err.Error())
 		debug.PrintStack()
-		return
+		return list, err
 	}
+
+	// 获取响应数据
 	text, err := res.ToString()
 	if err != nil {
 		log.Error(err.Error())
 		debug.PrintStack()
-		return
+		return list, err
 	}
-	var list []string
+
+	// 从响应数据中提取依赖名
 	regex := regexp.MustCompile("<a href=\".*/\">(.*)</a>")
 	for _, line := range strings.Split(text, "\n") {
 		arr := regex.FindStringSubmatch(line)
@@ -200,15 +246,65 @@ func FetchDepList() {
 		}
 		list = append(list, arr[1])
 	}
-	DepList = list
+
+	// 赋值给列表
+	return list, nil
 }
 
-var DepList []string
+func UpdatePythonDepList() {
+	// 从依赖源获取列表
+	list, _ := FetchPythonDepList()
+
+	// 序列化
+	listBytes, err := json.Marshal(list)
+	if err != nil {
+		log.Error(err.Error())
+		debug.PrintStack()
+		return
+	}
+
+	// 设置Redis
+	if err := database.RedisClient.HSet("system", "deps:python", string(listBytes)); err != nil {
+		log.Error(err.Error())
+		debug.PrintStack()
+		return
+	}
+}
+
+func GetPythonInstalledDepList(nodeId string) ([]entity.Dependency, error){
+	var list []entity.Dependency
+
+	lang := GetLangFromLangName(nodeId, constants.Python)
+	if !IsInstalledLang(nodeId, lang) {
+		return list, errors.New("python is not installed")
+	}
+	cmd := exec.Command("pip", "freeze")
+	outputBytes, err := cmd.Output()
+	if err != nil {
+		debug.PrintStack()
+		return list, err
+	}
+
+	for _, line := range strings.Split(string(outputBytes), "\n") {
+		arr := strings.Split(line, "==")
+		if len(arr) < 2 {
+			continue
+		}
+		dep := entity.Dependency{
+			Name:      strings.ToLower(arr[0]),
+			Version:   arr[1],
+			Installed: true,
+		}
+		list = append(list, dep)
+	}
+
+	return list, nil
+}
 
 func InitDepsFetcher() error {
 	c := cron.New(cron.WithSeconds())
 	c.Start()
-	if _, err := c.AddFunc("0 */5 * * * *", FetchDepList); err != nil {
+	if _, err := c.AddFunc("0 */5 * * * *", UpdatePythonDepList); err != nil {
 		return err
 	}
 	return nil

frontend/src/components/Node/NodeInstallation.vue

@@ -1,44 +1,63 @@
 <template>
   <div class="node-installation">
     <el-form inline>
-      <el-form-item>
-        <el-input
+      <el-form-item v-if="!isShowInstalled">
+        <el-autocomplete
           v-model="depName"
           style="width: 240px"
           :placeholder="$t('Search Dependencies')"
+          :fetchSuggestions="fetchAllDepList"
+          minlength="2"
+          @select="onSearch"
         />
       </el-form-item>
       <el-form-item>
-        <el-button icon="el-icon-search" type="success" @click="getDepList">
+        <el-button icon="el-icon-search" type="success" @click="onSearch">
           {{$t('Search')}}
         </el-button>
       </el-form-item>
+      <el-form-item>
+        <el-checkbox v-model="isShowInstalled" :label="$t('Show installed')" @change="onIsShowInstalledChange"/>
+      </el-form-item>
     </el-form>
     <el-tabs v-model="activeTab">
       <el-tab-pane v-for="lang in langList" :key="lang.name" :label="lang.name" :name="lang.executable_name"/>
     </el-tabs>
     <template v-if="activeLang.installed">
       <el-table
-        :data="depList"
+        height="calc(100vh - 280px)"
+        :data="computedDepList"
         :empty-text="depName ? $t('No Data') : $t('Please search dependencies')"
         v-loading="loading"
+        border
       >
         <el-table-column
           :label="$t('Name')"
           prop="name"
+          width="180"
         />
         <el-table-column
-          :label="$t('Version')"
+          :label="$t('Latest Version')"
           prop="version"
+          width="100"
         />
         <el-table-column
+          v-if="!isShowInstalled"
           :label="$t('Description')"
           prop="description"
         />
         <el-table-column
-          :label="$t('Installed')"
-          prop="installed"
-        />
+          :label="$t('Action')"
+        >
+          <template slot-scope="scope">
+            <el-button
+              size="mini"
+              :type="scope.row.installed ? 'danger' : 'primary' "
+            >
+              {{scope.row.installed ? $t('Uninstall') : $t('Install')}}
+            </el-button>
+          </template>
+        </el-table-column>
       </el-table>
     </template>
     <template v-else>
@@ -65,7 +84,9 @@ export default {
       langList: [],
       depName: '',
       depList: [],
-      loading: false
+      loading: false,
+      isShowInstalled: false,
+      installedDepList: []
     }
   },
   computed: {
@@ -79,15 +100,53 @@ export default {
         }
       }
       return {}
+    },
+    computedDepList () {
+      if (this.isShowInstalled) {
+        return this.installedDepList
+      } else {
+        return this.depList
+      }
     }
   },
   methods: {
     async getDepList () {
+      this.loading = true
       const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps`, {
         lang: this.activeLang.executable_name,
         dep_name: this.depName
       })
-      this.depList = res.data.data
+      this.loading = false
+      this.depList = res.data.data.sort((a, b) => a.name > b.name ? 1 : -1)
+    },
+    async getInstalledDepList () {
+      this.loading = true
+      const res = await this.$request.get(`/nodes/${this.nodeForm._id}/deps/installed`, {
+        lang: this.activeLang.executable_name
+      })
+      this.loading = false
+      this.installedDepList = res.data.data
+    },
+    async fetchAllDepList (queryString, callback) {
+      const res = await this.$request.get('/system/deps', {
+        lang: this.activeLang.executable_name,
+        dep_name: queryString
+      })
+      callback(res.data.data ? res.data.data.map(d => {
+        return { value: d, label: d }
+      }) : [])
+    },
+    onSearch () {
+      if (!this.isShowInstalled) {
+        this.getDepList()
+      } else {
+        this.getInstalledDepList()
+      }
+    },
+    onIsShowInstalledChange (val) {
+      if (val) {
+        this.getInstalledDepList()
+      }
     }
   },
   async created () {

frontend/src/i18n/zh.js

@@ -66,6 +66,7 @@ export default {
   'New File': '新建文件',
   'Rename': '重命名',
   'Install': '安装',
+  'Uninstall': '卸载',
 
   // 主页
   'Total Tasks': '总任务数',
@@ -265,6 +266,7 @@ export default {
   'ARCH': '操作架构',
   'Number of CPU': 'CPU数',
   'Executables': '执行文件',
+  'Latest Version': '最新版本',
 
   // 弹出框
   'Notification': '提示',
@@ -297,6 +299,7 @@ export default {
   'Disclaimer': '免责声明',
   'Please search dependencies': '请搜索依赖',
   'No Data': '暂无数据',
+  'Show installed': '只看已安装',
 
   // 登录
   'Sign in': '登录',