加入树形结构文件目录

backend/main.go

@@ -160,6 +160,7 @@ func main() {
 			authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId)    // 上传爬虫（ID）
 			authGroup.DELETE("/spiders/:id", routes.DeleteSpider)               // 删除爬虫
 			authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks)          // 爬虫任务列表
+			authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree)   // 爬虫文件目录树读取
 			authGroup.GET("/spiders/:id/file", routes.GetSpiderFile)            // 爬虫文件读取
 			authGroup.POST("/spiders/:id/file", routes.PostSpiderFile)          // 爬虫文件更改
 			authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile)            // 爬虫文件创建
@@ -210,6 +211,8 @@ func main() {
 			// 系统
 			authGroup.GET("/system/deps/:lang", routes.GetAllDepList)             // 节点所有第三方依赖列表
 			authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON
+			// 文件
+			authGroup.GET("/file", routes.GetFile) // 获取文件
 		}
 
 	}

backend/model/file.go

@@ -20,10 +20,12 @@ type GridFs struct {
 }
 
 type File struct {
-	Name  string `json:"name"`
-	Path  string `json:"path"`
-	IsDir bool   `json:"is_dir"`
-	Size  int64  `json:"size"`
+	Name     string `json:"name"`
+	Path     string `json:"path"`
+	IsDir    bool   `json:"is_dir"`
+	Size     int64  `json:"size"`
+	Children []File `json:"children"`
+	Label    string `json:"label"`
 }
 
 func (f *GridFs) Remove() {

backend/routes/spider.go

@@ -440,6 +440,8 @@ func GetSpiderTasks(c *gin.Context) {
 	})
 }
 
+// 爬虫文件管理
+
 func GetSpiderDir(c *gin.Context) {
 	// 爬虫ID
 	id := c.Param("id")
@@ -481,8 +483,6 @@ func GetSpiderDir(c *gin.Context) {
 	})
 }
 
-// 爬虫文件管理
-
 type SpiderFileReqBody struct {
 	Path    string `json:"path"`
 	Content string `json:"content"`
@@ -517,6 +517,36 @@ func GetSpiderFile(c *gin.Context) {
 	})
 }
 
+func GetSpiderFileTree(c *gin.Context) {
+	// 爬虫ID
+	id := c.Param("id")
+
+	// 获取爬虫
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取目录下文件列表
+	spiderPath := viper.GetString("spider.path")
+	spiderFilePath := filepath.Join(spiderPath, spider.Name)
+
+	// 获取文件目录树
+	fileNodeTree, err := services.GetFileNodeTree(spiderFilePath, 0)
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 返回结果
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    fileNodeTree,
+	})
+}
+
 func PostSpiderFile(c *gin.Context) {
 	// 爬虫ID
 	id := c.Param("id")

backend/services/file.go

@@ -0,0 +1,60 @@
+package services
+
+import (
+	"crawlab/model"
+	"github.com/apex/log"
+	"os"
+	"path"
+	"runtime/debug"
+)
+
+func GetFileNodeTree(dstPath string, level int) (f model.File, err error) {
+	dstF, err := os.Open(dstPath)
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return f, err
+	}
+	defer dstF.Close()
+	fileInfo, err := dstF.Stat()
+	if err != nil {
+		log.Errorf(err.Error())
+		debug.PrintStack()
+		return f, nil
+	}
+	if !fileInfo.IsDir() { //如果dstF是文件
+		return model.File{
+			Label:     fileInfo.Name(),
+			Name:     fileInfo.Name(),
+			Path:     dstPath,
+			IsDir:    false,
+			Size:     fileInfo.Size(),
+			Children: nil,
+		}, nil
+	} else { //如果dstF是文件夹
+		dir, err := dstF.Readdir(0) //获取文件夹下各个文件或文件夹的fileInfo
+		if err != nil {
+			log.Errorf(err.Error())
+			debug.PrintStack()
+			return f, nil
+		}
+		f = model.File{
+			Label:    path.Base(dstPath),
+			Name:     path.Base(dstPath),
+			Path:     dstPath,
+			IsDir:    true,
+			Size:     0,
+			Children: nil,
+		}
+		for _, subFileInfo := range dir {
+			subFileNode, err := GetFileNodeTree(path.Join(dstPath, subFileInfo.Name()), level+1)
+			if err != nil {
+				log.Errorf(err.Error())
+				debug.PrintStack()
+				return f, err
+			}
+			f.Children = append(f.Children, subFileNode)
+		}
+		return f, nil
+	}
+}

frontend/src/components/File/FileList.vue

@@ -1,101 +1,128 @@
 <template>
   <div class="file-list-container">
-    <div class="top-part">
-      <!--back-->
-      <div class="action-container">
-        <el-button v-if="showFile" type="primary" size="small" style="margin-right: 10px;" @click="onBackFile">
-          <font-awesome-icon :icon="['fa', 'arrow-left']"/>
-          {{$t('Back')}}
-        </el-button>
-        <el-popover v-model="isShowDelete">
-          <el-button size="small" type="default" @click="() => this.isShowDelete = false">
-            {{$t('Cancel')}}
-          </el-button>
-          <el-button size="small" type="danger" @click="onFileDelete">
-            {{$t('Confirm')}}
-          </el-button>
-          <template slot="reference">
-            <el-button v-if="currentPath !== ''" type="danger" size="small" style="margin-right: 10px;"
-                       @click="() => this.isShowDelete = true">
-              <font-awesome-icon :icon="['fa', 'trash']"/>
-              {{$t('Remove')}}
+    <div class="file-tree-wrapper">
+      <el-tree
+        :data="computedFileTree"
+        node-key="path"
+        :highlight-current="true"
+        @node-contextmenu="onFileRightClick"
+        @node-click="onFileClick"
+      >
+        <span class="custom-tree-node" slot-scope="{ node, data }">
+          <el-popover v-model="isShowCreatePopoverDict[data.path]" type="manual" placement="right"
+                      popper-class="create-item-popover">
+            <div class="create-item-title">{{$t('Create')}}</div>
+            <ul class="create-item-list">
+              <li class="create-item">
+                <font-awesome-icon :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
+                <span class="create-item-text">{{$t('Directory')}}</span>
+              </li>
+              <li class="create-item">
+                <font-awesome-icon icon="file-alt" color="rgba(3,47,98,.5)"/>
+                <span class="create-item-text">{{$t('File')}}</span>
+              </li>
+            </ul>
+            <template slot="reference">
+              <div>
+                <span class="item-icon">
+                  <font-awesome-icon v-if="data.is_dir" :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else-if="data.path.match(/\.py$/)" :icon="['fab','python']"
+                                     color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else-if="data.path.match(/\.js$/)" :icon="['fab','node-js']"
+                                     color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else-if="data.path.match(/\.(java|jar|class)$/)" :icon="['fab','java']"
+                                     color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else-if="data.path.match(/\.go$/)" :icon="['fab','go']"
+                                     color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else-if="data.path.match(/\.zip$/)" :icon="['fa','file-archive']"
+                                     color="rgba(3,47,98,.5)"/>
+                  <font-awesome-icon v-else icon="file-alt" color="rgba(3,47,98,.5)"/>
+                </span>
+                <span class="item-name" :class="isActiveFile(data) ? 'active' : ''">
+                  {{data.name}}
+                </span>
+              </div>
+            </template>
+          </el-popover>
+        </span>
+      </el-tree>
+    </div>
+    <div class="main-content">
+      <div class="top-part">
+        <!--back-->
+        <div class="action-container">
+          <!--          <el-button v-if="showFile" type="primary" size="small" style="margin-right: 10px;" @click="onBackFile">-->
+          <!--            <font-awesome-icon :icon="['fa', 'arrow-left']"/>-->
+          <!--            {{$t('Back')}}-->
+          <!--          </el-button>-->
+          <el-popover v-model="isShowDelete">
+            <el-button size="small" type="default" @click="() => this.isShowDelete = false">
+              {{$t('Cancel')}}
             </el-button>
-          </template>
-        </el-popover>
-        <el-popover v-model="isShowRename">
-          <el-input v-model="name" :placeholder="$t('Name')" style="margin-bottom: 10px"/>
-          <div style="text-align: right">
-            <el-button size="small" type="warning" @click="onRenameFile">
+            <el-button size="small" type="danger" @click="onFileDelete">
               {{$t('Confirm')}}
             </el-button>
-          </div>
-          <template slot="reference">
-            <el-button v-if="showFile" type="warning" size="small" style="margin-right: 10px;"
-                       @click="onOpenRename">
-              <font-awesome-icon :icon="['fa', 'redo']"/>
-              {{$t('Rename')}}
-            </el-button>
-          </template>
-        </el-popover>
-        <el-button v-if="showFile" type="success" size="small" style="margin-right: 10px;" @click="onFileSave">
-          <font-awesome-icon :icon="['fa', 'save']"/>
-          {{$t('Save')}}
-        </el-button>
-        <el-popover v-if="!showFile" v-model="isShowAdd" @hide="onHideAdd">
-          <el-input v-model="name" :placeholder="$t('Name')"/>
-          <div class="add-type-list">
-            <el-button size="small" type="success" icon="el-icon-document-add" @click="onAddFile">
-              {{$t('File')}}
-            </el-button>
-            <el-button size="small" type="primary" icon="el-icon-folder-add" @click="onAddDir">
-              {{$t('Directory')}}
-            </el-button>
-          </div>
-          <template slot="reference">
-            <el-button type="primary" size="small" style="margin-right: 10px;">
-              <font-awesome-icon :icon="['fa', 'plus']"/>
-              {{$t('Create')}}
-            </el-button>
-          </template>
-        </el-popover>
-      </div>
-      <!--./back-->
+            <template slot="reference">
+              <el-button v-if="currentPath !== ''" type="danger" size="small" style="margin-right: 10px;"
+                         @click="() => this.isShowDelete = true">
+                <font-awesome-icon :icon="['fa', 'trash']"/>
+                {{$t('Remove')}}
+              </el-button>
+            </template>
+          </el-popover>
+          <el-popover v-model="isShowRename">
+            <el-input v-model="name" :placeholder="$t('Name')" style="margin-bottom: 10px"/>
+            <div style="text-align: right">
+              <el-button size="small" type="warning" @click="onRenameFile">
+                {{$t('Confirm')}}
+              </el-button>
+            </div>
+            <template slot="reference">
+              <el-button v-if="showFile" type="warning" size="small" style="margin-right: 10px;"
+                         @click="onOpenRename">
+                <font-awesome-icon :icon="['fa', 'redo']"/>
+                {{$t('Rename')}}
+              </el-button>
+            </template>
+          </el-popover>
+          <el-button v-if="showFile" type="success" size="small" style="margin-right: 10px;" @click="onFileSave">
+            <font-awesome-icon :icon="['fa', 'save']"/>
+            {{$t('Save')}}
+          </el-button>
+          <el-popover v-model="isShowAdd" @hide="onHideAdd">
+            <el-input v-model="name" :placeholder="$t('Name')"/>
+            <div class="add-type-list">
+              <el-button size="small" type="success" icon="el-icon-document-add" @click="onAddFile">
+                {{$t('File')}}
+              </el-button>
+              <el-button size="small" type="primary" icon="el-icon-folder-add" @click="onAddDir">
+                {{$t('Directory')}}
+              </el-button>
+            </div>
+            <template slot="reference">
+              <el-button type="primary" size="small" style="margin-right: 10px;">
+                <font-awesome-icon :icon="['fa', 'plus']"/>
+                {{$t('Create')}}
+              </el-button>
+            </template>
+          </el-popover>
+        </div>
+        <!--./back-->
 
-      <!--file path-->
-      <div class="file-path-container">
-        <div class="file-path">. / {{currentPath}}</div>
+        <!--file path-->
+        <div class="file-path-container">
+          <div class="file-path">{{currentPath}}</div>
+        </div>
+        <!--./file path-->
       </div>
-      <!--./file path-->
+
+      <!--file list-->
+      <div v-if="!showFile" class="file-list">
+        {{$t('Please select a file on the left')}}
+      </div>
+
+      <file-detail v-else/>
     </div>
-
-    <!--file list-->
-    <ul v-if="!showFile" class="file-list">
-      <li v-if="currentPath" class="item" @click="onBack">
-        <span class="item-icon"></span>
-        <span class="item-name">..</span>
-      </li>
-      <li v-for="(item, index) in fileList" :key="index" class="item" @click="onItemClick(item)">
-            <span class="item-icon">
-              <font-awesome-icon v-if="item.is_dir" :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else-if="item.path.match(/\.py$/)" :icon="['fab','python']"
-                                 color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else-if="item.path.match(/\.js$/)" :icon="['fab','node-js']"
-                                 color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else-if="item.path.match(/\.(java|jar|class)$/)" :icon="['fab','java']"
-                                 color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else-if="item.path.match(/\.go$/)" :icon="['fab','go']"
-                                 color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else-if="item.path.match(/\.zip$/)" :icon="['fa','file-archive']"
-                                 color="rgba(3,47,98,.5)"/>
-              <font-awesome-icon v-else icon="file-alt" color="rgba(3,47,98,.5)"/>
-            </span>
-        <span class="item-name">
-              {{item.name}}
-            </span>
-      </li>
-    </ul>
-
-    <file-detail v-else/>
   </div>
 </template>
 
@@ -116,10 +143,20 @@ export default {
       name: '',
       isShowAdd: false,
       isShowDelete: false,
-      isShowRename: false
+      isShowRename: false,
+      isShowCreatePopoverDict: {},
+      currentFilePath: '.',
+      ignoreFileRegexList: [
+        '__pycache__',
+        'md5.txt',
+        '.pyc'
+      ]
     }
   },
   computed: {
+    ...mapState('spider', [
+      'fileTree'
+    ]),
     ...mapState('file', [
       'fileList'
     ]),
@@ -130,6 +167,12 @@ export default {
       get () {
         return this.$store.state.file.currentPath
       }
+    },
+    computedFileTree () {
+      if (!this.fileTree || !this.fileTree.children) return []
+      let nodes = this.sortFiles(this.fileTree.children)
+      nodes = this.filterFiles(nodes)
+      return nodes
     }
   },
   methods: {
@@ -221,9 +264,58 @@ export default {
       this.$message.success(this.$t('Renamed successfully'))
       this.isShowRename = false
       this.$st.sendEv('爬虫详情', '文件', '重命名')
+    },
+    async getFileTree () {
+      const arr = this.$route.path.split('/')
+      const id = arr[arr.length - 1]
+      await this.$store.dispatch('spider/getFileTree', { id })
+    },
+    async onFileClick (data) {
+      if (data.is_dir) {
+        return
+      }
+      this.currentFilePath = data.path
+      this.onItemClick(data)
+    },
+    sortFiles (nodes) {
+      nodes.forEach(node => {
+        if (node.is_dir) {
+          if (!node.children) node.children = []
+          node.children = this.sortFiles(node.children)
+        }
+      })
+      return nodes.sort((a, b) => {
+        if ((a.is_dir && b.is_dir) || (!a.is_dir && !b.is_dir)) {
+          return a.name > b.name ? 1 : -1
+        } else {
+          return a.is_dir ? -1 : 1
+        }
+      })
+    },
+    filterFiles (nodes) {
+      return nodes.filter(node => {
+        if (node.is_dir) {
+          node.children = this.filterFiles(node.children)
+        }
+        for (let i = 0; i < this.ignoreFileRegexList.length; i++) {
+          const regex = this.ignoreFileRegexList[i]
+          if (node.name.match(regex)) {
+            return false
+          }
+        }
+        return true
+      })
+    },
+    isActiveFile (node) {
+      return node.path === this.currentFilePath
+    },
+    onFileRightClick (ev, data) {
+      console.log(data.path)
+      this.$set(this.isShowCreatePopoverDict, data.path, true)
     }
   },
-  created () {
+  async created () {
+    await this.getFileTree()
   }
 }
 </script>
@@ -282,29 +374,17 @@ export default {
     }
 
     .file-list {
-      padding: 0;
-      margin: 0;
+      padding: 10px;
       list-style: none;
-      height: 450px;
+      height: 100%;
       overflow-y: auto;
       min-height: 100%;
       border-radius: 5px;
       border: 1px solid #eaecef;
-
-      .item {
-        padding: 10px 20px;
-        cursor: pointer;
-        color: #303133;
-
-        .item-icon {
-          .fa-folder {
-          }
-        }
-      }
-
-      .item:hover {
-        background-color: rgba(48, 65, 86, 0.1);
-      }
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      font-size: 24px;
     }
   }
 </style>
@@ -348,4 +428,61 @@ export default {
     cursor: pointer;
     font-weight: bolder;
   }
+
+  .file-tree-wrapper {
+    float: left;
+    width: 240px;
+  }
+
+  .file-tree-wrapper >>> .el-tree-node__content {
+    height: 30px;
+  }
+
+  .file-tree-wrapper >>> .el-tree-node__content .item-name.active {
+    font-weight: bolder;
+  }
+
+  .main-content {
+    float: left;
+    width: calc(100% - 240px);
+    height: calc(100vh - 200px);
+  }
+
+</style>
+
+<style>
+  .create-item-popover {
+    padding: 0;
+    margin: 0;
+  }
+
+  .create-item-list {
+    list-style: none;
+    padding: 0;
+    margin: 0
+  }
+
+  .create-item-title {
+    padding-top: 10px;
+    padding-left: 15px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #eaecef;
+  }
+
+  .create-item-list .create-item {
+    display: flex;
+    align-items: center;
+    height: 35px;
+    margin: 0 0 0 15px;
+    padding: 0;
+    cursor: pointer;
+  }
+
+  .create-item-list .create-item:hover {
+    background: #F5F7FA;
+  }
+
+  .create-item-list .create-item .create-item-text {
+    margin-left: 5px;
+  }
 </style>

frontend/src/store/modules/file.js

@@ -38,8 +38,7 @@ const actions = {
   },
   getFileContent ({ commit, rootState }, payload) {
     const { path } = payload
-    const spiderId = rootState.spider.spiderForm._id
-    return request.get(`/spiders/${spiderId}/file`, { path })
+    return request.get(`/file`, { path })
       .then(response => {
         commit('SET_FILE_CONTENT', response.data.data)
       })

frontend/src/store/modules/spider.js

@@ -38,7 +38,10 @@ const state = {
   previewCrawlData: [],
 
   // template list
-  templateList: []
+  templateList: [],
+
+  // spider file tree
+  fileTree: {}
 }
 
 const getters = {}
@@ -86,6 +89,9 @@ const mutations = {
   },
   SET_TEMPLATE_LIST (state, value) {
     state.templateList = value
+  },
+  SET_FILE_TREE (state, value) {
+    state.fileTree = value
   }
 }
 
@@ -185,6 +191,11 @@ const actions = {
     const { id } = payload
     const res = await request.get(`/spiders/${id}/schedules`)
     commit('schedule/SET_SCHEDULE_LIST', res.data.data, { root: true })
+  },
+  async getFileTree ({ state, commit }, payload) {
+    const { id } = payload
+    const res = await request.get(`/spiders/${id}/file/tree`)
+    commit('SET_FILE_TREE', res.data.data)
   }
 }
 

frontend/src/views/layout/components/AppMain.vue

@@ -22,7 +22,7 @@ export default {
 <style scoped>
   .app-main {
     /*50 = navbar  */
-    min-height: calc(100vh - 50px);
+    min-height: calc(100vh - 100px);
     position: relative;
     overflow: hidden;
   }