diff --git a/backend/main.go b/backend/main.go index 2b052171..20d70b22 100644 --- a/backend/main.go +++ b/backend/main.go @@ -160,6 +160,7 @@ func main() { authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 @@ -210,6 +211,8 @@ func main() { // 系统 authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON + // 文件 + authGroup.GET("/file", routes.GetFile) // 获取文件 } } diff --git a/backend/model/file.go b/backend/model/file.go index fe3ece0e..1af3b851 100644 --- a/backend/model/file.go +++ b/backend/model/file.go @@ -20,10 +20,12 @@ type GridFs struct { } type File struct { - Name string `json:"name"` - Path string `json:"path"` - IsDir bool `json:"is_dir"` - Size int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` + Children []File `json:"children"` + Label string `json:"label"` } func (f *GridFs) Remove() { diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 91bab47a..2534a7da 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -440,6 +440,8 @@ func GetSpiderTasks(c *gin.Context) { }) } +// 爬虫文件管理 + func GetSpiderDir(c *gin.Context) { // 爬虫ID id := c.Param("id") @@ -481,8 +483,6 @@ func GetSpiderDir(c *gin.Context) { }) } -// 爬虫文件管理 - type SpiderFileReqBody struct { Path string `json:"path"` Content string `json:"content"` @@ -517,6 +517,36 @@ func GetSpiderFile(c *gin.Context) { }) } +func GetSpiderFileTree(c *gin.Context) { + // 爬虫ID + id := c.Param("id") + + // 获取爬虫 + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取目录下文件列表 + spiderPath := viper.GetString("spider.path") + spiderFilePath := filepath.Join(spiderPath, spider.Name) + + // 获取文件目录树 + fileNodeTree, err := services.GetFileNodeTree(spiderFilePath, 0) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 返回结果 + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: fileNodeTree, + }) +} + func PostSpiderFile(c *gin.Context) { // 爬虫ID id := c.Param("id") diff --git a/backend/services/file.go b/backend/services/file.go new file mode 100644 index 00000000..0b810209 --- /dev/null +++ b/backend/services/file.go @@ -0,0 +1,60 @@ +package services + +import ( + "crawlab/model" + "github.com/apex/log" + "os" + "path" + "runtime/debug" +) + +func GetFileNodeTree(dstPath string, level int) (f model.File, err error) { + dstF, err := os.Open(dstPath) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return f, err + } + defer dstF.Close() + fileInfo, err := dstF.Stat() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return f, nil + } + if !fileInfo.IsDir() { //如果dstF是文件 + return model.File{ + Label: fileInfo.Name(), + Name: fileInfo.Name(), + Path: dstPath, + IsDir: false, + Size: fileInfo.Size(), + Children: nil, + }, nil + } else { //如果dstF是文件夹 + dir, err := dstF.Readdir(0) //获取文件夹下各个文件或文件夹的fileInfo + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return f, nil + } + f = model.File{ + Label: path.Base(dstPath), + Name: path.Base(dstPath), + Path: dstPath, + IsDir: true, + Size: 0, + Children: nil, + } + for _, subFileInfo := range dir { + subFileNode, err := GetFileNodeTree(path.Join(dstPath, subFileInfo.Name()), level+1) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return f, err + } + f.Children = append(f.Children, subFileNode) + } + return f, nil + } +} diff --git a/frontend/src/components/File/FileList.vue b/frontend/src/components/File/FileList.vue index 50b7d306..f7ec8a21 100644 --- a/frontend/src/components/File/FileList.vue +++ b/frontend/src/components/File/FileList.vue @@ -1,101 +1,128 @@ @@ -116,10 +143,20 @@ export default { name: '', isShowAdd: false, isShowDelete: false, - isShowRename: false + isShowRename: false, + isShowCreatePopoverDict: {}, + currentFilePath: '.', + ignoreFileRegexList: [ + '__pycache__', + 'md5.txt', + '.pyc' + ] } }, computed: { + ...mapState('spider', [ + 'fileTree' + ]), ...mapState('file', [ 'fileList' ]), @@ -130,6 +167,12 @@ export default { get () { return this.$store.state.file.currentPath } + }, + computedFileTree () { + if (!this.fileTree || !this.fileTree.children) return [] + let nodes = this.sortFiles(this.fileTree.children) + nodes = this.filterFiles(nodes) + return nodes } }, methods: { @@ -221,9 +264,58 @@ export default { this.$message.success(this.$t('Renamed successfully')) this.isShowRename = false this.$st.sendEv('爬虫详情', '文件', '重命名') + }, + async getFileTree () { + const arr = this.$route.path.split('/') + const id = arr[arr.length - 1] + await this.$store.dispatch('spider/getFileTree', { id }) + }, + async onFileClick (data) { + if (data.is_dir) { + return + } + this.currentFilePath = data.path + this.onItemClick(data) + }, + sortFiles (nodes) { + nodes.forEach(node => { + if (node.is_dir) { + if (!node.children) node.children = [] + node.children = this.sortFiles(node.children) + } + }) + return nodes.sort((a, b) => { + if ((a.is_dir && b.is_dir) || (!a.is_dir && !b.is_dir)) { + return a.name > b.name ? 1 : -1 + } else { + return a.is_dir ? -1 : 1 + } + }) + }, + filterFiles (nodes) { + return nodes.filter(node => { + if (node.is_dir) { + node.children = this.filterFiles(node.children) + } + for (let i = 0; i < this.ignoreFileRegexList.length; i++) { + const regex = this.ignoreFileRegexList[i] + if (node.name.match(regex)) { + return false + } + } + return true + }) + }, + isActiveFile (node) { + return node.path === this.currentFilePath + }, + onFileRightClick (ev, data) { + console.log(data.path) + this.$set(this.isShowCreatePopoverDict, data.path, true) } }, - created () { + async created () { + await this.getFileTree() } } @@ -282,29 +374,17 @@ export default { } .file-list { - padding: 0; - margin: 0; + padding: 10px; list-style: none; - height: 450px; + height: 100%; overflow-y: auto; min-height: 100%; border-radius: 5px; border: 1px solid #eaecef; - - .item { - padding: 10px 20px; - cursor: pointer; - color: #303133; - - .item-icon { - .fa-folder { - } - } - } - - .item:hover { - background-color: rgba(48, 65, 86, 0.1); - } + display: flex; + align-items: center; + justify-content: center; + font-size: 24px; } } @@ -348,4 +428,61 @@ export default { cursor: pointer; font-weight: bolder; } + + .file-tree-wrapper { + float: left; + width: 240px; + } + + .file-tree-wrapper >>> .el-tree-node__content { + height: 30px; + } + + .file-tree-wrapper >>> .el-tree-node__content .item-name.active { + font-weight: bolder; + } + + .main-content { + float: left; + width: calc(100% - 240px); + height: calc(100vh - 200px); + } + + + + diff --git a/frontend/src/store/modules/file.js b/frontend/src/store/modules/file.js index abdf5638..33f089fe 100644 --- a/frontend/src/store/modules/file.js +++ b/frontend/src/store/modules/file.js @@ -38,8 +38,7 @@ const actions = { }, getFileContent ({ commit, rootState }, payload) { const { path } = payload - const spiderId = rootState.spider.spiderForm._id - return request.get(`/spiders/${spiderId}/file`, { path }) + return request.get(`/file`, { path }) .then(response => { commit('SET_FILE_CONTENT', response.data.data) }) diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 4632483a..ea913287 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -38,7 +38,10 @@ const state = { previewCrawlData: [], // template list - templateList: [] + templateList: [], + + // spider file tree + fileTree: {} } const getters = {} @@ -86,6 +89,9 @@ const mutations = { }, SET_TEMPLATE_LIST (state, value) { state.templateList = value + }, + SET_FILE_TREE (state, value) { + state.fileTree = value } } @@ -185,6 +191,11 @@ const actions = { const { id } = payload const res = await request.get(`/spiders/${id}/schedules`) commit('schedule/SET_SCHEDULE_LIST', res.data.data, { root: true }) + }, + async getFileTree ({ state, commit }, payload) { + const { id } = payload + const res = await request.get(`/spiders/${id}/file/tree`) + commit('SET_FILE_TREE', res.data.data) } } diff --git a/frontend/src/views/layout/components/AppMain.vue b/frontend/src/views/layout/components/AppMain.vue index fae53f18..25dfabd9 100644 --- a/frontend/src/views/layout/components/AppMain.vue +++ b/frontend/src/views/layout/components/AppMain.vue @@ -22,7 +22,7 @@ export default {