added file management

2026-01-22 17:31:03 +01:00 · 2019-07-24 13:37:03 +08:00
parent c048346cfc
commit 6b3a2a369f
13 changed files with 293 additions and 64 deletions
--- a/backend/main.go
+++ b/backend/main.go
@@ -92,6 +92,8 @@ func main() {
 		app.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫
 		app.DELETE("/spiders/:id", routes.DeleteSpider)        // 删除爬虫
 		app.GET("/spiders/:id/tasks", routes.GetSpiderTasks)   // 爬虫任务列表
+		app.GET("/spiders/:id/file", routes.GetSpiderFile)     // 爬虫文件
+		app.GET("/spiders/:id/dir", routes.GetSpiderDir)       // 爬虫目录
 		// 任务
 		app.GET("/tasks", routes.GetTaskList)                                 // 任务列表
 		app.GET("/tasks/:id", routes.GetTask)                                 // 任务详情
--- a/backend/model/file.go
+++ b/backend/model/file.go
@@ -0,0 +1,8 @@
+package model
+
+type File struct {
+	Name  string `json:"name"`
+	Path  string `json:"path"`
+	IsDir bool   `json:"is_dir"`
+	Size  int64  `json:"size"`
+}
--- a/backend/routes/file.go
+++ b/backend/routes/file.go
@@ -0,0 +1,20 @@
+package routes
+
+import (
+	"github.com/gin-gonic/gin"
+	"io/ioutil"
+	"net/http"
+)
+
+func GetFile(c *gin.Context) {
+	path := c.Query("path")
+	fileBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+	}
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    string(fileBytes),
+	})
+}
--- a/backend/routes/spider.go
+++ b/backend/routes/spider.go
@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/viper"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -237,3 +238,71 @@ func GetSpiderTasks(c *gin.Context) {
 		Data:    tasks,
 	})
 }
+
+func GetSpiderDir(c *gin.Context) {
+	// 爬虫ID
+	id := c.Param("id")
+
+	// 目录相对路径
+	path := c.Query("path")
+
+	// 获取爬虫
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 获取目录下文件列表
+	f, err := ioutil.ReadDir(filepath.Join(spider.Src, path))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 遍历文件列表
+	var fileList []model.File
+	for _, file := range f {
+		fileList = append(fileList, model.File{
+			Name:  file.Name(),
+			IsDir: file.IsDir(),
+			Size:  file.Size(),
+			Path:  filepath.Join(path, file.Name()),
+		})
+	}
+
+	// 返回结果
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    fileList,
+	})
+}
+
+func GetSpiderFile(c *gin.Context) {
+	// 爬虫ID
+	id := c.Param("id")
+
+	// 文件相对路径
+	path := c.Query("path")
+
+	// 获取爬虫
+	spider, err := model.GetSpider(bson.ObjectIdHex(id))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+		return
+	}
+
+	// 读取文件
+	fileBytes, err := ioutil.ReadFile(filepath.Join(spider.Src, path))
+	if err != nil {
+		HandleError(http.StatusInternalServerError, c, err)
+	}
+
+	// 返回结果
+	c.JSON(http.StatusOK, Response{
+		Status:  "ok",
+		Message: "success",
+		Data:    string(fileBytes),
+	})
+}
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -30,6 +30,7 @@
    "vcrontab": "^0.3.3",
    "vue": "^2.5.22",
    "vue-ba": "^1.2.5",
+    "vue-codemirror": "^4.0.6",
    "vue-codemirror-lite": "^1.0.4",
    "vue-i18n": "^8.9.0",
    "vue-router": "^3.0.1",
--- a/frontend/src/components/File/FileDetail.vue
+++ b/frontend/src/components/File/FileDetail.vue
@@ -0,0 +1,61 @@
+<template>
+  <codemirror
+    class="file-content"
+    :options="options"
+    v-model="fileContent"
+  />
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+import { codemirror } from 'vue-codemirror'
+
+require('codemirror/mode/python/python.js')
+require('codemirror/mode/javascript/javascript.js')
+require('codemirror/mode/go/go.js')
+require('codemirror/mode/shell/shell.js')
+require('codemirror/addon/fold/foldcode.js')
+require('codemirror/addon/fold/foldgutter.js')
+require('codemirror/addon/fold/brace-fold.js')
+require('codemirror/addon/fold/xml-fold.js')
+require('codemirror/addon/fold/indent-fold.js')
+require('codemirror/addon/fold/markdown-fold.js')
+require('codemirror/addon/fold/comment-fold.js')
+
+export default {
+  name: 'FileDetail',
+  components: { codemirror },
+  data () {
+    return {
+      internalFileContent: '',
+      options: {
+        theme: 'darcula',
+        mode: 'python',
+        lineNumbers: true
+      }
+    }
+  },
+  computed: {
+    fileContent: {
+      get () {
+        return this.$store.state.file.fileContent
+      },
+      set (value) {
+        return this.$store.commit('file/SET_FILE_CONTENT', value)
+      }
+    }
+  },
+  created () {
+    this.internalFileContent = this.fileContent
+  }
+}
+</script>
+
+<style scoped>
+  .file-content {
+    border: 1px solid #eaecef;
+    min-height: 100%;
+  }
+</style>
--- a/frontend/src/components/FileList/FileList.vue
+++ b/frontend/src/components/FileList/FileList.vue
@@ -1,43 +1,54 @@
 <template>
  <div class="file-list-container">
    <div class="top-part">
+      <!--back-->
+      <div class="action-container" v-if="showFile">
+        <el-button type="primary" size="small" style="margin-right: 10px;" @click="showFile=false">
+          <font-awesome-icon :icon="['fa', 'arrow-left']"/>
+          {{$t('Back')}}
+        </el-button>
+      </div>
+      <!--./back-->
+
      <!--file path-->
      <div class="file-path-container">
-        <div class="left">
-          <i class="el-icon-back" @click="onBack"></i>
-          <div class="file-path" v-show="!isEdit">{{currentPath}}</div>
-          <el-input class="file-path"
-                    v-show="isEdit"
-                    v-model="currentPath"
-                    @change="onChange"
-                    @keypress.enter.native="onChangeSubmit">
-          </el-input>
-        </div>
-        <i class="el-icon-edit" @click="onEdit"></i>
+        <div class="file-path">. / {{currentPath}}</div>
      </div>
+      <!--./file path-->
+
      <!--action-->
      <div class="action-container">
-        <el-button type="success" size="mini">{{$t('Choose Folder')}}</el-button>
+        <el-button type="primary" size="small">
+          <font-awesome-icon :icon="['fa', 'upload']"/>
+          {{$t('Upload')}}
+        </el-button>
      </div>
+      <!--./action-->

    </div>

    <!--file list-->
-    <template v-if="true">
-      <!--<code-mirror v-model="code"/>-->
-      <ul class="file-list">
-        <li v-for="(item, index) in fileList" :key="index" class="item" @click="onItemClick(item)">
-        <span class="item-icon">
-          <i class="fa" :class="getIcon(item.type)"></i>
-        </span>
-          <span class="item-name">
-          {{item.path}}
-        </span>
-        </li>
-      </ul>
-    </template>
-    <template v-else>
-    </template>
+    <ul v-if="!showFile" class="file-list">
+      <li v-if="currentPath" class="item" @click="onBack">
+        <span class="item-icon"></span>
+        <span class="item-name">..</span>
+      </li>
+      <li v-for="(item, index) in fileList" :key="index" class="item" @click="onItemClick(item)">
+            <span class="item-icon">
+              <font-awesome-icon v-if="item.is_dir" :icon="['fa', 'folder']" color="rgba(3,47,98,.5)"/>
+              <font-awesome-icon v-else-if="item.path.match(/\.py$/)" :icon="['fab','python']"
+                                 color="rgba(3,47,98,.5)"/>
+              <font-awesome-icon v-else-if="item.path.match(/\.zip$/)" :icon="['fa','file-archive']"
+                                 color="rgba(3,47,98,.5)"/>
+              <font-awesome-icon v-else icon="file-alt" color="rgba(3,47,98,.5)"/>
+            </span>
+        <span class="item-name">
+              {{item.name}}
+            </span>
+      </li>
+    </ul>
+
+    <file-detail v-else/>
  </div>
 </template>

@@ -45,18 +56,16 @@
 import {
  mapState
 } from 'vuex'
-import path from 'path'
-// import { codemirror } from 'vue-codemirror-lite'
+import FileDetail from './FileDetail'

 export default {
  name: 'FileList',
-  components: {
-    // CodeMirror: codemirror
-  },
+  components: { FileDetail },
  data () {
    return {
      code: 'var hello = \'world\'',
-      isEdit: false
+      isEdit: false,
+      showFile: false,
    }
  },
  computed: {
@@ -88,12 +97,17 @@ export default {
    },
    onChangeSubmit () {
      this.isEdit = false
-      this.$store.dispatch('file/getFileList', this.currentPath)
+      this.$store.dispatch('file/getFileList', { path: this.currentPath })
    },
    onItemClick (item) {
-      if (item.type === 2) {
-        this.$store.commit('file/SET_CURRENT_PATH', path.join(this.currentPath, item.path))
-        this.$store.dispatch('file/getFileList', this.currentPath)
+      if (item.is_dir) {
+        // 目录
+        this.$store.dispatch('file/getFileList', { path: item.path })
+      } else {
+        // 文件
+        this.showFile = true
+        this.$store.commit('file/SET_CURRENT_PATH', item.path)
+        this.$store.dispatch('file/getFileContent', { path: item.path })
      }
    },
    onBack () {
@@ -102,7 +116,7 @@ export default {
      arr.splice(arr.length - 1, 1)
      const path = arr.join(sep)
      this.$store.commit('file/SET_CURRENT_PATH', path)
-      this.$store.dispatch('file/getFileList', this.currentPath)
+      this.$store.dispatch('file/getFileList', { path: this.currentPath })
    }
  }
 }
@@ -114,16 +128,18 @@ export default {

    .top-part {
      display: flex;
+      height: 33px;
      margin-bottom: 10px;

      .file-path-container {
        width: 100%;
        padding: 5px;
-        margin: 0 10px;
+        margin: 0 10px 0 0;
        border-radius: 5px;
-        border: 1px solid rgba(48, 65, 86, 0.4);
+        border: 1px solid #eaecef;
        display: flex;
        justify-content: space-between;
+        color: rgba(3, 47, 98, 1);

        .left {
          width: 100%;
@@ -148,8 +164,9 @@ export default {

      .action-container {
        text-align: right;
-        padding: 1px 5px;
-        height: 24px;
+        display: flex;
+        /*padding: 1px 5px;*/
+        /*height: 24px;*/

        .el-button {
          margin: 0;
@@ -163,6 +180,9 @@ export default {
      list-style: none;
      height: 450px;
      overflow-y: auto;
+      min-height: 100%;
+      border-radius: 5px;
+      border: 1px solid #eaecef;

      .item {
        padding: 10px 20px;
@@ -197,4 +217,18 @@ export default {
  .CodeMirror-line {
    padding-right: 20px;
  }
+
+  .item {
+    border-bottom: 1px solid #eaecef;
+  }
+
+  .item-icon {
+    display: inline-block;
+    width: 18px;
+  }
+
+  .item-name {
+    font-size: 14px;
+    color: rgba(3, 47, 98, 1);
+  }
 </style>
--- a/frontend/src/main.js
+++ b/frontend/src/main.js
@@ -15,7 +15,9 @@ import { fas } from '@fortawesome/free-solid-svg-icons'
 import { far } from '@fortawesome/free-regular-svg-icons'
 import { FontAwesomeIcon, FontAwesomeLayers, FontAwesomeLayersText } from '@fortawesome/vue-fontawesome'

+import { codemirror } from 'vue-codemirror'
 import 'codemirror/lib/codemirror.css'
+import 'codemirror/theme/darcula.css'

 import App from './App'
 import store from './store'
@@ -28,8 +30,13 @@ import request from './api/request'
 import i18n from './i18n'
 import utils from './utils'

+// code mirror
+Vue.use(codemirror)
+
+// element-ui
 Vue.use(ElementUI, { locale })

+// font-awesome
 library.add(fab)
 library.add(far)
 library.add(fas)
--- a/frontend/src/store/modules/file.js
+++ b/frontend/src/store/modules/file.js
@@ -2,7 +2,8 @@ import request from '../../api/request'

 const state = {
  currentPath: '',
-  fileList: []
+  fileList: [],
+  fileContent: ''
 }

 const getters = {}
@@ -13,33 +14,34 @@ const mutations = {
  },
  SET_FILE_LIST (state, value) {
    state.fileList = value
+  },
+  SET_FILE_CONTENT (state, value) {
+    state.fileContent = value
  }
 }

 const actions = {
-  getFileList ({ commit }, path) {
+  getFileList ({ commit, rootState }, payload) {
+    const { path } = payload
+    const spiderId = rootState.spider.spiderForm._id
    commit('SET_CURRENT_PATH', path)
-    request.get('/files', { path })
+    request.get(`/spiders/${spiderId}/dir`, { path })
      .then(response => {
-        let list = []
-        list = list.concat(response.data.folders.map(d => {
-          return { path: d, type: 2 }
-        }))
-        list = list.concat(response.data.files.map(d => {
-          return { path: d, type: 1 }
-        }))
-        commit('SET_FILE_LIST', list)
+        commit(
+          'SET_FILE_LIST',
+          response.data.data
+            .sort((a, b) => a.name > b.name ? -1 : 1)
+            .sort((a, b) => a.is_dir > b.is_dir ? -1 : 1)
+        )
      })
  },
-  getDefaultPath ({ commit }) {
-    return new Promise((resolve, reject) => {
-      request.get('/files/getDefaultPath')
-        .then(response => {
-          commit('SET_CURRENT_PATH', response.data.defaultPath)
-          resolve(response.data.defaultPath)
-        })
-        .catch(reject)
-    })
+  getFileContent ({ commit, rootState }, payload) {
+    const { path } = payload
+    const spiderId = rootState.spider.spiderForm._id
+    request.get(`/spiders/${spiderId}/file`, { path })
+      .then(response => {
+        commit('SET_FILE_CONTENT', response.data.data)
+      })
  }
 }

--- a/frontend/src/store/modules/spider.js
+++ b/frontend/src/store/modules/spider.js
@@ -117,6 +117,13 @@ const actions = {
          { root: true })
      })
  },
+  getDir ({ state, commit }, path) {
+    const id = state.spiderForm._id
+    return request.get(`/spiders/${id}/dir`)
+      .then(response => {
+        commit('')
+      })
+  },
  importGithub ({ state }) {
    const url = state.importForm.url
    return request.post('/spiders/import/github', { url })
--- a/frontend/src/views/result/ResultDetail.vue
+++ b/frontend/src/views/result/ResultDetail.vue
@@ -24,7 +24,7 @@
 import {
  mapState
 } from 'vuex'
-import FileList from '../../components/FileList/FileList'
+import FileList from '../../components/File/FileList'
 import SpiderOverview from '../../components/Overview/SpiderOverview'

 export default {
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -33,7 +33,7 @@
 import {
  mapState
 } from 'vuex'
-import FileList from '../../components/FileList/FileList'
+import FileList from '../../components/File/FileList'
 import SpiderOverview from '../../components/Overview/SpiderOverview'
 import EnvironmentList from '../../components/Environment/EnvironmentList'
 import SpiderStats from '../../components/Stats/SpiderStats'
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -2116,6 +2116,11 @@ codemirror@^5.22.0:
  version "5.44.0"
  resolved "http://registry.npm.taobao.org/codemirror/download/codemirror-5.44.0.tgz#80dc2a231eeb7aab25ec2405cdca37e693ccf9cc"

+codemirror@^5.41.0:
+  version "5.48.2"
+  resolved "https://registry.npm.taobao.org/codemirror/download/codemirror-5.48.2.tgz#a9dd3d426dea4cd59efd59cd98e20a9152a30922"
+  integrity sha1-qd09Qm3qTNWe/VnNmOIKkVKjCSI=
+
 collection-visit@^1.0.0:
  version "1.0.0"
  resolved "http://registry.npm.taobao.org/collection-visit/download/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0"
@@ -2767,6 +2772,11 @@ detect-node@^2.0.4:
  version "2.0.4"
  resolved "http://registry.npm.taobao.org/detect-node/download/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c"

+diff-match-patch@^1.0.0:
+  version "1.0.4"
+  resolved "https://registry.npm.taobao.org/diff-match-patch/download/diff-match-patch-1.0.4.tgz#6ac4b55237463761c4daf0dc603eb869124744b1"
+  integrity sha1-asS1UjdGN2HE2vDcYD64aRJHRLE=
+
 diff@^3.2.0:
  version "3.5.0"
  resolved "http://registry.npm.taobao.org/diff/download/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
@@ -8473,6 +8483,14 @@ vue-codemirror-lite@^1.0.4:
  dependencies:
    codemirror "^5.22.0"

+vue-codemirror@^4.0.6:
+  version "4.0.6"
+  resolved "https://registry.npm.taobao.org/vue-codemirror/download/vue-codemirror-4.0.6.tgz#b786bb80d8d762a93aab8e46f79a81006f0437c4"
+  integrity sha1-t4a7gNjXYqk6q45G95qBAG8EN8Q=
+  dependencies:
+    codemirror "^5.41.0"
+    diff-match-patch "^1.0.0"
+
 vue-eslint-parser@^2.0.3:
  version "2.0.3"
  resolved "http://registry.npm.taobao.org/vue-eslint-parser/download/vue-eslint-parser-2.0.3.tgz#c268c96c6d94cfe3d938a5f7593959b0ca3360d1"