diff --git a/backend/main.go b/backend/main.go index 79ea41d1..12030bb5 100644 --- a/backend/main.go +++ b/backend/main.go @@ -92,6 +92,8 @@ func main() { app.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 app.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 app.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + app.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件 + app.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 // 任务 app.GET("/tasks", routes.GetTaskList) // 任务列表 app.GET("/tasks/:id", routes.GetTask) // 任务详情 diff --git a/backend/model/file.go b/backend/model/file.go new file mode 100644 index 00000000..f8963d06 --- /dev/null +++ b/backend/model/file.go @@ -0,0 +1,8 @@ +package model + +type File struct { + Name string `json:"name"` + Path string `json:"path"` + IsDir bool `json:"is_dir"` + Size int64 `json:"size"` +} diff --git a/backend/routes/file.go b/backend/routes/file.go new file mode 100644 index 00000000..435f1fba --- /dev/null +++ b/backend/routes/file.go @@ -0,0 +1,20 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "io/ioutil" + "net/http" +) + +func GetFile(c *gin.Context) { + path := c.Query("path") + fileBytes, err := ioutil.ReadFile(path) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: string(fileBytes), + }) +} diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 9af2b8ce..e61ff007 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" uuid "github.com/satori/go.uuid" "github.com/spf13/viper" + "io/ioutil" "net/http" "os" "path/filepath" @@ -237,3 +238,71 @@ func GetSpiderTasks(c *gin.Context) { Data: tasks, }) } + +func GetSpiderDir(c *gin.Context) { + // 爬虫ID + id := c.Param("id") + + // 目录相对路径 + path := c.Query("path") + + // 获取爬虫 + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 获取目录下文件列表 + f, err := ioutil.ReadDir(filepath.Join(spider.Src, path)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 遍历文件列表 + var fileList []model.File + for _, file := range f { + fileList = append(fileList, model.File{ + Name: file.Name(), + IsDir: file.IsDir(), + Size: file.Size(), + Path: filepath.Join(path, file.Name()), + }) + } + + // 返回结果 + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: fileList, + }) +} + +func GetSpiderFile(c *gin.Context) { + // 爬虫ID + id := c.Param("id") + + // 文件相对路径 + path := c.Query("path") + + // 获取爬虫 + spider, err := model.GetSpider(bson.ObjectIdHex(id)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 读取文件 + fileBytes, err := ioutil.ReadFile(filepath.Join(spider.Src, path)) + if err != nil { + HandleError(http.StatusInternalServerError, c, err) + } + + // 返回结果 + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: string(fileBytes), + }) +} diff --git a/frontend/package.json b/frontend/package.json index 38eb81a6..139297d3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -30,6 +30,7 @@ "vcrontab": "^0.3.3", "vue": "^2.5.22", "vue-ba": "^1.2.5", + "vue-codemirror": "^4.0.6", "vue-codemirror-lite": "^1.0.4", "vue-i18n": "^8.9.0", "vue-router": "^3.0.1", diff --git a/frontend/src/components/File/FileDetail.vue b/frontend/src/components/File/FileDetail.vue new file mode 100644 index 00000000..a5b4da75 --- /dev/null +++ b/frontend/src/components/File/FileDetail.vue @@ -0,0 +1,61 @@ + + + + + + + diff --git a/frontend/src/components/FileList/FileList.vue b/frontend/src/components/File/FileList.vue similarity index 51% rename from frontend/src/components/FileList/FileList.vue rename to frontend/src/components/File/FileList.vue index 20a43bec..4b736297 100644 --- a/frontend/src/components/FileList/FileList.vue +++ b/frontend/src/components/File/FileList.vue @@ -1,43 +1,54 @@ + + + + + {{$t('Back')}} + + + + - - - {{currentPath}} - - - - + . / {{currentPath}} + + - {{$t('Choose Folder')}} + + + {{$t('Upload')}} + + - - - - - - - - - {{item.path}} - - - - - - + + + + .. + + + + + + + + + + {{item.name}} + + + + + @@ -45,18 +56,16 @@ import { mapState } from 'vuex' -import path from 'path' -// import { codemirror } from 'vue-codemirror-lite' +import FileDetail from './FileDetail' export default { name: 'FileList', - components: { - // CodeMirror: codemirror - }, + components: { FileDetail }, data () { return { code: 'var hello = \'world\'', - isEdit: false + isEdit: false, + showFile: false, } }, computed: { @@ -88,12 +97,17 @@ export default { }, onChangeSubmit () { this.isEdit = false - this.$store.dispatch('file/getFileList', this.currentPath) + this.$store.dispatch('file/getFileList', { path: this.currentPath }) }, onItemClick (item) { - if (item.type === 2) { - this.$store.commit('file/SET_CURRENT_PATH', path.join(this.currentPath, item.path)) - this.$store.dispatch('file/getFileList', this.currentPath) + if (item.is_dir) { + // 目录 + this.$store.dispatch('file/getFileList', { path: item.path }) + } else { + // 文件 + this.showFile = true + this.$store.commit('file/SET_CURRENT_PATH', item.path) + this.$store.dispatch('file/getFileContent', { path: item.path }) } }, onBack () { @@ -102,7 +116,7 @@ export default { arr.splice(arr.length - 1, 1) const path = arr.join(sep) this.$store.commit('file/SET_CURRENT_PATH', path) - this.$store.dispatch('file/getFileList', this.currentPath) + this.$store.dispatch('file/getFileList', { path: this.currentPath }) } } } @@ -114,16 +128,18 @@ export default { .top-part { display: flex; + height: 33px; margin-bottom: 10px; .file-path-container { width: 100%; padding: 5px; - margin: 0 10px; + margin: 0 10px 0 0; border-radius: 5px; - border: 1px solid rgba(48, 65, 86, 0.4); + border: 1px solid #eaecef; display: flex; justify-content: space-between; + color: rgba(3, 47, 98, 1); .left { width: 100%; @@ -148,8 +164,9 @@ export default { .action-container { text-align: right; - padding: 1px 5px; - height: 24px; + display: flex; + /*padding: 1px 5px;*/ + /*height: 24px;*/ .el-button { margin: 0; @@ -163,6 +180,9 @@ export default { list-style: none; height: 450px; overflow-y: auto; + min-height: 100%; + border-radius: 5px; + border: 1px solid #eaecef; .item { padding: 10px 20px; @@ -197,4 +217,18 @@ export default { .CodeMirror-line { padding-right: 20px; } + + .item { + border-bottom: 1px solid #eaecef; + } + + .item-icon { + display: inline-block; + width: 18px; + } + + .item-name { + font-size: 14px; + color: rgba(3, 47, 98, 1); + } diff --git a/frontend/src/main.js b/frontend/src/main.js index b548e734..59f5ab92 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -15,7 +15,9 @@ import { fas } from '@fortawesome/free-solid-svg-icons' import { far } from '@fortawesome/free-regular-svg-icons' import { FontAwesomeIcon, FontAwesomeLayers, FontAwesomeLayersText } from '@fortawesome/vue-fontawesome' +import { codemirror } from 'vue-codemirror' import 'codemirror/lib/codemirror.css' +import 'codemirror/theme/darcula.css' import App from './App' import store from './store' @@ -28,8 +30,13 @@ import request from './api/request' import i18n from './i18n' import utils from './utils' +// code mirror +Vue.use(codemirror) + +// element-ui Vue.use(ElementUI, { locale }) +// font-awesome library.add(fab) library.add(far) library.add(fas) diff --git a/frontend/src/store/modules/file.js b/frontend/src/store/modules/file.js index 8e01af3a..66b84651 100644 --- a/frontend/src/store/modules/file.js +++ b/frontend/src/store/modules/file.js @@ -2,7 +2,8 @@ import request from '../../api/request' const state = { currentPath: '', - fileList: [] + fileList: [], + fileContent: '' } const getters = {} @@ -13,33 +14,34 @@ const mutations = { }, SET_FILE_LIST (state, value) { state.fileList = value + }, + SET_FILE_CONTENT (state, value) { + state.fileContent = value } } const actions = { - getFileList ({ commit }, path) { + getFileList ({ commit, rootState }, payload) { + const { path } = payload + const spiderId = rootState.spider.spiderForm._id commit('SET_CURRENT_PATH', path) - request.get('/files', { path }) + request.get(`/spiders/${spiderId}/dir`, { path }) .then(response => { - let list = [] - list = list.concat(response.data.folders.map(d => { - return { path: d, type: 2 } - })) - list = list.concat(response.data.files.map(d => { - return { path: d, type: 1 } - })) - commit('SET_FILE_LIST', list) + commit( + 'SET_FILE_LIST', + response.data.data + .sort((a, b) => a.name > b.name ? -1 : 1) + .sort((a, b) => a.is_dir > b.is_dir ? -1 : 1) + ) }) }, - getDefaultPath ({ commit }) { - return new Promise((resolve, reject) => { - request.get('/files/getDefaultPath') - .then(response => { - commit('SET_CURRENT_PATH', response.data.defaultPath) - resolve(response.data.defaultPath) - }) - .catch(reject) - }) + getFileContent ({ commit, rootState }, payload) { + const { path } = payload + const spiderId = rootState.spider.spiderForm._id + request.get(`/spiders/${spiderId}/file`, { path }) + .then(response => { + commit('SET_FILE_CONTENT', response.data.data) + }) } } diff --git a/frontend/src/store/modules/spider.js b/frontend/src/store/modules/spider.js index 2c8ae07e..721b4341 100644 --- a/frontend/src/store/modules/spider.js +++ b/frontend/src/store/modules/spider.js @@ -117,6 +117,13 @@ const actions = { { root: true }) }) }, + getDir ({ state, commit }, path) { + const id = state.spiderForm._id + return request.get(`/spiders/${id}/dir`) + .then(response => { + commit('') + }) + }, importGithub ({ state }) { const url = state.importForm.url return request.post('/spiders/import/github', { url }) diff --git a/frontend/src/views/result/ResultDetail.vue b/frontend/src/views/result/ResultDetail.vue index d9f8d68f..f42bee5c 100644 --- a/frontend/src/views/result/ResultDetail.vue +++ b/frontend/src/views/result/ResultDetail.vue @@ -24,7 +24,7 @@ import { mapState } from 'vuex' -import FileList from '../../components/FileList/FileList' +import FileList from '../../components/File/FileList' import SpiderOverview from '../../components/Overview/SpiderOverview' export default { diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index ce7bfb5d..0925e9fd 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -33,7 +33,7 @@ import { mapState } from 'vuex' -import FileList from '../../components/FileList/FileList' +import FileList from '../../components/File/FileList' import SpiderOverview from '../../components/Overview/SpiderOverview' import EnvironmentList from '../../components/Environment/EnvironmentList' import SpiderStats from '../../components/Stats/SpiderStats' diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 14b963e3..e2a9386e 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2116,6 +2116,11 @@ codemirror@^5.22.0: version "5.44.0" resolved "http://registry.npm.taobao.org/codemirror/download/codemirror-5.44.0.tgz#80dc2a231eeb7aab25ec2405cdca37e693ccf9cc" +codemirror@^5.41.0: + version "5.48.2" + resolved "https://registry.npm.taobao.org/codemirror/download/codemirror-5.48.2.tgz#a9dd3d426dea4cd59efd59cd98e20a9152a30922" + integrity sha1-qd09Qm3qTNWe/VnNmOIKkVKjCSI= + collection-visit@^1.0.0: version "1.0.0" resolved "http://registry.npm.taobao.org/collection-visit/download/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -2767,6 +2772,11 @@ detect-node@^2.0.4: version "2.0.4" resolved "http://registry.npm.taobao.org/detect-node/download/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c" +diff-match-patch@^1.0.0: + version "1.0.4" + resolved "https://registry.npm.taobao.org/diff-match-patch/download/diff-match-patch-1.0.4.tgz#6ac4b55237463761c4daf0dc603eb869124744b1" + integrity sha1-asS1UjdGN2HE2vDcYD64aRJHRLE= + diff@^3.2.0: version "3.5.0" resolved "http://registry.npm.taobao.org/diff/download/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" @@ -8473,6 +8483,14 @@ vue-codemirror-lite@^1.0.4: dependencies: codemirror "^5.22.0" +vue-codemirror@^4.0.6: + version "4.0.6" + resolved "https://registry.npm.taobao.org/vue-codemirror/download/vue-codemirror-4.0.6.tgz#b786bb80d8d762a93aab8e46f79a81006f0437c4" + integrity sha1-t4a7gNjXYqk6q45G95qBAG8EN8Q= + dependencies: + codemirror "^5.41.0" + diff-match-patch "^1.0.0" + vue-eslint-parser@^2.0.3: version "2.0.3" resolved "http://registry.npm.taobao.org/vue-eslint-parser/download/vue-eslint-parser-2.0.3.tgz#c268c96c6d94cfe3d938a5f7593959b0ca3360d1"