optimized file upload

frontend/package.json

@@ -17,6 +17,7 @@
     "@popperjs/core": "^2.6.0",
     "@types/codemirror": "^0.0.103",
     "@types/echarts": "^4.9.8",
+    "@types/getos": "^3.0.1",
     "@types/humanize-duration": "^3.25.0",
     "@types/javascript-time-ago": "^2.0.2",
     "@types/md5": "^2.2.1",
@@ -31,6 +32,7 @@
     "echarts": "^5.1.2",
     "element-plus": "1.0.2-beta.40",
     "font-awesome": "^4.7.0",
+    "getos": "^3.2.1",
     "humanize-duration": "^3.26.0",
     "javascript-time-ago": "^2.3.6",
     "md5": "^2.3.0",

frontend/src/components/file/FileUpload.vue

@@ -19,15 +19,16 @@
           :http-request="() => {}"
           drag
           multiple
+          :show-file-list="false"
       >
         <i class="el-icon-upload"></i>
         <div class="el-upload__text">Drag files here, or <em>click to upload</em></div>
       </el-upload>
-      <input v-bind="getInputProps()">
+      <input v-bind="getInputProps()" multiple>
     </template>
     <template v-else-if="mode === FILE_UPLOAD_MODE_DIR">
       <div class="folder-upload">
-        <Button @click="open">
+        <Button size="large" @click="open">
           <i class="fa fa-folder"></i>
           Click to Select Folder to Upload
         </Button>
@@ -48,8 +49,16 @@
           />
         </template>
       </div>
-      <input v-bind="getInputProps()" webkitdirectory>
+      <input v-bind="getInputProps()" webkitdirectory multiple>
     </template>
+    <div v-if="dirInfo?.filePaths?.length > 0" class="file-list-wrapper">
+      <h4 class="title">Files to Upload</h4>
+      <ul class="file-list">
+        <li v-for="(path, $index) in dirInfo?.filePaths" :key="$index" class="file-item">
+          {{ path }}
+        </li>
+      </ul>
+    </div>
   </div>
 </template>
 
@@ -85,14 +94,14 @@ export default defineComponent({
   ],
   setup(props: FileUploadProps, {emit}) {
     const modeOptions: FileUploadModeOption[] = [
-      {
-        label: 'Files',
-        value: FILE_UPLOAD_MODE_FILES,
-      },
       {
         label: 'Folder',
         value: FILE_UPLOAD_MODE_DIR,
       },
+      {
+        label: 'Files',
+        value: FILE_UPLOAD_MODE_FILES,
+      },
     ];
     const internalMode = ref<string>();
 
@@ -122,11 +131,14 @@ export default defineComponent({
       internalMode.value = mode;
     });
 
-    const dirInfo = ref<FileUploadDirInfo>();
+    const dirInfo = ref<FileUploadInfo>();
 
-    const setDirInfo = (info: FileUploadDirInfo) => {
-      console.debug(info);
+    const setInfo = (info: FileUploadInfo) => {
       dirInfo.value = plainClone(info);
+    }
+
+    const resetInfo = (info: FileUploadInfo) => {
+      dirInfo.value = undefined;
     };
 
     return {
@@ -140,13 +152,16 @@ export default defineComponent({
       clearFiles,
       onModeChange,
       dirInfo,
-      setDirInfo,
+      setInfo,
+      resetInfo,
     };
   },
 });
 </script>
 
 <style scoped lang="scss">
+@import "../../styles/variables";
+
 .file-upload {
   .mode-select {
     margin-bottom: 20px;
@@ -159,6 +174,26 @@ export default defineComponent({
   .folder-upload {
     display: flex;
     align-items: center;
+
+  }
+
+  .file-list-wrapper {
+    .title {
+      margin-bottom: 0;
+      padding-bottom: 0;
+    }
+
+    .file-list {
+      list-style: none;
+      max-height: 400px;
+      overflow: auto;
+      border: 1px solid $infoPlainColor;
+      padding: 10px;
+      margin-top: 10px;
+
+      .file-item {
+      }
+    }
   }
 }
 </style>

frontend/src/constants/os.ts

@@ -0,0 +1,3 @@
+export const OS_WINDOWS = 'windows';
+export const OS_MAC = 'mac';
+export const OS_LINUX = 'linux';

frontend/src/interfaces/common/os.d.ts

@@ -0,0 +1,9 @@
+import {
+  OS_WINDOWS,
+  OS_MAC,
+  OS_LINUX,
+} from '@/constants/os';
+
+declare global {
+  type OS = OS_WINDOWS | OS_MAC | OS_LINUX;
+}

frontend/src/interfaces/components/file/FileUpload.d.ts

@@ -9,7 +9,8 @@ interface FileUploadModeOption {
   value: string;
 }
 
-interface FileUploadDirInfo {
-  dirName: string;
-  fileCount: number;
+interface FileUploadInfo {
+  dirName?: string;
+  fileCount?: number;
+  filePaths?: string[];
 }

frontend/src/utils/os.ts

@@ -0,0 +1,32 @@
+import getos from 'getos';
+import {OS_LINUX, OS_MAC, OS_WINDOWS} from '@/constants/os';
+
+let os: OS;
+
+getos((e, _os) => {
+  if (e) {
+    console.error(e);
+    return;
+  }
+
+  switch (_os.os) {
+    case 'win32':
+      return OS_WINDOWS;
+    case 'darwin':
+      return OS_MAC;
+    default:
+      return OS_LINUX;
+  }
+});
+
+export const getOS = (): OS => {
+  return os;
+};
+
+export const isWindows = (): boolean => {
+  return getOS() === OS_WINDOWS;
+};
+
+export const getOSPathSeparator = () => {
+  return isWindows() ? '\\' : '/';
+};

frontend/src/views/spider/detail/actions/SpiderDetailActionsFiles.vue

@@ -40,6 +40,8 @@ import FileUpload from '@/components/file/FileUpload.vue';
 import Dialog from '@/components/dialog/Dialog.vue';
 import {ElMessage} from 'element-plus';
 import {FILE_UPLOAD_MODE_DIR, FILE_UPLOAD_MODE_FILES} from '@/constants/file';
+import {FileWithPath} from 'file-selector';
+import {getOSPathSeparator} from '@/utils/os';
 
 export default defineComponent({
   name: 'SpiderDetailActionsFiles',
@@ -64,8 +66,8 @@ export default defineComponent({
       saveFileBinary,
     } = useSpiderService(store);
 
-    const mode = ref<string>(FILE_UPLOAD_MODE_FILES);
-    const files = ref<File[]>();
+    const mode = ref<string>(FILE_UPLOAD_MODE_DIR);
+    const files = ref<FileWithPath[]>([]);
 
     const id = computed<string>(() => route.params.id as string);
 
@@ -74,14 +76,50 @@ export default defineComponent({
     const confirmLoading = ref<boolean>(false);
     const confirmDisabled = computed<boolean>(() => !files.value?.length);
 
+    const hasMultiDir = computed<boolean>(() => {
+      if (!files.value) return false;
+      const set = new Set<string>();
+      for (const f of files.value) {
+        const lv1 = f.path?.split(getOSPathSeparator())[0] as string;
+        if (!set.has(lv1)) {
+          set.add(lv1);
+        }
+        if (set.size > 1) {
+          return true;
+        }
+      }
+      return false;
+    });
+
+    const getFilePath = (path: string) => {
+      if (hasMultiDir.value) {
+        return path;
+      } else {
+        return path.split(getOSPathSeparator()).filter((_, i) => i > 0).join(getOSPathSeparator());
+      }
+    };
+
+    const setInfo = () => {
+      // set file upload info
+      const info = {
+        fileCount: files.value.length,
+        filePaths: files.value.map(f => f.path || f.name),
+      } as FileUploadInfo;
+      if (mode.value === FILE_UPLOAD_MODE_DIR) {
+        const f = files.value[0];
+        info.dirName = f.path?.split(getOSPathSeparator())[0];
+      }
+      fileUploadRef.value?.setInfo(info);
+    };
+
     const onOpenFilesSettings = () => {
       store.commit(`${storeNamespace}/setEditorSettingsDialogVisible`, true);
     };
 
     const uploadFiles = async () => {
       if (!files.value) return;
-      await Promise.all(files.value.map(f => {
-        return saveFileBinary(id.value, f.name, f as File);
+      await Promise.all(files.value.map((f: FileWithPath) => {
+        return saveFileBinary(id.value, getFilePath(f.path as string), f as File);
       }));
       await listRootDir(id.value);
     };
@@ -90,20 +128,11 @@ export default defineComponent({
       getInputProps,
       open,
     } = useDropzone({
-      onDrop: async (fileList: InputFile[]) => {
-        if (mode.value === FILE_UPLOAD_MODE_DIR) {
-          if (!fileList.length) return;
-          const f = fileList[0];
-          const dirName = f.path?.split('/')[0];
-          const fileCount = fileList.length;
-          const dirInfo = {
-            dirName,
-            fileCount,
-          } as FileUploadDirInfo;
-          console.debug(fileList, dirInfo);
-          fileUploadRef.value?.setDirInfo(dirInfo);
-        }
-        files.value = fileList as File[];
+      onDrop: (fileList: InputFile[]) => {
+        files.value = fileList.map(f => f as FileWithPath);
+
+        // set file upload info
+        setInfo();
       },
     });
 
@@ -115,10 +144,19 @@ export default defineComponent({
 
     const onModeChange = (value: string) => {
       mode.value = value;
+
+      // reset file upload info
+      fileUploadRef.value?.resetInfo();
     };
 
-    const onFilesChange = (fileList: File[]) => {
-      files.value = fileList;
+    const onFilesChange = (fileList: FileWithPath[]) => {
+      if (!fileList.length) return;
+
+      // set files
+      files.value = fileList as FileWithPath[];
+
+      // set file upload info
+      setInfo();
     };
 
     const onUploadConfirm = async () => {

frontend/yarn.lock

@@ -1137,6 +1137,11 @@
     "@types/qs" "*"
     "@types/serve-static" "*"
 
+"@types/getos@^3.0.1":
+  version "3.0.1"
+  resolved "https://registry.nlark.com/@types/getos/download/@types/getos-3.0.1.tgz#6812bfdca81aaeba7ac2d0f4ad002b7e6566fc3b"
+  integrity sha1-aBK/3Kgarrp6wtD0rQArfmVm/Ds=
+
 "@types/glob@^7.1.1":
   version "7.1.3"
   resolved "https://registry.npm.taobao.org/@types/glob/download/@types/glob-7.1.3.tgz?cache=0&sync_timestamp=1605053412496&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2F%40types%2Fglob%2Fdownload%2F%40types%2Fglob-7.1.3.tgz#e6ba80f36b7daad2c685acd9266382e68985c183"
@@ -2268,6 +2273,11 @@ async@^2.6.2:
   dependencies:
     lodash "^4.17.14"
 
+async@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.npm.taobao.org/async/download/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720"
+  integrity sha1-s6JoXF67ZB094C0WEALGD8n4VyA=
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.npm.taobao.org/asynckit/download/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -4890,6 +4900,13 @@ get-value@^2.0.3, get-value@^2.0.6:
   resolved "https://registry.npm.taobao.org/get-value/download/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
   integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=
 
+getos@^3.2.1:
+  version "3.2.1"
+  resolved "https://registry.npm.taobao.org/getos/download/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5"
+  integrity sha1-ATTR9OAOtGFExanArE3Ah8uyfcU=
+  dependencies:
+    async "^3.2.0"
+
 getpass@^0.1.1:
   version "0.1.7"
   resolved "https://registry.npm.taobao.org/getpass/download/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"