From b7a69dcb307582a7e3f75b44fd0585f7869fabeb Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 20 Jul 2021 22:27:00 +0800 Subject: [PATCH] optimized file upload --- Dockerfile | 5 +- Dockerfile.cn | 7 +- backend/apps/api.go | 4 - backend/cmd/root.go | 21 ++- backend/go.mod | 10 +- backend/go.sum | 9 +- docker-compose.yml | 3 +- docker_init.sh | 6 +- frontend/src/components/chart/PieChart.vue | 31 +++- frontend/src/components/file/FileUpload.vue | 175 ++++++++++++++++++ .../src/components/input/InputWithButton.vue | 14 +- frontend/src/constants/file.ts | 3 + .../components/file/FileUpload.d.ts | 15 ++ frontend/src/router/index.ts | 4 +- frontend/src/services/request.ts | 19 ++ frontend/src/views/home/Home.vue | 2 +- frontend/src/views/login/Login.vue | 13 +- .../actions/SpiderDetailActionsCommon.vue | 3 +- .../actions/SpiderDetailActionsFiles.vue | 119 ++++++++++-- 19 files changed, 405 insertions(+), 58 deletions(-) create mode 100644 frontend/src/components/file/FileUpload.vue create mode 100644 frontend/src/interfaces/components/file/FileUpload.d.ts diff --git a/Dockerfile b/Dockerfile index 70198461..0f833b61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN chmod 777 /tmp \ && ln -s /usr/bin/python3 /usr/local/bin/python # install seaweedfs -RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.48/linux_amd64.tar.gz \ +RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \ && tar -zxf linux_amd64.tar.gz \ && cp weed /usr/local/bin @@ -46,9 +46,6 @@ RUN pip install scrapy pymongo bs4 requests crawlab-sdk # add files COPY ./backend/conf /app/backend/conf -#COPY ./backend/data /app/backend/data -#COPY ./backend/scripts /app/backend/scripts -#COPY ./backend/template /app/backend/template COPY ./nginx /app/nginx COPY ./docker_init.sh /app/docker_init.sh diff --git a/Dockerfile.cn b/Dockerfile.cn index 8c7390a4..20243c68 100644 --- a/Dockerfile.cn +++ b/Dockerfile.cn @@ -37,15 +37,16 @@ RUN chmod 777 /tmp \ && ln -s /usr/bin/pip3 /usr/local/bin/pip \ && ln -s /usr/bin/python3 /usr/local/bin/python +# install seaweedfs +RUN wget https://github.com/chrislusf/seaweedfs/releases/download/2.59/linux_amd64.tar.gz \ + && tar -zxf linux_amd64.tar.gz \ + && cp weed /usr/local/bin # install backend RUN pip install scrapy pymongo bs4 requests crawlab-sdk # add files COPY ./backend/conf /app/backend/conf -#COPY ./backend/data /app/backend/data -#COPY ./backend/scripts /app/backend/scripts -#COPY ./backend/template /app/backend/template COPY ./nginx /app/nginx COPY ./docker_init.sh /app/docker_init.sh diff --git a/backend/apps/api.go b/backend/apps/api.go index 72497916..43a34bc8 100644 --- a/backend/apps/api.go +++ b/backend/apps/api.go @@ -7,7 +7,6 @@ import ( "github.com/crawlab-team/crawlab-core/interfaces" "github.com/crawlab-team/crawlab-core/middlewares" "github.com/crawlab-team/crawlab-core/routes" - "github.com/crawlab-team/crawlab-db/mongo" "github.com/gin-gonic/gin" "github.com/spf13/viper" "net" @@ -25,9 +24,6 @@ type Api struct { } func (app *Api) Init() { - // initialize mongo - _ = initModule("mongo", mongo.InitMongo) - // initialize controllers _ = initModule("controllers", controllers.InitControllers) diff --git a/backend/cmd/root.go b/backend/cmd/root.go index 067d4d74..11285130 100644 --- a/backend/cmd/root.go +++ b/backend/cmd/root.go @@ -2,8 +2,8 @@ package cmd import ( "fmt" + "strings" - "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -33,20 +33,23 @@ func init() { func initConfig() { if cfgFile != "" { - // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { - // Find home directory. - home, err := homedir.Dir() - cobra.CheckErr(err) - - // Search config in home directory with name ".cobra" (without extension). - viper.AddConfigPath(home) - viper.SetConfigName(".cobra") + viper.AddConfigPath("./conf") + viper.SetConfigName("config") } + // file format as yaml + viper.SetConfigType("yaml") + + // auto load env viper.AutomaticEnv() + // env prefix as CRAWLAB + viper.SetEnvPrefix("CRAWLAB") + replacer := strings.NewReplacer(".", "_") + viper.SetEnvKeyReplacer(replacer) + if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } diff --git a/backend/go.mod b/backend/go.mod index 7ba872b5..2611ba80 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -2,13 +2,17 @@ module crawlab go 1.15 +replace ( + github.com/crawlab-team/crawlab-core => /Users/marvzhang/projects/crawlab-team/crawlab-core + github.com/crawlab-team/goseaweedfs => /Users/marvzhang/projects/crawlab-team/goseaweedfs +) + require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716 - github.com/crawlab-team/crawlab-db v0.1.0 + github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716.1817 github.com/crawlab-team/go-trace v0.1.0 + github.com/crawlab-team/goseaweedfs v0.2.0 github.com/gin-gonic/gin v1.6.3 - github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.7.1 go.mongodb.org/mongo-driver v1.6.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 848065c0..43db67b4 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -29,6 +29,7 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSi github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -70,11 +71,11 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/crawlab-team/crawlab-core v0.0.1/go.mod h1:6dJHMvrmIJbfYHhYNeGZkGOLEBvur+yGiFzLCRXx92k= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716 h1:9P/XryMK4yyIcxLpeI2f0RNqDw3lF6HjgH5DUzywxUk= -github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716/go.mod h1:OXO0hN3YwKN0hnJoa6bJ/DGWZjE4XzVAv+pfmCzOmds= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716.1817 h1:YWLhmiDxvDEFjk+n1D5eJUv4I7bGbqb4nvu/zgXRhB0= +github.com/crawlab-team/crawlab-core v0.6.0-beta.20210716.1817/go.mod h1:okTuM1EtQNk6Rl81GSW3rS5U6q4GbzVc3FsNGlvc7r8= github.com/crawlab-team/crawlab-db v0.0.2/go.mod h1:o7o4rbcyAWlFGHg9VS7V7tM/GqRq+N2mnAXO71cZA78= -github.com/crawlab-team/crawlab-db v0.1.0 h1:6dTVNb5+7cDkH8fkKOkFALk8laWNOuorYm3ZEKsvFLI= -github.com/crawlab-team/crawlab-db v0.1.0/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04= +github.com/crawlab-team/crawlab-db v0.1.1 h1:156h2fbbFKXAHs1mxprqRFC8zs2nrdyaG9JKG7patVw= +github.com/crawlab-team/crawlab-db v0.1.1/go.mod h1:t0VidSjXKzQgACqNSQV5wusXncFtL6lGEiQTbLfNR04= github.com/crawlab-team/crawlab-fs v0.0.0/go.mod h1:k2VXprQspLAmbgO5sSpqMjg/xP4iKDkW4RyTWY8eTZM= github.com/crawlab-team/crawlab-fs v0.1.0 h1:iKSJJY4Wvea8Qss+zC/tLiZ371VeV75Z3cuqlsxydzY= github.com/crawlab-team/crawlab-fs v0.1.0/go.mod h1:dOE0TeWPDz9krwzt1H72rjj0Fn/aHe53yn7GoOZHD0s= diff --git a/docker-compose.yml b/docker-compose.yml index 3ec2ce38..8ef8b450 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,8 @@ services: image: crawlabteam/crawlab:latest container_name: crawlab_master environment: - CRAWLAB_MONGO_HOST: 'mongo' + CRAWLAB_SERVER_MASTER: Y + CRAWLAB_MONGO_HOST: mongo ports: - "8080:8080" # frontend port mapping 前端端口映射 depends_on: diff --git a/docker_init.sh b/docker_init.sh index 2f13b567..b39acb3a 100755 --- a/docker_init.sh +++ b/docker_init.sh @@ -51,4 +51,8 @@ Host * EOF # start backend -crawlab-server \ No newline at end of file +if [ "${CRAWLAB_SERVER_MASTER}" = "Y" ]; + crawlab-server master +then + crawlab-server worker +fi diff --git a/frontend/src/components/chart/PieChart.vue b/frontend/src/components/chart/PieChart.vue index 18f84f08..e68ffd82 100644 --- a/frontend/src/components/chart/PieChart.vue +++ b/frontend/src/components/chart/PieChart.vue @@ -1,5 +1,8 @@ @@ -46,8 +49,16 @@ export default defineComponent({ const elRef = ref(); const chart = ref(); + const isEmpty = computed(() => { + const {config} = props; + const {data} = config; + if (!data) return true; + return data.length === 0; + + }); + const getSeriesData = (data: StatsResult[], key?: string) => { - const {valueKey, labelKey, config} = props; + const {valueKey, labelKey} = props; const _valueKey = !key ? valueKey : key; if (_valueKey) { @@ -69,7 +80,7 @@ export default defineComponent({ const seriesItem = { type: 'pie', - data: getSeriesData(data), + data: getSeriesData(data || []), radius: ['40%', '70%'], alignTo: 'labelLine', } as EChartSeries; @@ -119,6 +130,7 @@ export default defineComponent({ }); return { + isEmpty, style, elRef, render, @@ -128,7 +140,22 @@ export default defineComponent({ + + diff --git a/frontend/src/components/input/InputWithButton.vue b/frontend/src/components/input/InputWithButton.vue index fb965396..0e1090f5 100644 --- a/frontend/src/components/input/InputWithButton.vue +++ b/frontend/src/components/input/InputWithButton.vue @@ -15,8 +15,16 @@ - diff --git a/frontend/src/constants/file.ts b/frontend/src/constants/file.ts index dc2c1273..b89ba781 100644 --- a/frontend/src/constants/file.ts +++ b/frontend/src/constants/file.ts @@ -1 +1,4 @@ export const FILE_ROOT = '~'; + +export const FILE_UPLOAD_MODE_FILES = 'files'; +export const FILE_UPLOAD_MODE_DIR = 'dir'; diff --git a/frontend/src/interfaces/components/file/FileUpload.d.ts b/frontend/src/interfaces/components/file/FileUpload.d.ts new file mode 100644 index 00000000..2b0e7eac --- /dev/null +++ b/frontend/src/interfaces/components/file/FileUpload.d.ts @@ -0,0 +1,15 @@ +interface FileUploadProps { + mode?: string; + getInputProps?: Function; + open?: Function; +} + +interface FileUploadModeOption { + label: string; + value: string; +} + +interface FileUploadDirInfo { + dirName: string; + fileCount: number; +} diff --git a/frontend/src/router/index.ts b/frontend/src/router/index.ts index 33305864..bdf52285 100644 --- a/frontend/src/router/index.ts +++ b/frontend/src/router/index.ts @@ -1,4 +1,4 @@ -import {createRouter, createWebHistory, RouteRecordRaw} from 'vue-router'; +import {createRouter, createWebHashHistory, RouteRecordRaw} from 'vue-router'; import login from '@/router/login'; import home from '@/router/home'; import node from '@/router/node'; @@ -44,7 +44,7 @@ export const menuItems: MenuItem[] = [ ]; const router = createRouter({ - history: createWebHistory(process.env.BASE_URL), + history: createWebHashHistory(process.env.BASE_URL), routes, }); diff --git a/frontend/src/services/request.ts b/frontend/src/services/request.ts index 186cbe34..7122a86f 100644 --- a/frontend/src/services/request.ts +++ b/frontend/src/services/request.ts @@ -1,8 +1,27 @@ import axios, {AxiosRequestConfig} from 'axios'; +import {ElMessageBox} from 'element-plus'; +import router from '@/router'; // TODO: request interception // TODO: response interception +let msgBoxVisible = false; +axios.interceptors.response.use(res => { + return res; +}, err => { + const status = err?.response?.status; + if (status === 401) { + if (msgBoxVisible) return; + msgBoxVisible = true; + ElMessageBox.confirm('You seem to have been logged-out, try to login again?', 'Unauthorized', {type: 'warning'}) + .then(_ => router.push('/login')) + .finally(() => { + msgBoxVisible = false; + }); + } else { + console.error(err); + } +}); const useRequest = () => { // implementation diff --git a/frontend/src/views/home/Home.vue b/frontend/src/views/home/Home.vue index c1d581c2..f4bb662b 100644 --- a/frontend/src/views/home/Home.vue +++ b/frontend/src/views/home/Home.vue @@ -228,7 +228,7 @@ export default defineComponent({ // TODO: filter by date range? const {start, end} = dateRange.value; const res = await get(`/stats/daily`); - dailyConfig.value.data = spanDateRange(start, end, res.data, 'date'); + dailyConfig.value.data = spanDateRange(start, end, res.data || [], 'date'); }; const getTasks = async () => { diff --git a/frontend/src/views/login/Login.vue b/frontend/src/views/login/Login.vue index 8d0efc67..f9b746c5 100644 --- a/frontend/src/views/login/Login.vue +++ b/frontend/src/views/login/Login.vue @@ -94,12 +94,13 @@ github-stars -
+ +
中文 | English
-
+
{{ $t('Documentation') }}
@@ -129,8 +130,6 @@ const { export default defineComponent({ name: 'Login', setup() { - const {tm} = useI18n(); - const route = useRoute(); const router = useRouter(); @@ -150,7 +149,7 @@ export default defineComponent({ const validateUsername = (rule: any, value: any, callback: any) => { if (!isValidUsername(value)) { - callback(new Error(tm('Please enter the correct username'))); + callback(new Error('Please enter the correct username')); } else { callback(); } @@ -158,7 +157,7 @@ export default defineComponent({ const validatePass = (rule: any, value: any, callback: any) => { if (value.length < 5) { - callback(new Error(tm('Password length should be no shorter than 5'))); + callback(new Error('Password length should be no shorter than 5')); } else { callback(); } @@ -167,7 +166,7 @@ export default defineComponent({ const validateConfirmPass = (rule: any, value: any, callback: any) => { if (!isSignup.value) return callback(); if (value !== loginForm.value.password) { - callback(new Error(tm('Two passwords must be the same'))); + callback(new Error('Two passwords must be the same')); } else { callback(); } diff --git a/frontend/src/views/spider/detail/actions/SpiderDetailActionsCommon.vue b/frontend/src/views/spider/detail/actions/SpiderDetailActionsCommon.vue index 8afcb047..e58acf10 100644 --- a/frontend/src/views/spider/detail/actions/SpiderDetailActionsCommon.vue +++ b/frontend/src/views/spider/detail/actions/SpiderDetailActionsCommon.vue @@ -11,7 +11,8 @@ - + + diff --git a/frontend/src/views/spider/detail/actions/SpiderDetailActionsFiles.vue b/frontend/src/views/spider/detail/actions/SpiderDetailActionsFiles.vue index 8fd6f2af..94d6239b 100644 --- a/frontend/src/views/spider/detail/actions/SpiderDetailActionsFiles.vue +++ b/frontend/src/views/spider/detail/actions/SpiderDetailActionsFiles.vue @@ -1,16 +1,33 @@