From 1c7c20c11c68e55f1599aac6e25303f4cf2b42bc Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 15:37:06 +0800 Subject: [PATCH 01/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/task.go | 19 ++++++++-- backend/services/schedule.go | 6 +-- backend/services/task.go | 8 ++-- .../components/Common/CrawlConfirmDialog.vue | 37 ++++++++++++++----- frontend/src/views/task/TaskDetail.vue | 6 +-- 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/backend/routes/task.go b/backend/routes/task.go index 07105f2d..d1071881 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -100,6 +100,9 @@ func PutTask(c *gin.Context) { return } + // 任务ID + var taskIds []string + if reqBody.RunType == constants.RunTypeAllNodes { // 所有节点 nodes, err := model.GetNodeList(nil) @@ -115,10 +118,13 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + + taskIds = append(taskIds, id) } } else if reqBody.RunType == constants.RunTypeRandom { // 随机 @@ -127,10 +133,12 @@ func PutTask(c *gin.Context) { Param: reqBody.Param, UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + taskIds = append(taskIds, id) } else if reqBody.RunType == constants.RunTypeSelectedNodes { // 指定节点 for _, nodeId := range reqBody.NodeIds { @@ -141,16 +149,19 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + taskIds = append(taskIds, id) } } else { HandleErrorF(http.StatusInternalServerError, c, "invalid run_type") return } - HandleSuccess(c) + + HandleSuccessData(c, taskIds) } func DeleteTaskByStatus(c *gin.Context) { diff --git a/backend/services/schedule.go b/backend/services/schedule.go index a179b50f..1bf70e8a 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -37,7 +37,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { return } } @@ -49,7 +49,7 @@ func AddScheduleTask(s model.Schedule) func() { Param: s.Param, UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() return @@ -65,7 +65,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { return } } diff --git a/backend/services/task.go b/backend/services/task.go index f911159d..c71d344f 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -666,7 +666,7 @@ func CancelTask(id string) (err error) { return nil } -func AddTask(t model.Task) error { +func AddTask(t model.Task) (string, error) { // 生成任务ID id := uuid.NewV4() t.Id = id.String() @@ -683,17 +683,17 @@ func AddTask(t model.Task) error { if err := model.AddTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() - return err + return t.Id, err } // 加入任务队列 if err := AssignTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() - return err + return t.Id, err } - return nil + return t.Id, nil } func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string { diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index d25bfaaa..6739ae9e 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -30,13 +30,22 @@ - - 我已阅读并同意 《免责声明》 所有内容 +
+ + 我已阅读并同意 《免责声明》 所有内容 +
+
+ + 跳转到任务详情页 +
+
+ @@ -64,7 +73,8 @@ export default { param: '', nodeList: [] }, - isAllowDisclaimer: true + isAllowDisclaimer: true, + isRedirect: true } }, methods: { @@ -72,20 +82,27 @@ export default { this.$emit('close') }, onConfirm () { - this.$refs['form'].validate(res => { - if (!res) return + this.$refs['form'].validate(async valid => { + if (!valid) return - this.$store.dispatch('spider/crawlSpider', { + const res = await this.$store.dispatch('spider/crawlSpider', { spiderId: this.spiderId, nodeIds: this.form.nodeIds, param: this.form.param, runType: this.form.runType }) - .then(() => { - this.$message.success(this.$t('A task has been scheduled successfully')) - }) + + const id = res.data.data[0] + + this.$message.success(this.$t('A task has been scheduled successfully')) + this.$emit('close') this.$st.sendEv('爬虫确认', '确认运行', this.form.runType) + + if (this.isRedirect) { + this.$router.push('/tasks/' + id) + this.$st.sendEv('爬虫确认', '跳转到任务详情') + } }) }, onClickDisclaimer () { diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 39aca864..a5eec74d 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -10,14 +10,12 @@ - + - - - +
From 2c164f2f45b52685f9373e742aaf150121125d94 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 15:44:46 +0800 Subject: [PATCH 02/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E8=AF=A6=E6=83=85UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/node/NodeDetail.vue | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index ba53dcb8..24b81b63 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -18,7 +18,7 @@
- + @@ -136,8 +136,8 @@ export default { display: flex; align-items: center; position: absolute; - right: 20px; - margin-top: -7px; + right: 30px; + margin-top: 4px; /*float: right;*/ z-index: 999; } @@ -147,6 +147,8 @@ export default { } .label { + color: #909399; + font-weight: 100; width: 100px; text-align: right; } From 3baed65b0db2615dc020ecfc60e03c9e6d09d0e8 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 15:51:02 +0800 Subject: [PATCH 03/24] =?UTF-8?q?=E4=BC=98=E5=8C=96UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/setting/Setting.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue index 3832388d..c341ccb7 100644 --- a/frontend/src/views/setting/Setting.vue +++ b/frontend/src/views/setting/Setting.vue @@ -31,7 +31,7 @@ - + From b0e80d79515e2b289e2d7566c0e550ebcb918eea Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 17:47:01 +0800 Subject: [PATCH 04/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/ScrollView/LogItem.vue | 15 +++++- .../src/components/ScrollView/LogView.vue | 53 ++++++++++++++----- frontend/src/i18n/zh.js | 1 + 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/ScrollView/LogItem.vue b/frontend/src/components/ScrollView/LogItem.vue index 19e3e082..4e679eef 100644 --- a/frontend/src/components/ScrollView/LogItem.vue +++ b/frontend/src/components/ScrollView/LogItem.vue @@ -1,7 +1,10 @@ @@ -20,6 +23,16 @@ export default { isAnsi: { type: Boolean, default: false + }, + searchString: { + type: String, + default: '' + } + }, + computed: { + dataHtml () { + if (!this.searchString) return this.data + return this.data.replace(new RegExp(`(${this.searchString})`, 'gi'), '$1') } } } diff --git a/frontend/src/components/ScrollView/LogView.vue b/frontend/src/components/ScrollView/LogView.vue index c4c20228..4998917d 100644 --- a/frontend/src/components/ScrollView/LogView.vue +++ b/frontend/src/components/ScrollView/LogView.vue @@ -1,13 +1,22 @@ - - From 548ca3fdb991a4c11f60e7ecfe6f42936e6e3dc1 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Tue, 11 Feb 2020 18:24:49 +0800 Subject: [PATCH 06/24] =?UTF-8?q?=E4=BC=98=E5=8C=96UI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/Config/ConfigList.vue | 8 ++++++-- frontend/src/views/task/TaskDetail.vue | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 9e2f5de1..d4d83119 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -131,12 +131,16 @@
- {{$t('Run')}} + + {{$t('Run')}} + - {{$t('Save')}} + + {{$t('Save')}} +
diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index a5eec74d..d5d1fd04 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -19,7 +19,7 @@
- + {{$t('Download CSV')}}
From f34c0b529b1f5468bd9816aad186b44fc87c952b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 12 Feb 2020 11:49:37 +0800 Subject: [PATCH 07/24] fixed https://github.com/crawlab-team/crawlab/issues/546 --- Dockerfile | 3 +++ Dockerfile.local | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 40757d51..82b944ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend +# timezone environment +ENV TZ Asia/Shanghai + # frontend port EXPOSE 8080 diff --git a/Dockerfile.local b/Dockerfile.local index 240d84e4..ea4a1ff9 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -52,6 +52,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend +# timezone environment +ENV TZ Asia/Shanghai + # frontend port EXPOSE 8080 From f789cef8b78d3fb8fa581b5f49b09fedb52b7004 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 12 Feb 2020 13:52:16 +0800 Subject: [PATCH 08/24] updated docker-compose.yml --- docker-compose.yml | 4 ++++ jenkins/master/docker-compose.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 637083b2..976e5968 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: depends_on: - mongo - redis + # volumes: + # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 worker: image: tikazyq/crawlab:latest container_name: worker @@ -45,6 +47,8 @@ services: depends_on: - mongo - redis + # volumes: + # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 mongo: image: mongo:latest restart: always diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml index 75512bea..139fbb39 100644 --- a/jenkins/master/docker-compose.yaml +++ b/jenkins/master/docker-compose.yaml @@ -18,6 +18,8 @@ services: depends_on: - mongo - redis + volumes: + - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 worker: image: "tikazyq/crawlab:master" environment: @@ -30,6 +32,8 @@ services: depends_on: - mongo - redis + volumes: + - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 mongo: image: mongo:latest restart: always From 29d180f9ab2370818a9ee4fcd122fabffb392a4e Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 12 Feb 2020 18:23:52 +0800 Subject: [PATCH 09/24] =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG-zh.md | 11 +++ CHANGELOG.md | 11 +++ .../src/components/ScrollView/LogItem.vue | 49 +++++++++- .../src/components/ScrollView/LogView.vue | 95 ++++++++++++++++--- frontend/src/i18n/zh.js | 2 + 5 files changed, 152 insertions(+), 16 deletions(-) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index e0b5693f..14f2653c 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,14 @@ +# 0.4.6 (未知) +### 功能 / 优化 +- **Node.js SDK **. 用户可以将 SDK 应用到他们的 Node.js 爬虫中. +- **日志管理优化**. 日志搜索,错误高亮,自动滚动. +- **任务执行流程优化**. 允许用户在触发任务后跳转到该任务详情页. +- **任务展示优化**. 在爬虫详情页的最近任务表格中加入了“参数”列. [#295](https://github.com/crawlab-team/crawlab/issues/295) + +### Bug 修复 +- **定时任务配置失去焦点**. [#519](https://github.com/crawlab-team/crawlab/issues/519) +- **无法用 CLI 工具上传爬虫**. [#524](https://github.com/crawlab-team/crawlab/issues/524) + # 0.4.5 (2020-02-03) ### 功能 / 优化 - **交互式教程**. 引导用户了解 Crawlab 的主要功能. diff --git a/CHANGELOG.md b/CHANGELOG.md index a4bf4fa4..5ec1e280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# 0.4.6 (unkown) +### Features / Enhancement +- **SDK for Node.js**. Users can apply SDK in their Node.js spiders. +- **Log Management Optimization**. Log search, error highlight, auto-scrolling. +- **Task Execution Process Optimization**. Allow users to be redirected to task detail page after triggering a task. +- **Task Display Optimization**. Added "Param" in the Latest Tasks table in the spider detail page. [#295](https://github.com/crawlab-team/crawlab/issues/295) + +### Bug Fixes +- **Lost Focus in Schedule Configuration**. [#519](https://github.com/crawlab-team/crawlab/issues/519) +- **Unable to Upload Spider using CLI**. [#524](https://github.com/crawlab-team/crawlab/issues/524) + # 0.4.5 (2020-02-03) ### Features / Enhancement - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab. diff --git a/frontend/src/components/ScrollView/LogItem.vue b/frontend/src/components/ScrollView/LogItem.vue index 4e679eef..1c82a7cb 100644 --- a/frontend/src/components/ScrollView/LogItem.vue +++ b/frontend/src/components/ScrollView/LogItem.vue @@ -1,9 +1,13 @@ @@ -29,10 +33,31 @@ export default { default: '' } }, + data () { + const token = ' :,.' + return { + errorRegex: new RegExp(`(?:[${token}]|^)((?:error|exception|traceback)s?)(?:[${token}]|$)`, 'gi') + // errorRegex: new RegExp('(error|exception|traceback)', 'gi') + } + }, computed: { dataHtml () { - if (!this.searchString) return this.data - return this.data.replace(new RegExp(`(${this.searchString})`, 'gi'), '$1') + let html = this.data.replace(this.errorRegex, ' $1 ') + if (!this.searchString) return html + html = html.replace(new RegExp(`(${this.searchString})`, 'gi'), '$1') + return html + }, + style () { + let color = '' + if (this.data.match(this.errorRegex)) { + color = '#F56C6C' + } + return { + color + } + }, + isLogEnd () { + return this.data === '###LOG_END###' } } } @@ -64,4 +89,18 @@ export default { word-break: break-word; flex-basis: calc(100% - 50px); } + + .loading-text { + animation: blink; + } + + @keyframes blink { + 0% { + opacity: 0; + } + + 100% { + opacity: 100%; + } + } diff --git a/frontend/src/components/ScrollView/LogView.vue b/frontend/src/components/ScrollView/LogView.vue index 4998917d..bfb4edec 100644 --- a/frontend/src/components/ScrollView/LogView.vue +++ b/frontend/src/components/ScrollView/LogView.vue @@ -1,30 +1,50 @@ diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 4ff4cd13..2dcb46d2 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -209,6 +209,8 @@ export default { 'Random': '随机', 'Selected Nodes': '指定节点', 'Search Log': '搜索日志', + 'Auto-Scroll': '自动滚动', + 'Updating log...': '正在更新日志...', // 任务列表 'Node': '节点', From 0ea95e676acdcba21f092ac2ffe4a035dcd19bea Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 12 Feb 2020 18:30:45 +0800 Subject: [PATCH 10/24] fixed docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 976e5968..9d5acbb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,7 +59,7 @@ services: redis: image: redis:latest restart: always - # command: redis --requirepass "password" # set redis password 设置 Redis 密码 + # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码 # volumes: # - "/opt/crawlab/redis/data:/data" # make data persistent 持久化 # ports: From 7c4ddd2824474832ea4ea3701d53a1136dab35a1 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 11:15:51 +0800 Subject: [PATCH 11/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=EF=BC=8C=E9=94=99=E8=AF=AF=E6=A0=87=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/InfoView/TaskInfoView.vue | 25 +- .../src/components/Overview/TaskOverview.vue | 2 +- .../src/components/ScrollView/LogItem.vue | 50 ++-- .../src/components/ScrollView/LogView.vue | 250 ++++++++++++++---- frontend/src/i18n/zh.js | 2 + frontend/src/store/modules/task.js | 31 +++ frontend/src/utils/index.js | 4 +- frontend/src/utils/log.js | 5 + frontend/src/views/task/TaskDetail.vue | 14 +- 9 files changed, 298 insertions(+), 85 deletions(-) create mode 100644 frontend/src/utils/log.js diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index 34318d84..d55945b0 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -11,6 +11,15 @@ + + + {{$t('Log with errors')}} + + @@ -28,7 +37,7 @@ - + @@ -37,7 +46,7 @@ - + @@ -59,7 +68,8 @@ diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 2dcb46d2..5dbf2919 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -211,6 +211,8 @@ export default { 'Search Log': '搜索日志', 'Auto-Scroll': '自动滚动', 'Updating log...': '正在更新日志...', + 'Error Count': '错误数', + 'Log with errors': '日志错误', // 任务列表 'Node': '节点', diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 95153be5..563fc907 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -1,4 +1,5 @@ import request from '../../api/request' +import utils from '../../utils' const state = { // TaskList @@ -6,6 +7,7 @@ const state = { taskListTotalCount: 0, taskForm: {}, taskLog: '', + currentLogIndex: 0, taskResultsData: [], taskResultsColumns: [], taskResultsTotalCount: 0, @@ -36,6 +38,32 @@ const getters = { } } return keys + }, + logData (state) { + const data = state.taskLog.split('\n') + .map((d, i) => { + return { + index: i + 1, + data: d, + active: state.currentLogIndex === i + 1 + } + }) + if (state.taskForm && state.taskForm.status === 'running') { + data.push({ + index: data.length + 1, + data: '###LOG_END###' + }) + data.push({ + index: data.length + 1, + data: '' + }) + } + return data + }, + errorLogData (state, getters) { + return getters.logData.filter(d => { + return d.data.match(utils.log.errorRegex) + }) } } @@ -49,6 +77,9 @@ const mutations = { SET_TASK_LOG (state, value) { state.taskLog = value }, + SET_CURRENT_LOG_INDEX (state, value) { + state.currentLogIndex = value + }, SET_TASK_RESULTS_DATA (state, value) { state.taskResultsData = value }, diff --git a/frontend/src/utils/index.js b/frontend/src/utils/index.js index ab33114f..9867c812 100644 --- a/frontend/src/utils/index.js +++ b/frontend/src/utils/index.js @@ -1,9 +1,11 @@ import stats from './stats' import encrypt from './encrypt' import tour from './tour' +import log from './log' export default { stats, encrypt, - tour + tour, + log } diff --git a/frontend/src/utils/log.js b/frontend/src/utils/log.js new file mode 100644 index 00000000..8b1ea0ab --- /dev/null +++ b/frontend/src/utils/log.js @@ -0,0 +1,5 @@ +const regexToken = ' :,.' + +export default { + errorRegex: new RegExp(`(?:[${regexToken}]|^)((?:error|exception|traceback)s?)(?:[${regexToken}]|$)`, 'gi') +} diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index d5d1fd04..8af32049 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -12,10 +12,10 @@ - + - +
@@ -55,7 +55,6 @@ export default { return { activeTabName: 'overview', handle: undefined, - taskLog: '', // tutorial tourSteps: [ @@ -137,7 +136,8 @@ export default { ...mapState('task', [ 'taskForm', 'taskResultsData', - 'taskResultsTotalCount' + 'taskResultsTotalCount', + 'taskLog' ]), ...mapGetters('task', [ 'taskResultsColumns' @@ -186,11 +186,7 @@ export default { this.$st.sendEv('任务详情', '结果', '下载CSV') }, getTaskLog () { - if (this.$route.params.id) { - request.get(`/tasks/${this.$route.params.id}/log`).then(response => { - this.taskLog = response.data.data - }) - } + this.$store.dispatch('task/getTaskLog', this.$route.params.id) } }, created () { From d840450c1bbca9abe9e97bf9f269f29570cc1e84 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 11:44:28 +0800 Subject: [PATCH 12/24] code cleanup --- frontend/src/components/ScrollView/LogView.vue | 4 +++- frontend/src/views/task/TaskDetail.vue | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/ScrollView/LogView.vue b/frontend/src/components/ScrollView/LogView.vue index d1edd60d..04905cb8 100644 --- a/frontend/src/components/ScrollView/LogView.vue +++ b/frontend/src/components/ScrollView/LogView.vue @@ -212,11 +212,12 @@ export default { } }, mounted () { + this.currentLogIndex = 0 this.handle = setInterval(() => { if (this.isToBottom) { this.toBottom() } - }, 500) + }, 200) }, destroyed () { clearInterval(this.handle) @@ -269,6 +270,7 @@ export default { border-bottom: 1px solid #DCDFE6; height: calc(100vh - 240px); font-size: 16px; + overflow: auto; } .errors-wrapper.collapsed { diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index 8af32049..11051ed0 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -42,7 +42,6 @@ import { import TaskOverview from '../../components/Overview/TaskOverview' import GeneralTableView from '../../components/TableView/GeneralTableView' import LogView from '../../components/ScrollView/LogView' -import request from '../../api/request' export default { name: 'TaskDetail', From 73ed6bef3031989d0e999d2eeff979a8fcbf0f3b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 12:10:18 +0800 Subject: [PATCH 13/24] =?UTF-8?q?=E9=9B=B6=E5=80=BC=E6=8A=A5=E8=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/InfoView/TaskInfoView.vue | 9 +++++++++ frontend/src/components/Status/StatusTag.vue | 13 ++++++++++++- frontend/src/i18n/zh.js | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index d55945b0..d4b55d68 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -17,9 +17,18 @@ style="margin-left:10px; cursor:pointer;" > + {{$t('Log with errors')}} + + + {{$t('Empty results')}} + diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue index 1f4f0c89..befe2ab3 100644 --- a/frontend/src/components/Status/StatusTag.vue +++ b/frontend/src/components/Status/StatusTag.vue @@ -1,5 +1,6 @@ @@ -38,6 +39,16 @@ export default { return s.label } return 'NA' + }, + icon () { + if (this.status === 'finished') { + return 'el-icon-check' + } else if (this.status === 'running') { + return 'el-icon-loading' + } else if (this.status === 'error') { + return 'el-icon-error' + } + return '' } } } diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index 5dbf2919..419fcd9d 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -213,6 +213,7 @@ export default { 'Updating log...': '正在更新日志...', 'Error Count': '错误数', 'Log with errors': '日志错误', + 'Empty results': '空结果', // 任务列表 'Node': '节点', From 6be0ef3adfc384af1b656abf45a6a46a3d8b89c8 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 12:13:20 +0800 Subject: [PATCH 14/24] =?UTF-8?q?=E4=BC=98=E5=8C=96icon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/InfoView/TaskInfoView.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index d4b55d68..7e08af8d 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -69,7 +69,9 @@ - {{$t('Stop')}} + + {{$t('Stop')}} +
From d4651a421ff126770d4d9ebee88f5acc5926d761 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 13:42:56 +0800 Subject: [PATCH 15/24] fixed https://github.com/crawlab-team/crawlab/issues/505 --- frontend/src/views/spider/SpiderList.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index bbfbee75..3a1d38ca 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -399,7 +399,8 @@ export default { { name: 'type', label: 'Spider Type', width: '120', sortable: true }, { name: 'last_status', label: 'Last Status', width: '120' }, { name: 'last_run_ts', label: 'Last Run', width: '140' }, - // { name: 'update_ts', label: 'Update Time', width: '140' }, + { name: 'update_ts', label: 'Update Time', width: '140' }, + { name: 'create_ts', label: 'Create Time', width: '140' }, { name: 'remark', label: 'Remark', width: '140' } ], spiderFormRules: { From d04c71b7d101cc1ac7168f7f7eabd1599ade79a9 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 13:44:33 +0800 Subject: [PATCH 16/24] updated CHANGELOG --- CHANGELOG-zh.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 14f2653c..6ff633ce 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -4,6 +4,7 @@ - **日志管理优化**. 日志搜索,错误高亮,自动滚动. - **任务执行流程优化**. 允许用户在触发任务后跳转到该任务详情页. - **任务展示优化**. 在爬虫详情页的最近任务表格中加入了“参数”列. [#295](https://github.com/crawlab-team/crawlab/issues/295) +- **爬虫列表优化**. 在爬虫列表页加入"更新时间"和"创建时间". [#505](https://github.com/crawlab-team/crawlab/issues/505) ### Bug 修复 - **定时任务配置失去焦点**. [#519](https://github.com/crawlab-team/crawlab/issues/519) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec1e280..ee8de024 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - **Log Management Optimization**. Log search, error highlight, auto-scrolling. - **Task Execution Process Optimization**. Allow users to be redirected to task detail page after triggering a task. - **Task Display Optimization**. Added "Param" in the Latest Tasks table in the spider detail page. [#295](https://github.com/crawlab-team/crawlab/issues/295) +- **Spider List Optimization**. Added "Update Time" and "Create Time" in spider list page. ### Bug Fixes - **Lost Focus in Schedule Configuration**. [#519](https://github.com/crawlab-team/crawlab/issues/519) From c75bd604fa0cae29f058226b144807e827a17b6c Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 13:45:12 +0800 Subject: [PATCH 17/24] updated version --- backend/conf/config.yml | 2 +- frontend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 385834bd..1fa80aeb 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -35,7 +35,7 @@ task: workers: 4 other: tmppath: "/tmp" -version: 0.4.5 +version: 0.4.6 setting: allowRegister: "N" notification: diff --git a/frontend/package.json b/frontend/package.json index 638189e8..197bde7f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.4.5", + "version": "0.4.6", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", From 5b6669de5dc5cb8119253745b5cf65742ae3ec2c Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 13:49:30 +0800 Subject: [PATCH 18/24] updated CHANGELOG --- CHANGELOG-zh.md | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index 6ff633ce..0c8c3646 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,4 +1,4 @@ -# 0.4.6 (未知) +# 0.4.6 (2020-02-13) ### 功能 / 优化 - **Node.js SDK **. 用户可以将 SDK 应用到他们的 Node.js 爬虫中. - **日志管理优化**. 日志搜索,错误高亮,自动滚动. diff --git a/CHANGELOG.md b/CHANGELOG.md index ee8de024..da4c2f77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 0.4.6 (unkown) +# 0.4.6 (2020-02-13) ### Features / Enhancement - **SDK for Node.js**. Users can apply SDK in their Node.js spiders. - **Log Management Optimization**. Log search, error highlight, auto-scrolling. From 025db544adee9f2c303e213de936a74c14c1b05a Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 14:21:03 +0800 Subject: [PATCH 19/24] fixed SpiderList view --- frontend/src/components/InfoView/SpiderInfoView.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 48b70a12..801c0fce 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -23,6 +23,7 @@ v-model="spiderForm.project_id" :placeholder="$t('Project')" filterable + :disabled="isView" > - + From 168e2a8f4c5a7ea456e0d19a7965cf51f9002f5b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 14:53:47 +0800 Subject: [PATCH 20/24] fixed log refresh issue --- frontend/src/store/modules/task.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 563fc907..595ec7b0 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -141,7 +141,6 @@ const actions = { }) }, getTaskLog ({ state, commit }, id) { - commit('SET_TASK_LOG', '') return request.get(`/tasks/${id}/log`) .then(response => { commit('SET_TASK_LOG', response.data.data) From 6260298f29ea62b5c652adacdfcabb93f99bd189 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 14:59:09 +0800 Subject: [PATCH 21/24] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/components/InfoView/TaskInfoView.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index 7e08af8d..b85fef40 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -127,6 +127,7 @@ export default { }, onClickLogWithErrors () { this.$emit('click-log') + this.$st.sendEv('任务详情', '概览', '点击日志错误') } } } From 769be8842879c52c6b1a7731f21f7b11e7ce13cb Mon Sep 17 00:00:00 2001 From: marvzhang Date: Thu, 13 Feb 2020 17:04:26 +0800 Subject: [PATCH 22/24] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E9=A1=B5=E9=9D=A2?= =?UTF-8?q?=E5=8A=A0=E8=BD=BDplaceholder?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/favicon.ico | Bin 1150 -> 0 bytes frontend/index.html | 20 ---- frontend/public/index.html | 133 ++++++++++++++++++++++++++- frontend/src/views/layout/Layout.vue | 1 + frontend/src/views/login/index.vue | 2 +- 5 files changed, 133 insertions(+), 23 deletions(-) delete mode 100644 frontend/favicon.ico delete mode 100644 frontend/index.html diff --git a/frontend/favicon.ico b/frontend/favicon.ico deleted file mode 100644 index 12b5c475b32588363c7b7d59e4a569bb5f881303..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1150 zcmbW0J4;;R%BVgnPvIWPrJiCaFj9;J6Hp(d<1gmh+5$WIQ|KIX-ei1WBkt-9OE@_PC8b| zlUV@{=77uJZu@ftXkESrF=G4g-hr>JK(Rm-fw|5w4j5}-jx*1(+!NW0WU1YVUmNw9 zlhr6D@~vioi&J3hThgaJx{hKFmz-CgWG(NXi;ekTi;v;82LImrxbv!YXbZ>5*TDA< zV5%kQ6C2lW`kmKKHO#ifal>_hYOnUAH?}*Q-oL%9$K`Xt{tU1&0sK97oAz;}KIFsw z`{(?;0@gnR`73ACYd_C+xb`<83+`KResl&oFQ?Mycf1jp{Q%5&@f`EN2AMakg~tZx zSG*ee;QnwdB;t~czA0ayVx3C&^ab~q{sGQa+gu>O%d+Wir=Y7Uv%{xm3s20tOI)dd L#7pQ`tQfJkBIvd6 diff --git a/frontend/index.html b/frontend/index.html deleted file mode 100644 index 5066906e..00000000 --- a/frontend/index.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - Crawlab - - - -
- - - diff --git a/frontend/public/index.html b/frontend/public/index.html index ccf318fc..191da08f 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -9,12 +9,141 @@ + Crawlab - -
+
+
+
+

+ C + R + A + W + L + A + B +

+
+
+ Loading... +
+
+
+ diff --git a/frontend/src/views/layout/Layout.vue b/frontend/src/views/layout/Layout.vue index 2b182d31..119c7337 100644 --- a/frontend/src/views/layout/Layout.vue +++ b/frontend/src/views/layout/Layout.vue @@ -60,6 +60,7 @@ export default { position: relative; height: 100%; width: 100%; + background: white; &.mobile.openSidebar { position: fixed; diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index b3559e49..ab6cd54f 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -362,7 +362,7 @@ const initCanvas = () => {