diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md index e0b5693f..cd8e09e8 100644 --- a/CHANGELOG-zh.md +++ b/CHANGELOG-zh.md @@ -1,3 +1,16 @@ +# 0.4.6 (2020-02-13) +### 功能 / 优化 +- **Node.js SDK **. 用户可以将 SDK 应用到他们的 Node.js 爬虫中. +- **日志管理优化**. 日志搜索,错误高亮,自动滚动. +- **任务执行流程优化**. 允许用户在触发任务后跳转到该任务详情页. +- **任务展示优化**. 在爬虫详情页的最近任务表格中加入了“参数”列. [#295](https://github.com/crawlab-team/crawlab/issues/295) +- **爬虫列表优化**. 在爬虫列表页加入"更新时间"和"创建时间". [#505](https://github.com/crawlab-team/crawlab/issues/505) +- **页面加载展位器**. + +### Bug 修复 +- **定时任务配置失去焦点**. [#519](https://github.com/crawlab-team/crawlab/issues/519) +- **无法用 CLI 工具上传爬虫**. [#524](https://github.com/crawlab-team/crawlab/issues/524) + # 0.4.5 (2020-02-03) ### 功能 / 优化 - **交互式教程**. 引导用户了解 Crawlab 的主要功能. diff --git a/CHANGELOG.md b/CHANGELOG.md index a4bf4fa4..707dd09d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +# 0.4.6 (2020-02-13) +### Features / Enhancement +- **SDK for Node.js**. Users can apply SDK in their Node.js spiders. +- **Log Management Optimization**. Log search, error highlight, auto-scrolling. +- **Task Execution Process Optimization**. Allow users to be redirected to task detail page after triggering a task. +- **Task Display Optimization**. Added "Param" in the Latest Tasks table in the spider detail page. [#295](https://github.com/crawlab-team/crawlab/issues/295) +- **Spider List Optimization**. Added "Update Time" and "Create Time" in spider list page. +- **Page Loading Placeholder**. + +### Bug Fixes +- **Lost Focus in Schedule Configuration**. [#519](https://github.com/crawlab-team/crawlab/issues/519) +- **Unable to Upload Spider using CLI**. [#524](https://github.com/crawlab-team/crawlab/issues/524) + # 0.4.5 (2020-02-03) ### Features / Enhancement - **Interactive Tutorial**. Guide users through the main functionalities of Crawlab. diff --git a/Dockerfile b/Dockerfile index 40757d51..82b944ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend +# timezone environment +ENV TZ Asia/Shanghai + # frontend port EXPOSE 8080 diff --git a/Dockerfile.local b/Dockerfile.local index 240d84e4..ea4a1ff9 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -52,6 +52,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d # working directory WORKDIR /app/backend +# timezone environment +ENV TZ Asia/Shanghai + # frontend port EXPOSE 8080 diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 385834bd..1fa80aeb 100644 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -35,7 +35,7 @@ task: workers: 4 other: tmppath: "/tmp" -version: 0.4.5 +version: 0.4.6 setting: allowRegister: "N" notification: diff --git a/backend/routes/task.go b/backend/routes/task.go index 07105f2d..d1071881 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -100,6 +100,9 @@ func PutTask(c *gin.Context) { return } + // 任务ID + var taskIds []string + if reqBody.RunType == constants.RunTypeAllNodes { // 所有节点 nodes, err := model.GetNodeList(nil) @@ -115,10 +118,13 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + + taskIds = append(taskIds, id) } } else if reqBody.RunType == constants.RunTypeRandom { // 随机 @@ -127,10 +133,12 @@ func PutTask(c *gin.Context) { Param: reqBody.Param, UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + taskIds = append(taskIds, id) } else if reqBody.RunType == constants.RunTypeSelectedNodes { // 指定节点 for _, nodeId := range reqBody.NodeIds { @@ -141,16 +149,19 @@ func PutTask(c *gin.Context) { UserId: services.GetCurrentUser(c).Id, } - if err := services.AddTask(t); err != nil { + id, err := services.AddTask(t); + if err != nil { HandleError(http.StatusInternalServerError, c, err) return } + taskIds = append(taskIds, id) } } else { HandleErrorF(http.StatusInternalServerError, c, "invalid run_type") return } - HandleSuccess(c) + + HandleSuccessData(c, taskIds) } func DeleteTaskByStatus(c *gin.Context) { diff --git a/backend/services/schedule.go b/backend/services/schedule.go index a179b50f..1bf70e8a 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -37,7 +37,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { return } } @@ -49,7 +49,7 @@ func AddScheduleTask(s model.Schedule) func() { Param: s.Param, UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() return @@ -65,7 +65,7 @@ func AddScheduleTask(s model.Schedule) func() { UserId: s.UserId, } - if err := AddTask(t); err != nil { + if _, err := AddTask(t); err != nil { return } } diff --git a/backend/services/task.go b/backend/services/task.go index f911159d..c71d344f 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -666,7 +666,7 @@ func CancelTask(id string) (err error) { return nil } -func AddTask(t model.Task) error { +func AddTask(t model.Task) (string, error) { // 生成任务ID id := uuid.NewV4() t.Id = id.String() @@ -683,17 +683,17 @@ func AddTask(t model.Task) error { if err := model.AddTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() - return err + return t.Id, err } // 加入任务队列 if err := AssignTask(t); err != nil { log.Errorf(err.Error()) debug.PrintStack() - return err + return t.Id, err } - return nil + return t.Id, nil } func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string { diff --git a/docker-compose.yml b/docker-compose.yml index 637083b2..9d5acbb3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: depends_on: - mongo - redis + # volumes: + # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 worker: image: tikazyq/crawlab:latest container_name: worker @@ -45,6 +47,8 @@ services: depends_on: - mongo - redis + # volumes: + # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化 mongo: image: mongo:latest restart: always @@ -55,7 +59,7 @@ services: redis: image: redis:latest restart: always - # command: redis --requirepass "password" # set redis password 设置 Redis 密码 + # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码 # volumes: # - "/opt/crawlab/redis/data:/data" # make data persistent 持久化 # ports: diff --git a/frontend/favicon.ico b/frontend/favicon.ico deleted file mode 100644 index 12b5c475..00000000 Binary files a/frontend/favicon.ico and /dev/null differ diff --git a/frontend/index.html b/frontend/index.html deleted file mode 100644 index 5066906e..00000000 --- a/frontend/index.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - Crawlab - - - -
- - - diff --git a/frontend/package.json b/frontend/package.json index 638189e8..197bde7f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.4.5", + "version": "0.4.6", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/public/index.html b/frontend/public/index.html index ccf318fc..0bb56a58 100644 --- a/frontend/public/index.html +++ b/frontend/public/index.html @@ -9,11 +9,140 @@ + Crawlab - +
+
+
+

+ C + R + A + W + L + A + B +

+
+
+ Loading... +
+
+
diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index d25bfaaa..6739ae9e 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -30,13 +30,22 @@ - - 我已阅读并同意 《免责声明》 所有内容 +
+ + 我已阅读并同意 《免责声明》 所有内容 +
+
+ + 跳转到任务详情页 +
+
+ @@ -64,7 +73,8 @@ export default { param: '', nodeList: [] }, - isAllowDisclaimer: true + isAllowDisclaimer: true, + isRedirect: true } }, methods: { @@ -72,20 +82,27 @@ export default { this.$emit('close') }, onConfirm () { - this.$refs['form'].validate(res => { - if (!res) return + this.$refs['form'].validate(async valid => { + if (!valid) return - this.$store.dispatch('spider/crawlSpider', { + const res = await this.$store.dispatch('spider/crawlSpider', { spiderId: this.spiderId, nodeIds: this.form.nodeIds, param: this.form.param, runType: this.form.runType }) - .then(() => { - this.$message.success(this.$t('A task has been scheduled successfully')) - }) + + const id = res.data.data[0] + + this.$message.success(this.$t('A task has been scheduled successfully')) + this.$emit('close') this.$st.sendEv('爬虫确认', '确认运行', this.form.runType) + + if (this.isRedirect) { + this.$router.push('/tasks/' + id) + this.$st.sendEv('爬虫确认', '跳转到任务详情') + } }) }, onClickDisclaimer () { diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 9e2f5de1..d4d83119 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -131,12 +131,16 @@
- {{$t('Run')}} + + {{$t('Run')}} + - {{$t('Save')}} + + {{$t('Save')}} +
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index 48b70a12..801c0fce 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -23,6 +23,7 @@ v-model="spiderForm.project_id" :placeholder="$t('Project')" filterable + :disabled="isView" > - + diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index 34318d84..b85fef40 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -11,6 +11,24 @@ + + + + {{$t('Log with errors')}} + + + + + {{$t('Empty results')}} + @@ -28,7 +46,7 @@ - + @@ -37,7 +55,7 @@ - + @@ -51,7 +69,9 @@ - {{$t('Stop')}} + + {{$t('Stop')}} + @@ -59,7 +79,8 @@ diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue index 1f4f0c89..befe2ab3 100644 --- a/frontend/src/components/Status/StatusTag.vue +++ b/frontend/src/components/Status/StatusTag.vue @@ -1,5 +1,6 @@ @@ -38,6 +39,16 @@ export default { return s.label } return 'NA' + }, + icon () { + if (this.status === 'finished') { + return 'el-icon-check' + } else if (this.status === 'running') { + return 'el-icon-loading' + } else if (this.status === 'error') { + return 'el-icon-error' + } + return '' } } } diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue index 3f7693f3..a95a08d5 100644 --- a/frontend/src/components/TableView/GeneralTableView.vue +++ b/frontend/src/components/TableView/GeneralTableView.vue @@ -8,7 +8,12 @@