diff --git a/CHANGELOG-zh.md b/CHANGELOG-zh.md
index e0b5693f..cd8e09e8 100644
--- a/CHANGELOG-zh.md
+++ b/CHANGELOG-zh.md
@@ -1,3 +1,16 @@
+# 0.4.6 (2020-02-13)
+### 功能 / 优化
+- **Node.js SDK **. 用户可以将 SDK 应用到他们的 Node.js 爬虫中.
+- **日志管理优化**. 日志搜索,错误高亮,自动滚动.
+- **任务执行流程优化**. 允许用户在触发任务后跳转到该任务详情页.
+- **任务展示优化**. 在爬虫详情页的最近任务表格中加入了“参数”列. [#295](https://github.com/crawlab-team/crawlab/issues/295)
+- **爬虫列表优化**. 在爬虫列表页加入"更新时间"和"创建时间". [#505](https://github.com/crawlab-team/crawlab/issues/505)
+- **页面加载展位器**.
+
+### Bug 修复
+- **定时任务配置失去焦点**. [#519](https://github.com/crawlab-team/crawlab/issues/519)
+- **无法用 CLI 工具上传爬虫**. [#524](https://github.com/crawlab-team/crawlab/issues/524)
+
# 0.4.5 (2020-02-03)
### 功能 / 优化
- **交互式教程**. 引导用户了解 Crawlab 的主要功能.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4bf4fa4..707dd09d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+# 0.4.6 (2020-02-13)
+### Features / Enhancement
+- **SDK for Node.js**. Users can apply SDK in their Node.js spiders.
+- **Log Management Optimization**. Log search, error highlight, auto-scrolling.
+- **Task Execution Process Optimization**. Allow users to be redirected to task detail page after triggering a task.
+- **Task Display Optimization**. Added "Param" in the Latest Tasks table in the spider detail page. [#295](https://github.com/crawlab-team/crawlab/issues/295)
+- **Spider List Optimization**. Added "Update Time" and "Create Time" in spider list page.
+- **Page Loading Placeholder**.
+
+### Bug Fixes
+- **Lost Focus in Schedule Configuration**. [#519](https://github.com/crawlab-team/crawlab/issues/519)
+- **Unable to Upload Spider using CLI**. [#524](https://github.com/crawlab-team/crawlab/issues/524)
+
# 0.4.5 (2020-02-03)
### Features / Enhancement
- **Interactive Tutorial**. Guide users through the main functionalities of Crawlab.
diff --git a/Dockerfile b/Dockerfile
index 40757d51..82b944ec 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -54,6 +54,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend
+# timezone environment
+ENV TZ Asia/Shanghai
+
# frontend port
EXPOSE 8080
diff --git a/Dockerfile.local b/Dockerfile.local
index 240d84e4..ea4a1ff9 100644
--- a/Dockerfile.local
+++ b/Dockerfile.local
@@ -52,6 +52,9 @@ COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
# working directory
WORKDIR /app/backend
+# timezone environment
+ENV TZ Asia/Shanghai
+
# frontend port
EXPOSE 8080
diff --git a/backend/conf/config.yml b/backend/conf/config.yml
index 385834bd..1fa80aeb 100644
--- a/backend/conf/config.yml
+++ b/backend/conf/config.yml
@@ -35,7 +35,7 @@ task:
workers: 4
other:
tmppath: "/tmp"
-version: 0.4.5
+version: 0.4.6
setting:
allowRegister: "N"
notification:
diff --git a/backend/routes/task.go b/backend/routes/task.go
index 07105f2d..d1071881 100644
--- a/backend/routes/task.go
+++ b/backend/routes/task.go
@@ -100,6 +100,9 @@ func PutTask(c *gin.Context) {
return
}
+ // 任务ID
+ var taskIds []string
+
if reqBody.RunType == constants.RunTypeAllNodes {
// 所有节点
nodes, err := model.GetNodeList(nil)
@@ -115,10 +118,13 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
- if err := services.AddTask(t); err != nil {
+ id, err := services.AddTask(t);
+ if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
+
+ taskIds = append(taskIds, id)
}
} else if reqBody.RunType == constants.RunTypeRandom {
// 随机
@@ -127,10 +133,12 @@ func PutTask(c *gin.Context) {
Param: reqBody.Param,
UserId: services.GetCurrentUser(c).Id,
}
- if err := services.AddTask(t); err != nil {
+ id, err := services.AddTask(t);
+ if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
+ taskIds = append(taskIds, id)
} else if reqBody.RunType == constants.RunTypeSelectedNodes {
// 指定节点
for _, nodeId := range reqBody.NodeIds {
@@ -141,16 +149,19 @@ func PutTask(c *gin.Context) {
UserId: services.GetCurrentUser(c).Id,
}
- if err := services.AddTask(t); err != nil {
+ id, err := services.AddTask(t);
+ if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
+ taskIds = append(taskIds, id)
}
} else {
HandleErrorF(http.StatusInternalServerError, c, "invalid run_type")
return
}
- HandleSuccess(c)
+
+ HandleSuccessData(c, taskIds)
}
func DeleteTaskByStatus(c *gin.Context) {
diff --git a/backend/services/schedule.go b/backend/services/schedule.go
index a179b50f..1bf70e8a 100644
--- a/backend/services/schedule.go
+++ b/backend/services/schedule.go
@@ -37,7 +37,7 @@ func AddScheduleTask(s model.Schedule) func() {
UserId: s.UserId,
}
- if err := AddTask(t); err != nil {
+ if _, err := AddTask(t); err != nil {
return
}
}
@@ -49,7 +49,7 @@ func AddScheduleTask(s model.Schedule) func() {
Param: s.Param,
UserId: s.UserId,
}
- if err := AddTask(t); err != nil {
+ if _, err := AddTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
@@ -65,7 +65,7 @@ func AddScheduleTask(s model.Schedule) func() {
UserId: s.UserId,
}
- if err := AddTask(t); err != nil {
+ if _, err := AddTask(t); err != nil {
return
}
}
diff --git a/backend/services/task.go b/backend/services/task.go
index f911159d..c71d344f 100644
--- a/backend/services/task.go
+++ b/backend/services/task.go
@@ -666,7 +666,7 @@ func CancelTask(id string) (err error) {
return nil
}
-func AddTask(t model.Task) error {
+func AddTask(t model.Task) (string, error) {
// 生成任务ID
id := uuid.NewV4()
t.Id = id.String()
@@ -683,17 +683,17 @@ func AddTask(t model.Task) error {
if err := model.AddTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
- return err
+ return t.Id, err
}
// 加入任务队列
if err := AssignTask(t); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
- return err
+ return t.Id, err
}
- return nil
+ return t.Id, nil
}
func GetTaskEmailMarkdownContent(t model.Task, s model.Spider) string {
diff --git a/docker-compose.yml b/docker-compose.yml
index 637083b2..9d5acbb3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,6 +35,8 @@ services:
depends_on:
- mongo
- redis
+ # volumes:
+ # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
worker:
image: tikazyq/crawlab:latest
container_name: worker
@@ -45,6 +47,8 @@ services:
depends_on:
- mongo
- redis
+ # volumes:
+ # - "/var/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
mongo:
image: mongo:latest
restart: always
@@ -55,7 +59,7 @@ services:
redis:
image: redis:latest
restart: always
- # command: redis --requirepass "password" # set redis password 设置 Redis 密码
+ # command: redis-server --requirepass "password" # set redis password 设置 Redis 密码
# volumes:
# - "/opt/crawlab/redis/data:/data" # make data persistent 持久化
# ports:
diff --git a/frontend/favicon.ico b/frontend/favicon.ico
deleted file mode 100644
index 12b5c475..00000000
Binary files a/frontend/favicon.ico and /dev/null differ
diff --git a/frontend/index.html b/frontend/index.html
deleted file mode 100644
index 5066906e..00000000
--- a/frontend/index.html
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
- Crawlab
-
-
-
-
-
-
-
diff --git a/frontend/package.json b/frontend/package.json
index 638189e8..197bde7f 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -1,6 +1,6 @@
{
"name": "crawlab",
- "version": "0.4.5",
+ "version": "0.4.6",
"private": true,
"scripts": {
"serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development",
diff --git a/frontend/public/index.html b/frontend/public/index.html
index ccf318fc..0bb56a58 100644
--- a/frontend/public/index.html
+++ b/frontend/public/index.html
@@ -9,11 +9,140 @@
+
Crawlab
-
+
+
+
+
+ C
+ R
+ A
+ W
+ L
+ A
+ B
+
+
+
+ Loading...
+
+
+
diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue
index d25bfaaa..6739ae9e 100644
--- a/frontend/src/components/Common/CrawlConfirmDialog.vue
+++ b/frontend/src/components/Common/CrawlConfirmDialog.vue
@@ -30,13 +30,22 @@
-
- 我已阅读并同意 《免责声明》 所有内容
+
+
+
+ 跳转到任务详情页
+
+
+
{{$t('Cancel')}}
- {{$t('Confirm')}}
+ {{$t('Confirm')}}
+
@@ -64,7 +73,8 @@ export default {
param: '',
nodeList: []
},
- isAllowDisclaimer: true
+ isAllowDisclaimer: true,
+ isRedirect: true
}
},
methods: {
@@ -72,20 +82,27 @@ export default {
this.$emit('close')
},
onConfirm () {
- this.$refs['form'].validate(res => {
- if (!res) return
+ this.$refs['form'].validate(async valid => {
+ if (!valid) return
- this.$store.dispatch('spider/crawlSpider', {
+ const res = await this.$store.dispatch('spider/crawlSpider', {
spiderId: this.spiderId,
nodeIds: this.form.nodeIds,
param: this.form.param,
runType: this.form.runType
})
- .then(() => {
- this.$message.success(this.$t('A task has been scheduled successfully'))
- })
+
+ const id = res.data.data[0]
+
+ this.$message.success(this.$t('A task has been scheduled successfully'))
+
this.$emit('close')
this.$st.sendEv('爬虫确认', '确认运行', this.form.runType)
+
+ if (this.isRedirect) {
+ this.$router.push('/tasks/' + id)
+ this.$st.sendEv('爬虫确认', '跳转到任务详情')
+ }
})
},
onClickDisclaimer () {
diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue
index 9e2f5de1..d4d83119 100644
--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -131,12 +131,16 @@
diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue
index 48b70a12..801c0fce 100644
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -23,6 +23,7 @@
v-model="spiderForm.project_id"
:placeholder="$t('Project')"
filterable
+ :disabled="isView"
>
-
+
diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue
index 34318d84..b85fef40 100644
--- a/frontend/src/components/InfoView/TaskInfoView.vue
+++ b/frontend/src/components/InfoView/TaskInfoView.vue
@@ -11,6 +11,24 @@
+
+
+
+ {{$t('Log with errors')}}
+
+
+
+
+ {{$t('Empty results')}}
+
@@ -28,7 +46,7 @@
-
+
@@ -37,7 +55,7 @@
-
+
@@ -51,7 +69,9 @@
- {{$t('Stop')}}
+
+ {{$t('Stop')}}
+
@@ -59,7 +79,8 @@
diff --git a/frontend/src/components/Status/StatusTag.vue b/frontend/src/components/Status/StatusTag.vue
index 1f4f0c89..befe2ab3 100644
--- a/frontend/src/components/Status/StatusTag.vue
+++ b/frontend/src/components/Status/StatusTag.vue
@@ -1,5 +1,6 @@
-
+
+
{{$t(label)}}
@@ -38,6 +39,16 @@ export default {
return s.label
}
return 'NA'
+ },
+ icon () {
+ if (this.status === 'finished') {
+ return 'el-icon-check'
+ } else if (this.status === 'running') {
+ return 'el-icon-loading'
+ } else if (this.status === 'error') {
+ return 'el-icon-error'
+ }
+ return ''
}
}
}
diff --git a/frontend/src/components/TableView/GeneralTableView.vue b/frontend/src/components/TableView/GeneralTableView.vue
index 3f7693f3..a95a08d5 100644
--- a/frontend/src/components/TableView/GeneralTableView.vue
+++ b/frontend/src/components/TableView/GeneralTableView.vue
@@ -8,7 +8,12 @@
-
+
+
{{getString(scope.row[col])}}
@@ -68,6 +73,11 @@ export default {
}
},
methods: {
+ isUrl (value) {
+ if (!value) return false
+ if (!value.match) return false
+ return !!value.match(/^https?:\/\//)
+ },
onPageChange () {
this.$emit('page-change', { pageNum: this.pageNum, pageSize: this.pageSize })
},
diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js
index 9b1fc883..419fcd9d 100644
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -208,6 +208,12 @@ export default {
'Run Type': '运行类型',
'Random': '随机',
'Selected Nodes': '指定节点',
+ 'Search Log': '搜索日志',
+ 'Auto-Scroll': '自动滚动',
+ 'Updating log...': '正在更新日志...',
+ 'Error Count': '错误数',
+ 'Log with errors': '日志错误',
+ 'Empty results': '空结果',
// 任务列表
'Node': '节点',
diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js
index 95153be5..595ec7b0 100644
--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -1,4 +1,5 @@
import request from '../../api/request'
+import utils from '../../utils'
const state = {
// TaskList
@@ -6,6 +7,7 @@ const state = {
taskListTotalCount: 0,
taskForm: {},
taskLog: '',
+ currentLogIndex: 0,
taskResultsData: [],
taskResultsColumns: [],
taskResultsTotalCount: 0,
@@ -36,6 +38,32 @@ const getters = {
}
}
return keys
+ },
+ logData (state) {
+ const data = state.taskLog.split('\n')
+ .map((d, i) => {
+ return {
+ index: i + 1,
+ data: d,
+ active: state.currentLogIndex === i + 1
+ }
+ })
+ if (state.taskForm && state.taskForm.status === 'running') {
+ data.push({
+ index: data.length + 1,
+ data: '###LOG_END###'
+ })
+ data.push({
+ index: data.length + 1,
+ data: ''
+ })
+ }
+ return data
+ },
+ errorLogData (state, getters) {
+ return getters.logData.filter(d => {
+ return d.data.match(utils.log.errorRegex)
+ })
}
}
@@ -49,6 +77,9 @@ const mutations = {
SET_TASK_LOG (state, value) {
state.taskLog = value
},
+ SET_CURRENT_LOG_INDEX (state, value) {
+ state.currentLogIndex = value
+ },
SET_TASK_RESULTS_DATA (state, value) {
state.taskResultsData = value
},
@@ -110,7 +141,6 @@ const actions = {
})
},
getTaskLog ({ state, commit }, id) {
- commit('SET_TASK_LOG', '')
return request.get(`/tasks/${id}/log`)
.then(response => {
commit('SET_TASK_LOG', response.data.data)
diff --git a/frontend/src/utils/index.js b/frontend/src/utils/index.js
index ab33114f..9867c812 100644
--- a/frontend/src/utils/index.js
+++ b/frontend/src/utils/index.js
@@ -1,9 +1,11 @@
import stats from './stats'
import encrypt from './encrypt'
import tour from './tour'
+import log from './log'
export default {
stats,
encrypt,
- tour
+ tour,
+ log
}
diff --git a/frontend/src/utils/log.js b/frontend/src/utils/log.js
new file mode 100644
index 00000000..8b1ea0ab
--- /dev/null
+++ b/frontend/src/utils/log.js
@@ -0,0 +1,5 @@
+const regexToken = ' :,.'
+
+export default {
+ errorRegex: new RegExp(`(?:[${regexToken}]|^)((?:error|exception|traceback)s?)(?:[${regexToken}]|$)`, 'gi')
+}
diff --git a/frontend/src/views/layout/Layout.vue b/frontend/src/views/layout/Layout.vue
index 2b182d31..119c7337 100644
--- a/frontend/src/views/layout/Layout.vue
+++ b/frontend/src/views/layout/Layout.vue
@@ -60,6 +60,7 @@ export default {
position: relative;
height: 100%;
width: 100%;
+ background: white;
&.mobile.openSidebar {
position: fixed;
diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue
index b3559e49..ab6cd54f 100644
--- a/frontend/src/views/login/index.vue
+++ b/frontend/src/views/login/index.vue
@@ -362,7 +362,7 @@ const initCanvas = () => {
diff --git a/frontend/src/views/setting/Setting.vue b/frontend/src/views/setting/Setting.vue
index 3832388d..c341ccb7 100644
--- a/frontend/src/views/setting/Setting.vue
+++ b/frontend/src/views/setting/Setting.vue
@@ -31,7 +31,7 @@
-
+
diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue
index bbfbee75..3a1d38ca 100644
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -399,7 +399,8 @@ export default {
{ name: 'type', label: 'Spider Type', width: '120', sortable: true },
{ name: 'last_status', label: 'Last Status', width: '120' },
{ name: 'last_run_ts', label: 'Last Run', width: '140' },
- // { name: 'update_ts', label: 'Update Time', width: '140' },
+ { name: 'update_ts', label: 'Update Time', width: '140' },
+ { name: 'create_ts', label: 'Create Time', width: '140' },
{ name: 'remark', label: 'Remark', width: '140' }
],
spiderFormRules: {
diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue
index 39aca864..11051ed0 100644
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -10,18 +10,16 @@
-
+
-
+
-
-
-
+
-
+
{{$t('Download CSV')}}
@@ -44,7 +42,6 @@ import {
import TaskOverview from '../../components/Overview/TaskOverview'
import GeneralTableView from '../../components/TableView/GeneralTableView'
import LogView from '../../components/ScrollView/LogView'
-import request from '../../api/request'
export default {
name: 'TaskDetail',
@@ -57,7 +54,6 @@ export default {
return {
activeTabName: 'overview',
handle: undefined,
- taskLog: '',
// tutorial
tourSteps: [
@@ -139,7 +135,8 @@ export default {
...mapState('task', [
'taskForm',
'taskResultsData',
- 'taskResultsTotalCount'
+ 'taskResultsTotalCount',
+ 'taskLog'
]),
...mapGetters('task', [
'taskResultsColumns'
@@ -188,11 +185,7 @@ export default {
this.$st.sendEv('任务详情', '结果', '下载CSV')
},
getTaskLog () {
- if (this.$route.params.id) {
- request.get(`/tasks/${this.$route.params.id}/log`).then(response => {
- this.taskLog = response.data.data
- })
- }
+ this.$store.dispatch('task/getTaskLog', this.$route.params.id)
}
},
created () {
diff --git a/jenkins/master/docker-compose.yaml b/jenkins/master/docker-compose.yaml
index 75512bea..139fbb39 100644
--- a/jenkins/master/docker-compose.yaml
+++ b/jenkins/master/docker-compose.yaml
@@ -18,6 +18,8 @@ services:
depends_on:
- mongo
- redis
+ volumes:
+ - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
worker:
image: "tikazyq/crawlab:master"
environment:
@@ -30,6 +32,8 @@ services:
depends_on:
- mongo
- redis
+ volumes:
+ - "/opt/crawlab/log:/var/logs/crawlab" # log persistent 日志持久化
mongo:
image: mongo:latest
restart: always