git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

加入日志分页

Browse Source
This commit is contained in:
marvzhang
2020-04-15 13:48:27 +08:00
parent 909420a14e
commit 94e1d6c431
8 changed files with 169 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/model/log.go
View File

@@ -77,3 +77,16 @@ func GetLogItemList(filter interface{}, skip int, limit int, sortStr string) ([]
return logItems, nil
}
func GetLogItemTotal(filter interface{}) (int, error) {
s, c := database.GetCol("logs")
defer s.Close()
total, err := c.Find(filter).Count()
if err != nil {
debug.PrintStack()
return total, err
}
return total, nil
}

23
backend/model/task.go
View File

@@ -109,17 +109,30 @@ func (t *Task) GetResults(pageNum int, pageSize int) (results []interface{}, tot
return
}
func (t *Task) GetLogItems() (logItems []LogItem, err error) {
func (t *Task) GetLogItems(keyword string, page int, pageSize int) (logItems []LogItem, logTotal int, err error) {
query := bson.M{
"task_id": t.Id,
}
logItems, err = GetLogItemList(query, 0, constants.Infinite, "+_id")
if err != nil {
return logItems, err
if keyword != "" {
query["msg"] = bson.M{
"$regex": bson.RegEx{
Pattern: keyword,
Options: "i",
},
}
}
return logItems, nil
logItems, err = GetLogItemList(query, (page - 1) * pageSize, pageSize, "+_id")
if err != nil {
return logItems, logTotal, err
}
logTotal, err = GetLogItemTotal(query)
if err != nil {
return logItems, logTotal, err
}
return logItems, logTotal, nil
}
func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Task, error) {

21
backend/routes/task.go
View File

@@ -234,13 +234,28 @@ func DeleteTask(c *gin.Context) {
}
func GetTaskLog(c *gin.Context) {
type RequestData struct {
PageNum int `form:"page_num"`
PageSize int `form:"page_size"`
Keyword string `form:"keyword"`
}
id := c.Param("id")
logItems, err := services.GetTaskLog(id)
var reqData RequestData
if err := c.ShouldBindQuery(&reqData); err != nil {
HandleErrorF(http.StatusBadRequest, c, "invalid request")
return
}
logItems, logTotal, err := services.GetTaskLog(id, reqData.Keyword, reqData.PageNum, reqData.PageSize)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
HandleSuccessData(c, logItems)
c.JSON(http.StatusOK, ListResponse{
Status: "ok",
Message: "success",
Data: logItems,
Total: logTotal,
})
}
func GetTaskResults(c *gin.Context) {
@@ -364,4 +379,4 @@ func RestartTask(c *gin.Context) {
return
}
HandleSuccess(c)
}
}

51
backend/services/task.go
View File

@@ -622,60 +622,19 @@ func SpiderFileCheck(t model.Task, spider model.Spider) error {
return nil
}
func GetTaskLog(id string) (logItems []model.LogItem, err error) {
func GetTaskLog(id string, keyword string, page int, pageSize int) (logItems []model.LogItem, logTotal int, err error) {
task, err := model.GetTask(id)
if err != nil {
return
}
logItems, err = task.GetLogItems()
// TODO: 日志分页
logItems, logTotal, err = task.GetLogItems(keyword, page, pageSize)
if err != nil {
return logItems, err
return logItems, logTotal, err
}
return logItems, nil
//if IsMasterNode(task.NodeId.Hex()) {
// if !utils.Exists(task.LogPath) {
// fileDir, err := MakeLogDir(task)
//
// if err != nil {
// log.Errorf(err.Error())
// }
//
// fileP := GetLogFilePaths(fileDir, task)
//
// // 获取日志文件路径
// fLog, err := os.Create(fileP)
// defer fLog.Close()
// if err != nil {
// log.Errorf("create task log file error: %s", fileP)
// debug.PrintStack()
// }
// task.LogPath = fileP
// if err := task.Save(); err != nil {
// log.Errorf(err.Error())
// debug.PrintStack()
// }
//
// }
// // 若为主节点，获取本机日志
// logBytes, err := model.GetLocalLog(task.LogPath)
// if err != nil {
// log.Errorf(err.Error())
// logStr = err.Error()
// } else {
// logStr = utils.BytesToString(logBytes)
// }
// return logStr, err
//}
//// 若不为主节点，获取远端日志
//logStr, err = GetRemoteLog(task)
//if err != nil {
// log.Errorf(err.Error())
//
//}
//return logStr, err
return logItems, logTotal, nil
}
func CancelTask(id string) (err error) {

7
frontend/src/components/InfoView/TaskInfoView.vue
View File

@@ -79,8 +79,7 @@
<script>
import {
mapState,
mapGetters
mapState
} from 'vuex'
import StatusTag from '../Status/StatusTag'
import dayjs from 'dayjs'
@@ -91,9 +90,7 @@ export default {
computed: {
...mapState('task', [
'taskForm',
'taskLog'
]),
...mapGetters('task', [
'taskLog',
'errorLogData'
]),
isRunning () {

78
frontend/src/components/ScrollView/LogView.vue
View File

@@ -13,14 +13,31 @@
{{$t('Auto-Scroll')}}
</el-button>
<el-input
v-model="searchString"
v-model="logKeyword"
size="small"
suffix-icon="el-icon-search"
:placeholder="$t('Search Log')"
style="width: 240px; margin-right: 10px"
/>
<el-button
size="small"
type="primary"
icon="el-icon-search"
@click="onSearchLog"
>
{{$t('Search Log')}}
</el-button>
</div>
<div class="right">
<el-pagination
size="small"
:total="taskLogTotal"
:current-page.sync="taskLogPage"
:page-sizes="[1000, 2000, 5000, 10000]"
:page-size.sync="taskLogPageSize"
:pager-count="3"
layout="sizes, prev, pager, next"
/>
<el-badge
v-if="errorLogData.length > 0"
:value="errorLogData.length"
@@ -68,11 +85,11 @@
:class="currentLogIndex === item.index ? 'active' : ''"
@click="onClickError(item)"
>
<span class="line-no">
{{item.index}}
</span>
<!-- <span class="line-no">-->
<!-- {{item.index}}-->
<!-- </span>-->
<span class="line-content">
{{item.data}}
{{item.msg}}
</span>
</li>
</ul>
@@ -119,7 +136,8 @@ export default {
},
computed: {
...mapState('task', [
'taskForm'
'taskForm',
'taskLogTotal'
]),
...mapGetters('task', [
'logData',
@@ -133,6 +151,30 @@ export default {
this.$store.commit('task/SET_CURRENT_LOG_INDEX', value)
}
},
logKeyword: {
get () {
return this.$store.state.task.logKeyword
},
set (value) {
this.$store.commit('task/SET_LOG_KEYWORD', value)
}
},
taskLogPage: {
get () {
return this.$store.state.task.taskLogPage
},
set (value) {
this.$store.commit('task/SET_TASK_LOG_PAGE', value)
}
},
taskLogPageSize: {
get () {
return this.$store.state.task.taskLogPageSize
},
set (value) {
this.$store.commit('task/SET_TASK_LOG_PAGE_SIZE', value)
}
},
filteredLogData () {
return this.logData.filter(d => {
if (!this.searchString) return true
@@ -145,8 +187,13 @@ export default {
}
},
watch: {
searchString () {
this.$st.sendEv('任务详情', '日志', '搜索日志')
taskLogPage () {
this.$emit('search')
this.$st.sendEv('任务详情', '日志', '改变页数')
},
taskLogPageSize () {
this.$emit('search')
this.$st.sendEv('任务详情', '日志', '改变日志每页条数')
}
},
methods: {
@@ -160,7 +207,7 @@ export default {
index: logItem.index,
logItem,
data: isAnsi ? convert.toHtml(logItem.data) : logItem.data,
searchString: this.searchString,
searchString: this.logKeyword,
active: logItem.active,
isAnsi
}
@@ -211,6 +258,10 @@ export default {
setTimeout(() => {
clearInterval(handle)
}, 500)
},
onSearchLog () {
this.$emit('search')
this.$st.sendEv('任务详情', '日志', '搜索日志')
}
},
mounted () {
@@ -319,4 +370,13 @@ export default {
width: calc(100% - 70px);
padding-left: 10px;
}
.right {
display: flex;
align-items: center;
}
.right .el-pagination {
margin-right: 10px;
}
</style>

43
frontend/src/store/modules/task.js
View File

@@ -7,6 +7,9 @@ const state = {
taskListTotalCount: 0,
taskForm: {},
taskLog: [],
taskLogTotal: 0,
taskLogPage: 1,
taskLogPageSize: 5000,
currentLogIndex: 0,
taskResultsData: [],
taskResultsColumns: [],
@@ -21,6 +24,9 @@ const state = {
// pagination
pageNum: 1,
pageSize: 10,
// log
logKeyword: '',
errorLogData: [],
// results
resultsPageNum: 1,
resultsPageSize: 10
@@ -63,8 +69,11 @@ const getters = {
return data
},
errorLogData (state, getters) {
return getters.logData.filter(d => {
return d.data.match(utils.log.errorRegex)
const idxList = getters.logData.map(d => d._id)
return state.errorLogData.map(d => {
const idx = idxList.indexOf(d._id)
d.index = getters.logData[idx].index
return d
})
}
}
@@ -79,6 +88,9 @@ const mutations = {
SET_TASK_LOG (state, value) {
state.taskLog = value
},
SET_TASK_LOG_TOTAL (state, value) {
state.taskLogTotal = value
},
SET_CURRENT_LOG_INDEX (state, value) {
state.currentLogIndex = value
},
@@ -105,6 +117,18 @@ const mutations = {
},
SET_TASK_RESULTS_TOTAL_COUNT (state, value) {
state.taskResultsTotalCount = value
},
SET_LOG_KEYWORD (state, value) {
state.logKeyword = value
},
SET_ERROR_LOG_DATA (state, value) {
state.errorLogData = value
},
SET_TASK_LOG_PAGE (state, value) {
state.taskLogPage = value
},
SET_TASK_LOG_PAGE_SIZE (state, value) {
state.taskLogPageSize = value
}
}
@@ -149,10 +173,21 @@ const actions = {
dispatch('getTaskList')
})
},
getTaskLog ({ state, commit }, id) {
return request.get(`/tasks/${id}/log`)
getTaskLog ({ state, commit }, { id, keyword }) {
return request.get(`/tasks/${id}/log`, {
keyword,
page_num: state.taskLogPage,
page_size: state.taskLogPageSize
})
.then(response => {
commit('SET_TASK_LOG', response.data.data || [])
commit('SET_TASK_LOG_TOTAL', response.data.total || 0)
})
},
getTaskErrorLog ({ state, commit }, id) {
return request.get(`/tasks/${id}/log`, { keyword: utils.log.errorRegex.source })
.then(response => {
commit('SET_ERROR_LOG_DATA', response.data.data || [])
})
},
getTaskResults ({ state, commit }, id) {

8
frontend/src/views/task/TaskDetail.vue
View File

@@ -15,7 +15,7 @@
<task-overview @click-log="activeTabName = 'log'"/>
</el-tab-pane>
<el-tab-pane :label="$t('Log')" name="log">
<log-view/>
<log-view @search="getTaskLog"/>
</el-tab-pane>
<el-tab-pane :label="$t('Results')" name="results">
<div class="button-group">
@@ -136,7 +136,8 @@ export default {
'taskForm',
'taskResultsData',
'taskResultsTotalCount',
'taskLog'
'taskLog',
'logKeyword'
]),
...mapGetters('task', [
'taskResultsColumns'
@@ -185,7 +186,8 @@ export default {
this.$st.sendEv('任务详情', '结果', '下载CSV')
},
getTaskLog () {
this.$store.dispatch('task/getTaskLog', this.$route.params.id)
this.$store.dispatch('task/getTaskLog', { id: this.$route.params.id, keyword: this.logKeyword })
this.$store.dispatch('task/getTaskErrorLog', this.$route.params.id)
}
},
created () {