git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-25 17:42:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

完成爬虫的分类

Browse Source
This commit is contained in:
陈景阳
2019-09-30 16:35:36 +08:00
parent a12f4ec3ac
commit 8294e7cb4b
11 changed files with 65 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
backend/entity/common.go Normal file
View File

@@ -0,0 +1,17 @@
package entity
import "strconv"
type Page struct {
Skip int
Limit int
PageNum int
PageSize int
}
func (p *Page)GetPage(pageNum string, pageSize string) {
p.PageNum, _ = strconv.Atoi(pageNum)
p.PageSize, _ = strconv.Atoi(pageSize)
p.Skip = p.PageSize * (p.PageNum - 1)
p.Limit = p.PageSize
}

4
backend/entity/spider.go
View File

@@ -1,6 +1,6 @@
package entity
type SpiderType struct {
Type string `json:"type"`
Count int `json:"count"`
Type string `json:"type" bson:"_id"`
Count int `json:"count" bson:"count"`
}

6
backend/model/spider.go
View File

@@ -256,13 +256,15 @@ func GetSpiderTypes() ([]*entity.SpiderType, error) {
s, c := database.GetCol("spiders")
defer s.Close()
group := bson.M{
"$group": bson.M{
"type": bson.M{"$sum": 1},
"_id": "$type",
"count": bson.M{"$sum": 1},
},
}
var types []*entity.SpiderType
if err := c.Pipe([]bson.M{group}).All(&types); err != nil {
if err := c.Pipe([]bson.M{ group}).All(&types); err != nil {
log.Errorf("get spider types error: %s", err.Error())
debug.PrintStack()
return nil, err

23
backend/routes/spider.go
View File

@@ -3,6 +3,7 @@ package routes
import (
"crawlab/constants"
"crawlab/database"
"crawlab/entity"
"crawlab/model"
"crawlab/services"
"crawlab/utils"
@@ -24,14 +25,22 @@ import (
)
func GetSpiderList(c *gin.Context) {
pageNumStr, _ := c.GetQuery("pageNum")
pageSizeStr, _ := c.GetQuery("pageSize")
pageNum, _ := c.GetQuery("pageNum")
pageSize, _ := c.GetQuery("pageSize")
keyword, _ := c.GetQuery("keyword")
pageNum, _ := strconv.Atoi(pageNumStr)
pageSize, _ := strconv.Atoi(pageSizeStr)
skip := pageSize * (pageNum - 1)
filter := bson.M{"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}}}
results, count, err := model.GetSpiderList(filter, skip, pageSize)
t, _ := c.GetQuery("type")
filter := bson.M{
"name": bson.M{"$regex": bson.RegEx{Pattern: keyword, Options: "im"}},
}
if t != "" {
filter["type"] = t
}
page := &entity.Page{}
page.GetPage(pageNum, pageSize)
results, count, err := model.GetSpiderList(filter, page.Skip, page.Limit)
if err != nil {
HandleError(http.StatusInternalServerError, c, err)
return

2
frontend/src/components/Common/DialogView.vue
View File

@@ -151,7 +151,7 @@ export default {
}
},
mounted () {
if (!this.spiderList || !this.spiderList.length) this.$store.dispatch('spider/getSpiderList')
// if (!this.spiderList || !this.spiderList.length) this.$store.dispatch('spider/getSpiderList')
if (!this.nodeList || !this.nodeList.length) this.$store.dispatch('node/getNodeList')
}
}

24
frontend/src/components/InfoView/SpiderInfoView.vue
View File

@@ -18,10 +18,10 @@
<el-form-item :label="$t('Spider Name')">
<el-input v-model="spiderForm.display_name" :placeholder="$t('Spider Name')" :disabled="isView"></el-input>
</el-form-item>
<el-form-item v-if="isCustomized" :label="$t('Source Folder')">
<el-form-item :label="$t('Source Folder')">
<el-input v-model="spiderForm.src" :placeholder="$t('Source Folder')" disabled></el-input>
</el-form-item>
<el-form-item v-if="isCustomized" :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
<el-form-item :label="$t('Execute Command')" prop="cmd" required :inline-message="true">
<el-input v-model="spiderForm.cmd" :placeholder="$t('Execute Command')"
:disabled="isView"></el-input>
</el-form-item>
@@ -39,10 +39,11 @@
</el-autocomplete>
</el-form-item>
<el-form-item :label="$t('Spider Type')">
<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>
<el-option value="configurable" :label="$t('Configurable')"></el-option>
<el-option value="customized" :label="$t('Customized')"></el-option>
</el-select>
<!--<el-select v-model="spiderForm.type" :placeholder="$t('Spider Type')" :disabled="true" clearable>-->
<!--<el-option value="configurable" :label="$t('Configurable')"></el-option>-->
<!--<el-option value="customized" :label="$t('Customized')"></el-option>-->
<!--</el-select>-->
<el-input v-model="spiderForm.type" placeholder="爬虫类型" clearable/>
</el-form-item>
<el-form-item :label="$t('Remark')">
<el-input v-model="spiderForm.remark"/>
@@ -102,16 +103,7 @@ export default {
'spiderForm'
]),
isShowRun () {
if (this.isCustomized) {
// customized spider
return !!this.spiderForm.cmd
} else {
// configurable spider
return !!this.spiderForm.fields
}
},
isCustomized () {
return this.spiderForm.type === 'customized'
return !!this.spiderForm.cmd
}
},
methods: {

2
frontend/src/views/result/ResultDetail.vue
View File

@@ -59,7 +59,7 @@ export default {
},
created () {
// get the list of the spiders
this.$store.dispatch('spider/getSpiderList')
// this.$store.dispatch('spider/getSpiderList')
// get spider basic info
this.$store.dispatch('spider/getSpiderData', this.$route.params.id)

4
frontend/src/views/result/ResultList.vue
View File

@@ -195,7 +195,7 @@ export default {
this.dialogVisible = true
},
onRefresh () {
this.$store.dispatch('spider/getSpiderList')
// this.$store.dispatch('spider/getSpiderList')
},
onSubmit () {
const vm = this
@@ -257,7 +257,7 @@ export default {
}
},
created () {
this.$store.dispatch('spider/getSpiderList')
// this.$store.dispatch('spider/getSpiderList')
}
}
</script>

2
frontend/src/views/schedule/ScheduleList.vue
View File

@@ -269,7 +269,7 @@ export default {
},
created () {
this.$store.dispatch('schedule/getScheduleList')
this.$store.dispatch('spider/getSpiderList')
// this.$store.dispatch('spider/getSpiderList')
this.$store.dispatch('node/getNodeList')
}
}

2
frontend/src/views/spider/SpiderDetail.vue
View File

@@ -87,7 +87,7 @@ export default {
},
created () {
// get the list of the spiders
this.$store.dispatch('spider/getSpiderList')
// this.$store.dispatch('spider/getSpiderList')
// get spider basic info
this.$store.dispatch('spider/getSpiderData', this.$route.params.id)

19
frontend/src/views/spider/SpiderList.vue
View File

@@ -111,12 +111,13 @@
<div class="left">
<el-form :inline="true">
<el-form-item>
<el-select placeholder="爬虫类型" size="small" v-model="filter.type">
<el-option v-for="type in types" :value="type" :key="type" :label="type"/>
<el-select clearable @change="onSpiderTypeChange" placeholder="爬虫类型" size="small" v-model="filter.type">
<el-option v-for="item in types" :value="item.type" :key="item.type"
:label="item.type === 'customized'? '自定义':item.type "/>
</el-select>
</el-form-item>
<el-form-item>
<el-input @keyup.enter.native="onSearch" size="small" placeholder="名称" v-model="filter.keyword">
<el-input clearable @keyup.enter.native="onSearch" size="small" placeholder="名称" v-model="filter.keyword">
<i slot="suffix" class="el-input__icon el-icon-search"></i>
</el-input>
</el-form-item>
@@ -156,8 +157,7 @@
align="left"
:width="col.width">
<template slot-scope="scope">
<el-tag type="success" v-if="scope.row.type === 'configurable'">{{$t('Configurable')}}</el-tag>
<el-tag type="primary" v-else-if="scope.row.type === 'customized'">{{$t('Customized')}}</el-tag>
{{scope.row.type === 'customized' ? '自定义' : scope.row.type}}
</template>
</el-table-column>
<el-table-column v-else-if="col.name === 'last_5_errors'"
@@ -302,6 +302,10 @@ export default {
])
},
methods: {
onSpiderTypeChange (val) {
this.filter.type = val
this.getList()
},
onPageSizeChange (val) {
this.pagination.pageSize = val
this.getList()
@@ -490,13 +494,14 @@ export default {
let params = {
pageNum: this.pagination.pageNum,
pageSize: this.pagination.pageSize,
keyword: this.filter.keyword
keyword: this.filter.keyword,
type: this.filter.type
}
this.$store.dispatch('spider/getSpiderList', params)
},
getTypes () {
request.get(`/spider/types`).then(resp => {
console.log('resp', resp)
this.types = resp.data.data
})
}
},