mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-22 17:31:03 +01:00
added site list
This commit is contained in:
Marvin Zhang
@@ -9,6 +9,7 @@ from flask import Flask
|
||||
from flask_cors import CORS
|
||||
from flask_restful import Api
|
||||
# from flask_restplus import Api
|
||||
from routes.sites import SiteApi
|
||||
from utils.log import other
|
||||
from constants.node import NodeStatus
|
||||
from db.manager import db_manager
|
||||
@@ -68,6 +69,9 @@ api.add_resource(StatsApi,
|
||||
api.add_resource(ScheduleApi,
|
||||
'/api/schedules',
|
||||
'/api/schedules/<string:id>')
|
||||
api.add_resource(SiteApi,
|
||||
'/api/sites',
|
||||
'/api/sites/<string:id>')
|
||||
|
||||
|
||||
def monitor_nodes_status(celery_app):
|
||||
|
||||
64
crawlab/routes/sites.py
Normal file
64
crawlab/routes/sites.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import json
|
||||
|
||||
from bson import ObjectId
|
||||
from pymongo import ASCENDING
|
||||
|
||||
from db.manager import db_manager
|
||||
from routes.base import BaseApi
|
||||
from utils import jsonify
|
||||
|
||||
|
||||
class SiteApi(BaseApi):
|
||||
col_name = 'sites'
|
||||
|
||||
arguments = (
|
||||
('keyword', str),
|
||||
('category', str),
|
||||
)
|
||||
|
||||
def get(self, id: str = None, action: str = None):
|
||||
# action by id
|
||||
if action is not None:
|
||||
if not hasattr(self, action):
|
||||
return {
|
||||
'status': 'ok',
|
||||
'code': 400,
|
||||
'error': 'action "%s" invalid' % action
|
||||
}, 400
|
||||
return getattr(self, action)(id)
|
||||
|
||||
elif id is not None:
|
||||
site = db_manager.get(col_name=self.col_name, id=id)
|
||||
return jsonify(site)
|
||||
|
||||
# list tasks
|
||||
args = self.parser.parse_args()
|
||||
page_size = args.get('page_size') or 10
|
||||
page_num = args.get('page_num') or 1
|
||||
filter_str = args.get('filter')
|
||||
keyword = args.get('keyword')
|
||||
filter_ = {}
|
||||
if filter_str is not None:
|
||||
filter_ = json.loads(filter_str)
|
||||
if keyword is not None:
|
||||
filter_['$or'] = [
|
||||
{'description': {'$regex': keyword}},
|
||||
{'name': {'$regex': keyword}}
|
||||
]
|
||||
|
||||
items = db_manager.list(
|
||||
col_name=self.col_name,
|
||||
cond=filter_,
|
||||
limit=page_size,
|
||||
skip=page_size * (page_num - 1),
|
||||
sort_key='rank',
|
||||
sort_direction=ASCENDING
|
||||
)
|
||||
|
||||
return {
|
||||
'status': 'ok',
|
||||
'total_count': db_manager.count(self.col_name, filter_),
|
||||
'page_num': page_num,
|
||||
'page_size': page_size,
|
||||
'items': jsonify(items)
|
||||
}
|
||||
@@ -36,7 +36,6 @@ class TaskApi(BaseApi):
|
||||
'code': 400,
|
||||
'error': 'action "%s" invalid' % action
|
||||
}, 400
|
||||
# other.info(f"到这了{action},{id}")
|
||||
return getattr(self, action)(id)
|
||||
|
||||
elif id is not None:
|
||||
@@ -78,9 +77,6 @@ class TaskApi(BaseApi):
|
||||
sort_key='create_ts')
|
||||
items = []
|
||||
for task in tasks:
|
||||
# celery tasks
|
||||
# _task = db_manager.get('tasks_celery', id=task['_id'])
|
||||
|
||||
# get spider
|
||||
_spider = db_manager.get(col_name='spiders', id=str(task['spider_id']))
|
||||
|
||||
|
||||
@@ -38,26 +38,6 @@
|
||||
<el-option value="go" label="Go"></el-option>
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<!--<el-form-item :label="$t('Schedule Enabled')">-->
|
||||
<!--<el-switch v-model="spiderForm.cron_enabled" :disabled="isView">-->
|
||||
<!--</el-switch>-->
|
||||
<!--</el-form-item>-->
|
||||
<!--<el-form-item :label="$t('Schedule Cron')" v-if="spiderForm.cron_enabled"-->
|
||||
<!--prop="cron"-->
|
||||
<!--:rules="cronRules"-->
|
||||
<!--:inline-message="true">-->
|
||||
<!--<template slot="label">-->
|
||||
<!--<el-tooltip :content="$t('Cron Format: [second] [minute] [hour] [day of month] [month] [day of week]')"-->
|
||||
<!--placement="top">-->
|
||||
<!--<span>-->
|
||||
<!--{{$t('Schedule Cron')}}-->
|
||||
<!--<i class="fa fa-exclamation-circle"></i>-->
|
||||
<!--</span>-->
|
||||
<!--</el-tooltip>-->
|
||||
<!--</template>-->
|
||||
<!--<el-input v-model="spiderForm.cron" :placeholder="$t('Schedule Cron')"-->
|
||||
<!--:disabled="isView"></el-input>-->
|
||||
<!--</el-form-item>-->
|
||||
</el-form>
|
||||
</el-row>
|
||||
<el-row class="button-container" v-if="!isView">
|
||||
|
||||
59
frontend/src/store/modules/site.js
Normal file
59
frontend/src/store/modules/site.js
Normal file
@@ -0,0 +1,59 @@
|
||||
import request from '../../api/request'
|
||||
|
||||
const state = {
|
||||
tableData: [],
|
||||
|
||||
// filter
|
||||
filter: {},
|
||||
keyword: '',
|
||||
|
||||
// pagination
|
||||
pageNum: 1,
|
||||
pageSize: 10,
|
||||
totalCount: 0
|
||||
}
|
||||
|
||||
const getters = {}
|
||||
|
||||
const mutations = {
|
||||
SET_KEYWORD (state, value) {
|
||||
state.keyword = value
|
||||
},
|
||||
SET_TABLE_DATA (state, value) {
|
||||
state.tableData = value
|
||||
},
|
||||
SET_PAGE_NUM (state, value) {
|
||||
state.pageNum = value
|
||||
},
|
||||
SET_PAGE_SIZE (state, value) {
|
||||
state.pageSize = value
|
||||
},
|
||||
SET_TOTAL_COUNT (state, value) {
|
||||
state.totalCount = value
|
||||
}
|
||||
}
|
||||
|
||||
const actions = {
|
||||
getSiteList ({ state, commit }) {
|
||||
return request.get('/sites', {
|
||||
page_num: state.pageNum,
|
||||
page_size: state.pageSize,
|
||||
keyword: state.keyword || undefined,
|
||||
filter: {
|
||||
category: state.filter.category || undefined
|
||||
}
|
||||
})
|
||||
.then(response => {
|
||||
commit('SET_TABLE_DATA', response.data.items)
|
||||
commit('SET_TOTAL_COUNT', response.data.total_count)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
namespaced: true,
|
||||
state,
|
||||
getters,
|
||||
mutations,
|
||||
actions
|
||||
}
|
||||
172
frontend/src/views/site/SiteList.vue
Normal file
172
frontend/src/views/site/SiteList.vue
Normal file
@@ -0,0 +1,172 @@
|
||||
<template>
|
||||
<div class="app-container">
|
||||
<!--filter-->
|
||||
<div class="filter">
|
||||
<el-input prefix-icon="el-icon-search"
|
||||
:placeholder="$t('Search')"
|
||||
class="filter-search"
|
||||
v-model="keyword">
|
||||
</el-input>
|
||||
<el-button type="success"
|
||||
icon="el-icon-refresh"
|
||||
class="btn refresh"
|
||||
@click="onSearch">
|
||||
{{$t('Search')}}
|
||||
</el-button>
|
||||
</div>
|
||||
|
||||
<!--table list-->
|
||||
<el-table :data="tableData"
|
||||
class="table"
|
||||
:header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}"
|
||||
border>
|
||||
<template v-for="col in columns">
|
||||
<el-table-column v-if="col.name === 'category'"
|
||||
:key="col.name"
|
||||
:label="$t(col.label)"
|
||||
:width="col.width"
|
||||
:align="col.align">
|
||||
<template slot-scope="scope">
|
||||
<el-select v-model="scope.row[col.name]" :placeholder="$t('Select')">
|
||||
<el-option v-for="op in categoryList"
|
||||
:key="op"
|
||||
:value="op"
|
||||
:label="op">
|
||||
</el-option>
|
||||
</el-select>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column v-else
|
||||
:key="col.name"
|
||||
:property="col.name"
|
||||
:label="$t(col.label)"
|
||||
:sortable="col.sortable"
|
||||
:align="col.align || 'center'"
|
||||
:width="col.width">
|
||||
</el-table-column>
|
||||
</template>
|
||||
<el-table-column :label="$t('Action')" align="left" width="200">
|
||||
<template slot-scope="scope">
|
||||
<el-tooltip :content="$t('View')" placement="top">
|
||||
<el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
<el-tooltip :content="$t('Remove')" placement="top">
|
||||
<el-button type="danger" icon="el-icon-delete" size="mini" @click="onRemove(scope.row)"></el-button>
|
||||
</el-tooltip>
|
||||
</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<div class="pagination">
|
||||
<el-pagination
|
||||
@current-change="onPageChange"
|
||||
@size-change="onPageChange"
|
||||
:current-page.sync="pageNum"
|
||||
:page-sizes="[10, 20, 50, 100]"
|
||||
:page-size.sync="pageSize"
|
||||
layout="sizes, prev, pager, next"
|
||||
:total="totalCount">
|
||||
</el-pagination>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script>
|
||||
import {
|
||||
mapState
|
||||
} from 'vuex'
|
||||
|
||||
export default {
|
||||
name: 'SiteList',
|
||||
data () {
|
||||
return {
|
||||
categoryList: [
|
||||
'新闻',
|
||||
'搜索引擎',
|
||||
'综合',
|
||||
'金融',
|
||||
'购物',
|
||||
'社交',
|
||||
'视频',
|
||||
'音乐',
|
||||
'资讯',
|
||||
'政企官网',
|
||||
'其他'
|
||||
],
|
||||
columns: [
|
||||
{ name: 'rank', label: 'Rank', align: 'center', width: '80' },
|
||||
{ name: 'name', label: 'Name', align: 'left', width: '120' },
|
||||
{ name: 'domain', label: 'Domain', align: 'left', width: '150' },
|
||||
{ name: 'description', label: 'Description', align: 'left' },
|
||||
{ name: 'category', label: 'Category', align: 'center', width: '180' }
|
||||
]
|
||||
}
|
||||
},
|
||||
computed: {
|
||||
...mapState('site', [
|
||||
'tableData',
|
||||
'totalCount'
|
||||
]),
|
||||
keyword: {
|
||||
get () {
|
||||
return this.$store.state.site.keyword
|
||||
},
|
||||
set (value) {
|
||||
this.$store.commit('site/SET_KEYWORD', value)
|
||||
}
|
||||
},
|
||||
pageNum: {
|
||||
get () {
|
||||
return this.$store.state.site.pageNum
|
||||
},
|
||||
set (value) {
|
||||
this.$store.commit('site/SET_PAGE_NUM', value)
|
||||
}
|
||||
},
|
||||
pageSize: {
|
||||
get () {
|
||||
return this.$store.state.site.pageSize
|
||||
},
|
||||
set (value) {
|
||||
this.$store.commit('site/SET_PAGE_SIZE', value)
|
||||
}
|
||||
}
|
||||
},
|
||||
methods: {
|
||||
onSearch () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
},
|
||||
onPageChange () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
}
|
||||
},
|
||||
created () {
|
||||
this.$store.dispatch('site/getSiteList')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.filter {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.filter .filter-search {
|
||||
width: 180px;
|
||||
}
|
||||
|
||||
.filter .btn {
|
||||
margin-left: 20px;
|
||||
}
|
||||
|
||||
.table {
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.table >>> .el-select .el-input__inner {
|
||||
height: 32px;
|
||||
}
|
||||
|
||||
.table >>> .el-select .el-select__caret {
|
||||
line-height: 32px;
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user