diff --git a/README-zh.md b/README-zh.md index 102eeceb..36e9d82d 100644 --- a/README-zh.md +++ b/README-zh.md @@ -5,14 +5,15 @@ +中文 | [English](https://github.com/tikazyq/crawlab/blob/master/README.md) + 基于Celery的爬虫分布式爬虫管理平台，支持多种编程语言以及多种爬虫框架. -[查看演示 Demo](http://114.67.75.98:8080) - -[English Documentation](https://github.com/tikazyq/crawlab/blob/master/README.md) +[查看演示 Demo](http://114.67.75.98:8080) | [文档](https://tikazyq.github.io/crawlab) ## 要求 -- Python3 +- Python 3.6+ +- Node.js 8.12+ - MongoDB - Redis @@ -177,6 +178,8 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 - [ ] 登录和用户管理 - [ ] 全局搜索 +## 社区 & 赞助 + 如果您觉得Crawlab对您的日常开发或公司有帮助，请加作者微信 tikazyq1 并注明"Crawlab"，作者会将你拉入群。或者，您可以扫下方支付宝二维码给作者打赏去升级团队协作软件或买一杯咖啡。

diff --git a/README.md b/README.md index 4c6729c1..2914ffa2 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,16 @@ +[中文](https://github.com/tikazyq/crawlab/blob/master/README-zh.md) | English + Celery-based web crawler admin platform for managing distributed web spiders regardless of languages and frameworks. -[Demo](http://114.67.75.98:8080) +[Demo](http://114.67.75.98:8080) | [Documentation](https://tikazyq.github.io/crawlab) -[中文文档](https://github.com/tikazyq/crawlab/blob/master/README-zh.md) ## Pre-requisite -- Python3 +- Python 3.6+ +- Node.js 8.12+ - MongoDB - Redis @@ -166,12 +168,14 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any - [ ] More spider examples ##### Frontend -- [ ] Task Stats/Analytics -- [ ] Table Filters +- [x] Task Stats/Analytics +- [x] Table Filters - [x] Multi-Language Support (中文) - [ ] Login & User Management - [ ] General Search +## Community & Sponsorship + If you feel Crawlab could benefit your daily work or your company, please add the author's Wechat account noting "Crawlab" to enter the discussion group. Or you scan the Alipay QR code below to give us a reward to upgrade our teamwork software or buy a coffee.

diff --git a/docs/Architecture/App.html b/docs/Architecture/App.html index cb7a52aa..56e27371 100644 --- a/docs/Architecture/App.html +++ b/docs/Architecture/App.html @@ -397,7 +397,7 @@ diff --git a/docs/Architecture/Celery.html b/docs/Architecture/Celery.html index 4f1085bb..2f2bea93 100644 --- a/docs/Architecture/Celery.html +++ b/docs/Architecture/Celery.html @@ -397,7 +397,7 @@ diff --git a/docs/Architecture/index.html b/docs/Architecture/index.html index 62567bdb..c271a5db 100644 --- a/docs/Architecture/index.html +++ b/docs/Architecture/index.html @@ -397,7 +397,7 @@ diff --git a/docs/Concept/Deploy.html b/docs/Concept/Deploy.html index d09f8817..f788cb78 100644 --- a/docs/Concept/Deploy.html +++ b/docs/Concept/Deploy.html @@ -399,7 +399,7 @@ diff --git a/docs/Concept/Node.html b/docs/Concept/Node.html index 7f5a3a6c..4a40be7c 100644 --- a/docs/Concept/Node.html +++ b/docs/Concept/Node.html @@ -398,7 +398,7 @@ diff --git a/docs/Concept/Spider.html b/docs/Concept/Spider.html index 8d3cbe6f..1bfba7ec 100644 --- a/docs/Concept/Spider.html +++ b/docs/Concept/Spider.html @@ -403,7 +403,7 @@ diff --git a/docs/Concept/Task.html b/docs/Concept/Task.html index 44893200..d31824cc 100644 --- a/docs/Concept/Task.html +++ b/docs/Concept/Task.html @@ -398,7 +398,7 @@ diff --git a/docs/Concept/index.html b/docs/Concept/index.html index 3aa4b7ee..92e359c1 100644 --- a/docs/Concept/index.html +++ b/docs/Concept/index.html @@ -397,7 +397,7 @@ diff --git a/docs/Examples/index.html b/docs/Examples/index.html index 3be28a22..af2ddff6 100644 --- a/docs/Examples/index.html +++ b/docs/Examples/index.html @@ -397,7 +397,7 @@ diff --git a/docs/Functions/FunctionList.md b/docs/Functions/FunctionList.md new file mode 100644 index 00000000..636126fa --- /dev/null +++ b/docs/Functions/FunctionList.md @@ -0,0 +1,61 @@ +# 功能列表 + +类别 | 功能名称 | 已统计 | 备注 +--- | --- | --- | --- +全局 | 打开页面 | Y | _trackPageview +全局 | 切换中英文 | Y +全局 | 允许/禁止统计 | Y +节点 | 刷新 | Y +节点 | 查看 | Y +节点 | 删除 | Y +节点详情 | 保存 | Y +节点详情 | 切换节点 | Y +爬虫 | 部署所有爬虫 | Y +爬虫 | 导入爬虫 | Y +爬虫 | 添加爬虫-可配置爬虫 | Y +爬虫 | 添加爬虫-自定义爬虫 | Y +爬虫 | 刷新 | Y +爬虫 | 查看 | Y +爬虫 | 删除 | Y +爬虫 | 部署 | Y +爬虫 | 运行 | Y +爬虫 | 搜索网站 | Y +爬虫详情 | 切换爬虫 | Y +爬虫详情 | 切换标签 | Y +爬虫详情-概览 | 保存 | Y +爬虫详情-概览 | 部署 | Y +爬虫详情-概览 | 运行 | Y +爬虫详情-环境 | 添加 | Y +爬虫详情-环境 | 删除 | Y +爬虫详情-环境 | 保存 | Y +爬虫详情-配置 | 保存 | Y +爬虫详情-配置 | 预览 | Y +爬虫详情-配置 | 提取字段 | Y +爬虫详情-配置 | 运行 | Y +爬虫详情-配置 | 添加字段 | Y +爬虫详情-配置 | 更改字段 | Y +爬虫详情-配置 | 删除字段 | Y +爬虫详情-配置 | 设置详情页URL | Y +任务 | 选择节点 | Y +任务 | 选择爬虫 | Y +任务 | 点击爬虫详情 | Y +任务 | 点击节点详情 | Y +任务 | 搜索 | Y +任务 | 查看 | Y +任务 | 删除 | Y +任务详情 | 切换标签 | Y +任务详情-概览 | 点击爬虫详情 | Y +任务详情-概览 | 点击节点详情 | Y +任务详情-结果 | 下载CSV | Y +定时任务 | 添加 | Y +定时任务 | 修改 | Y +定时任务 | 删除 | Y +定时任务 | 提交 | Y +部署 | 刷新 | Y +网站 | 搜索 | Y +网站 | 选择主类别 | Y +网站 | 选择类别 | Y +网站 | 点击域名 | Y +网站 | 点击爬虫数 | Y +网站 | 点击Robots协议 | N + diff --git a/docs/QuickStart/Installation.html b/docs/QuickStart/Installation.html index 01bcb7bf..3e23c4a9 100644 --- a/docs/QuickStart/Installation.html +++ b/docs/QuickStart/Installation.html @@ -408,7 +408,7 @@ npm install diff --git a/docs/QuickStart/Run.html b/docs/QuickStart/Run.html index 7e47391d..c8beac8f 100644 --- a/docs/QuickStart/Run.html +++ b/docs/QuickStart/Run.html @@ -440,7 +440,7 @@ npm run serve diff --git a/docs/QuickStart/index.html b/docs/QuickStart/index.html index 9e036b43..0885c354 100644 --- a/docs/QuickStart/index.html +++ b/docs/QuickStart/index.html @@ -401,7 +401,7 @@ diff --git a/docs/index.html b/docs/index.html index 8d46e552..c9f4837f 100644 --- a/docs/index.html +++ b/docs/index.html @@ -548,7 +548,7 @@ MONGO_DB = 'crawlab_test' diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 975a627f..06a41cce 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -29,9 +29,11 @@ export default { localStorage.setItem('useStats', value) document.querySelector('.el-message__closeBtn').click() if (value === 1) { - window._hmt.push(['_trackPageview', '/allow_stats']) + this.$st.sendPv('/allow_stats') + this.$st.sendEv('全局', '允许/禁止统计', 'value', 'allow') } else { - window._hmt.push(['_trackPageview', '/disallow_stats']) + this.$st.sendPv('/disallow_stats') + this.$st.sendEv('全局', '允许/禁止统计', 'value', 'disallow') } } diff --git a/frontend/src/components/Config/ConfigList.vue b/frontend/src/components/Config/ConfigList.vue index 1b0e9d58..c964d57b 100644 --- a/frontend/src/components/Config/ConfigList.vue +++ b/frontend/src/components/Config/ConfigList.vue @@ -159,6 +159,7 @@ export default { this.spiderForm.crawl_type = value }, onSave () { + this.$st.sendEv('爬虫详情-配置', '保存') return new Promise((resolve, reject) => { this.saveLoading = true this.$store.dispatch('spider/updateSpiderFields') @@ -203,6 +204,7 @@ export default { .finally(() => { this.previewLoading = false }) + this.$st.sendEv('爬虫详情-配置', '预览') }) }, onCrawl () { @@ -215,6 +217,7 @@ export default { .then(() => { this.$message.success(this.$t(`Spider task has been scheduled`)) }) + this.$st.sendEv('爬虫详情-配置', '运行') }) }, onExtractFields () { @@ -235,6 +238,7 @@ export default { .finally(() => { this.extractFieldsLoading = false }) + this.$st.sendEv('爬虫详情-配置', '提取字段') }) } }, diff --git a/frontend/src/components/Environment/EnvironmentList.vue b/frontend/src/components/Environment/EnvironmentList.vue index 9e0b1413..a46432c0 100644 --- a/frontend/src/components/Environment/EnvironmentList.vue +++ b/frontend/src/components/Environment/EnvironmentList.vue @@ -49,10 +49,11 @@ export default { name: '', value: '' }) - console.log(this.spiderForm) + this.$st.sendEv('爬虫详情-环境', '添加') }, deleteEnv (index) { this.spiderForm.envs.splice(index, 1) + this.$st.sendEv('爬虫详情-环境', '删除') }, save () { this.$store.dispatch('spider/updateSpiderEnvs') @@ -62,6 +63,7 @@ export default { .catch(error => { this.$message.error(error) }) + this.$st.sendEv('爬虫详情-环境', '保存') } } } diff --git a/frontend/src/components/InfoView/NodeInfoView.vue b/frontend/src/components/InfoView/NodeInfoView.vue index 69a3601a..bc6e687a 100644 --- a/frontend/src/components/InfoView/NodeInfoView.vue +++ b/frontend/src/components/InfoView/NodeInfoView.vue @@ -55,6 +55,7 @@ export default { }) } }) + this.$st.sendEv('节点详情', '保存') } } } diff --git a/frontend/src/components/InfoView/SpiderInfoView.vue b/frontend/src/components/InfoView/SpiderInfoView.vue index e896bbc0..d095f81a 100644 --- a/frontend/src/components/InfoView/SpiderInfoView.vue +++ b/frontend/src/components/InfoView/SpiderInfoView.vue @@ -127,6 +127,7 @@ export default { .then(() => { this.$message.success(this.$t(`Spider task has been scheduled`)) }) + this.$st.sendEv('爬虫详情-概览', '运行') }) } }) @@ -149,6 +150,7 @@ export default { .then(() => { this.$message.success(this.$t(`Spider has been deployed`)) }) + this.$st.sendEv('爬虫详情-概览', '部署') }) } }) @@ -165,6 +167,7 @@ export default { }) } }) + this.$st.sendEv('爬虫详情-概览', '保存') }, fetchSiteSuggestions (keyword, callback) { this.$request.get('/sites', { diff --git a/frontend/src/components/Overview/TaskOverview.vue b/frontend/src/components/Overview/TaskOverview.vue index d306bb41..603f28ef 100644 --- a/frontend/src/components/Overview/TaskOverview.vue +++ b/frontend/src/components/Overview/TaskOverview.vue @@ -52,9 +52,11 @@ export default { methods: { onClickNodeTitle () { this.$router.push(`/nodes/${this.nodeForm._id}`) + this.$st.sendEv('任务详情-概览', '点击节点详情') }, onClickSpiderTitle () { this.$router.push(`/spiders/${this.spiderForm._id}`) + this.$st.sendEv('任务详情-概览', '点击爬虫详情') } }, created () { diff --git a/frontend/src/components/TableView/FieldsTableView.vue b/frontend/src/components/TableView/FieldsTableView.vue index ab2376f7..836a8f49 100644 --- a/frontend/src/components/TableView/FieldsTableView.vue +++ b/frontend/src/components/TableView/FieldsTableView.vue @@ -109,14 +109,17 @@ export default { type: 'css', extract_type: 'text' }) + this.$st.sendEv('爬虫详情-配置', '添加字段') }, deleteField (index) { this.fields.splice(index, 1) + this.$st.sendEv('爬虫详情-配置', '删除字段') }, onNameChange (row) { if (this.fields.filter(d => d.name === row.name).length > 1) { this.$message.error(this.$t(`Duplicated field names for ${row.name}`)) } + this.$st.sendEv('爬虫详情-配置', '更改字段') }, onCheck (row) { this.fields.forEach(d => { @@ -124,6 +127,7 @@ export default { this.$set(d, 'is_detail', false) } }) + this.$st.sendEv('爬虫详情-配置', '设置详情页URL') } } } diff --git a/frontend/src/main.js b/frontend/src/main.js index 721f0f63..a3008fae 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -12,8 +12,6 @@ import 'font-awesome/scss/font-awesome.scss'// FontAwesome import 'codemirror/lib/codemirror.css' -// import ba from 'vue-ba' - import App from './App' import store from './store' import router from './router' @@ -23,6 +21,7 @@ import '@/permission' // permission control import request from './api/request' import i18n from './i18n' +import utils from './utils' Vue.use(ElementUI, { locale }) @@ -45,6 +44,12 @@ window._hmt = window._hmt || []; // inject request api Vue.prototype.$request = request +// inject utils +Vue.prototype.$utils = utils + +// inject stats +Vue.prototype.$st = utils.stats + const app = new Vue({ el: '#app', i18n, diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index 4eddc102..0bfae74f 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -1,6 +1,8 @@ import Vue from 'vue' import Router from 'vue-router' +import stats from '../utils/stats' + /* Layout */ import Layout from '../views/layout/Layout' @@ -224,9 +226,7 @@ router.beforeEach((to, from, next) => { router.afterEach((to, from, next) => { if (to.path) { - if (localStorage.getItem('useStats') !== '0') { - window._hmt.push(['_trackPageview', to.path]) - } + stats.sendPv(to.path) } }) diff --git a/frontend/src/utils/index.js b/frontend/src/utils/index.js index e69de29b..d5a118af 100644 --- a/frontend/src/utils/index.js +++ b/frontend/src/utils/index.js @@ -0,0 +1,5 @@ +import stats from './stats' + +export default { + stats +} diff --git a/frontend/src/utils/stats.js b/frontend/src/utils/stats.js index e69de29b..f2fa0186 100644 --- a/frontend/src/utils/stats.js +++ b/frontend/src/utils/stats.js @@ -0,0 +1,12 @@ +export default { + sendPv (page) { + if (localStorage.getItem('useStats') !== '0') { + window._hmt.push(['_trackPageview', page]) + } + }, + sendEv (category, eventName, optLabel, optValue) { + if (localStorage.getItem('useStats') !== '0') { + window._hmt.push(['_trackEvent', category, eventName, optLabel, optValue]) + } + } +} diff --git a/frontend/src/views/deploy/DeployList.vue b/frontend/src/views/deploy/DeployList.vue index 5cc6d118..16c8d319 100644 --- a/frontend/src/views/deploy/DeployList.vue +++ b/frontend/src/views/deploy/DeployList.vue @@ -134,6 +134,7 @@ export default { }, onRefresh () { this.$store.dispatch('deploy/getDeployList') + this.$st.sendEv('部署', '刷新') }, onView (row) { this.$router.push(`/deploys/${row._id}`) diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 5b1f97f8..eae27a91 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -57,6 +57,8 @@ export default { window.localStorage.setItem('lang', lang) this.$i18n.locale = lang this.$store.commit('lang/SET_LANG', lang) + + this.$st.sendEv('全局', '切换中英文', 'lang', lang) } } } diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index e95eb472..7de7a677 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -47,6 +47,7 @@ export default { }, onNodeChange (id) { this.$router.push(`/nodes/${id}`) + this.$st.sendEv('节点详情', '切换节点') } }, created () { diff --git a/frontend/src/views/node/NodeList.vue b/frontend/src/views/node/NodeList.vue index 35dc7d18..be916f44 100644 --- a/frontend/src/views/node/NodeList.vue +++ b/frontend/src/views/node/NodeList.vue @@ -123,8 +123,7 @@ export default { } }, methods: { - onSearch (value) { - console.log(value) + onSearch () { }, onAdd () { this.$store.commit('node/SET_NODE_FORM', []) @@ -133,6 +132,7 @@ export default { }, onRefresh () { this.$store.dispatch('node/getNodeList') + this.$st.sendEv('节点', '刷新') }, onSubmit () { const vm = this @@ -159,7 +159,6 @@ export default { this.dialogVisible = false }, onEdit (row) { - console.log(row) this.isEditMode = true this.$store.commit('node/SET_NODE_FORM', row) this.dialogVisible = true @@ -177,10 +176,13 @@ export default { message: 'Deleted successfully' }) }) + this.$st.sendEv('节点', '删除', 'id', row._id) }) }, onView (row) { this.$router.push(`/nodes/${row._id}`) + + this.$st.sendEv('节点', '查看', 'id', row._id) }, onPageChange () { this.$store.dispatch('node/getNodeList') diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index e9fe0bce..7a1fc9ba 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -160,6 +160,7 @@ export default { this.isEdit = false this.dialogVisible = true this.$store.commit('schedule/SET_SCHEDULE_FORM', {}) + this.$st.sendEv('定时任务', '添加') }, onAddSubmit () { this.$refs.scheduleForm.validate(res => { @@ -179,6 +180,7 @@ export default { }) } }) + this.$st.sendEv('定时任务', '提交') }, isShowRun () { }, @@ -186,6 +188,7 @@ export default { this.$store.commit('schedule/SET_SCHEDULE_FORM', row) this.dialogVisible = true this.isEdit = true + this.$st.sendEv('定时任务', '修改', 'id', row._id) }, onRemove (row) { this.$store.dispatch('schedule/removeSchedule', row._id) @@ -195,6 +198,7 @@ export default { this.$message.success(`Schedule "${row.name}" has been removed`) }, 100) }) + this.$st.sendEv('定时任务', '删除', 'id', row._id) }, onCrawl () { } diff --git a/frontend/src/views/site/SiteList.vue b/frontend/src/views/site/SiteList.vue index 98fa1365..450ec16b 100644 --- a/frontend/src/views/site/SiteList.vue +++ b/frontend/src/views/site/SiteList.vue @@ -8,11 +8,11 @@ v-model="keyword"> + clearable filterable @change="onSelectMainCategory"> + clearable filterable @change="onSelectCategory"> @@ -84,7 +85,7 @@