From 7d189e9567603ee2519729bf22928c5c7245a451 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 8 Jan 2020 10:14:12 +0800 Subject: [PATCH 01/65] fixed baidu tongji issue --- frontend/src/views/node/NodeDetail.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/node/NodeDetail.vue b/frontend/src/views/node/NodeDetail.vue index 4dab386b..7d8f5c67 100644 --- a/frontend/src/views/node/NodeDetail.vue +++ b/frontend/src/views/node/NodeDetail.vue @@ -48,8 +48,8 @@ export default { ]) }, methods: { - onTabClick (name) { - this.$st.sendEv('节点详情', '切换标签', name) + onTabClick (tab) { + this.$st.sendEv('节点详情', '切换标签', tab.name) }, onNodeChange (id) { this.$router.push(`/nodes/${id}`) From aa5ae87eebd63c6c7830950cf8c879933f5887ec Mon Sep 17 00:00:00 2001 From: marvzhang Date: Wed, 8 Jan 2020 20:11:05 +0800 Subject: [PATCH 02/65] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=99=BE=E5=BA=A6?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/TableView/TaskTableView.vue | 15 ++++++++++++++- frontend/src/views/layout/components/TagsView.vue | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/TableView/TaskTableView.vue b/frontend/src/components/TableView/TaskTableView.vue index cd2ec10c..56abfab8 100644 --- a/frontend/src/components/TableView/TaskTableView.vue +++ b/frontend/src/components/TableView/TaskTableView.vue @@ -76,6 +76,9 @@ export default { this.clicked = false }, 100) this.$router.push(`/spiders/${row.spider_id}`) + if (this.$route.path.match(/\/nodes\//)) { + this.$st.sendEv('节点详情', '概览', '查看爬虫') + } }, onClickNode (row) { this.clicked = true @@ -83,9 +86,19 @@ export default { this.clicked = false }, 100) this.$router.push(`/nodes/${row.node_id}`) + if (this.$route.path.match(/\/spiders\//)) { + this.$st.sendEv('爬虫详情', '概览', '查看节点') + } }, onClickTask (row) { - if (!this.clicked) this.$router.push(`/tasks/${row._id}`) + if (!this.clicked) { + this.$router.push(`/tasks/${row._id}`) + if (this.$route.path.match(/\/nodes\//)) { + this.$st.sendEv('节点详情', '概览', '查看任务') + } else if (this.$route.path.match(/\/spiders\//)) { + this.$st.sendEv('爬虫详情', '概览', '查看任务') + } + } }, onRefresh () { if (this.$route.path.split('/')[1] === 'spiders') { diff --git a/frontend/src/views/layout/components/TagsView.vue b/frontend/src/views/layout/components/TagsView.vue index b72d874a..1222a854 100644 --- a/frontend/src/views/layout/components/TagsView.vue +++ b/frontend/src/views/layout/components/TagsView.vue @@ -9,6 +9,7 @@ :key="tag.path" tag="span" class="tags-view-item" + @click.native="clickSelectedTag(tag)" @click.middle.native="closeSelectedTag(tag)" @contextmenu.prevent.native="openMenu(tag,$event)"> {{ $t(generateTitle(tag.title)) }} @@ -138,6 +139,9 @@ export default { }) }) }, + clickSelectedTag (tag) { + this.$st.sendEv('全局', '点击标签', tag.name) + }, closeSelectedTag (view) { this.$store.dispatch('delView', view).then(({ visitedViews }) => { if (this.isActive(view)) { From d124c6c96f6171ca7dc723420ea68f890e57790f Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 10 Jan 2020 14:37:50 +0800 Subject: [PATCH 03/65] =?UTF-8?q?=E5=8A=A0=E5=85=A5api.crawlab.cn=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E8=BF=BD=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/.env.development | 1 + frontend/.env.production | 1 + frontend/.env.test | 3 ++- frontend/src/App.vue | 16 ++++++++++++++++ frontend/src/utils/encrypt.js | 13 +++++++++++++ frontend/src/utils/index.js | 4 +++- frontend/src/utils/stats.js | 17 +++++++++++++++++ .../views/layout/components/Sidebar/index.vue | 1 + 8 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 frontend/src/utils/encrypt.js diff --git a/frontend/.env.development b/frontend/.env.development index 779d28e4..580de54d 100644 --- a/frontend/.env.development +++ b/frontend/.env.development @@ -1,2 +1,3 @@ NODE_ENV='development' VUE_APP_BASE_URL=http://localhost:8000 +VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn diff --git a/frontend/.env.production b/frontend/.env.production index 09f11100..86742de1 100644 --- a/frontend/.env.production +++ b/frontend/.env.production @@ -1,2 +1,3 @@ NODE_ENV='production' VUE_APP_BASE_URL='http://localhost:8000' +VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn diff --git a/frontend/.env.test b/frontend/.env.test index acff44de..cc3d7823 100644 --- a/frontend/.env.test +++ b/frontend/.env.test @@ -1,2 +1,3 @@ NODE_ENV='test' -VUE_APP_BASE_URL='http://114.67.75.98:8000/api' +VUE_APP_BASE_URL='http://localhost:8000' +VUE_APP_CRAWLAB_BASE_URL=http://api.crawlab.cn diff --git a/frontend/src/App.vue b/frontend/src/App.vue index a7ba8069..9bcfaca6 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -25,6 +25,12 @@ export default { ...mapState('setting', ['setting']), useStats () { return localStorage.getItem('useStats') + }, + uid () { + return localStorage.getItem('uid') + }, + sid () { + return sessionStorage.getItem('sid') } }, methods: {}, @@ -58,6 +64,16 @@ export default { '' }) } + + // set uid if first visit + if (this.uid === undefined || this.uid === null) { + localStorage.setItem('uid', this.$utils.encrypt.UUID()) + } + + // set session id if starting a session + if (this.sid === undefined || this.sid === null) { + sessionStorage.setItem('sid', this.$utils.encrypt.UUID()) + } } } diff --git a/frontend/src/utils/encrypt.js b/frontend/src/utils/encrypt.js new file mode 100644 index 00000000..87c8c7fb --- /dev/null +++ b/frontend/src/utils/encrypt.js @@ -0,0 +1,13 @@ +export default { + UUID: () => { + let s = [] + let hexDigits = '0123456789abcdef' + for (let i = 0; i < 36; i++) { + s[i] = hexDigits.substr(Math.floor(Math.random() * 0x10), 1) + } + s[14] = '4' // bits 12-15 of the time_hi_and_version field to 0010 + s[8] = s[13] = s[18] = s[23] = '-' + + return s.join('') + } +} diff --git a/frontend/src/utils/index.js b/frontend/src/utils/index.js index d5a118af..0578bb07 100644 --- a/frontend/src/utils/index.js +++ b/frontend/src/utils/index.js @@ -1,5 +1,7 @@ import stats from './stats' +import encrypt from './encrypt' export default { - stats + stats, + encrypt } diff --git a/frontend/src/utils/stats.js b/frontend/src/utils/stats.js index f2fa0186..0a6dabf2 100644 --- a/frontend/src/utils/stats.js +++ b/frontend/src/utils/stats.js @@ -1,12 +1,29 @@ +import axios from 'axios' + +const sendEvCrawlab = async (eventCategory, eventAction, eventLabel) => { + await axios.get(process.env.VUE_APP_CRAWLAB_BASE_URL + '/track', { + params: { + uid: localStorage.getItem('uid'), + sid: sessionStorage.getItem('sid'), + ec: eventCategory, + ea: eventAction, + el: eventLabel, + v: sessionStorage.getItem('v') + } + }) +} + export default { sendPv (page) { if (localStorage.getItem('useStats') !== '0') { window._hmt.push(['_trackPageview', page]) + sendEvCrawlab('访问页面', page, '') } }, sendEv (category, eventName, optLabel, optValue) { if (localStorage.getItem('useStats') !== '0') { window._hmt.push(['_trackEvent', category, eventName, optLabel, optValue]) + sendEvCrawlab(category, eventName, optLabel) } } } diff --git a/frontend/src/views/layout/components/Sidebar/index.vue b/frontend/src/views/layout/components/Sidebar/index.vue index 35b599b9..8b371a44 100644 --- a/frontend/src/views/layout/components/Sidebar/index.vue +++ b/frontend/src/views/layout/components/Sidebar/index.vue @@ -57,6 +57,7 @@ export default { async created () { const res = await this.$request.get('/version') this.version = res.data.data + sessionStorage.setItem('v', this.version) } } From 65261a5797b4ac30621700d580973c8fe13fedb4 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 10 Jan 2020 15:16:34 +0800 Subject: [PATCH 04/65] =?UTF-8?q?=E4=BF=AE=E5=A4=8Drpc=20timeout=20?= =?UTF-8?q?=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/rpc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/rpc.go b/backend/services/rpc.go index 9b6a5b74..1743a8d2 100644 --- a/backend/services/rpc.go +++ b/backend/services/rpc.go @@ -185,7 +185,7 @@ func InitRpcService() error { } // 获取获取消息队列信息 - dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 300) + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0) if err != nil { log.Errorf(err.Error()) debug.PrintStack() From 95083bc47cc7f7f41150e9b64167590b8a217b65 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 10 Jan 2020 17:22:14 +0800 Subject: [PATCH 05/65] updated README --- README-zh.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README-zh.md b/README-zh.md index 9057fcc3..d998540d 100644 --- a/README-zh.md +++ b/README-zh.md @@ -139,7 +139,7 @@ Docker部署的详情，请见[相关文档](https://tikazyq.github.io/crawlab-d Crawlab的架构包括了一个主节点（Master Node）和多个工作节点（Worker Node），以及负责通信和数据储存的Redis和MongoDB数据库。 -![](https://raw.githubusercontent.com/tikazyq/crawlab-docs/master/images/architecture.png) +![](http://static-docs.crawlab.cn/architecture.png) 前端应用向主节点请求数据，主节点通过MongoDB和Redis来执行任务派发调度以及部署，工作节点收到任务之后，开始执行爬虫任务，并将任务结果储存到MongoDB。架构相对于`v0.3.0`之前的Celery版本有所精简，去除了不必要的节点监控模块Flower，节点监控主要由Redis完成。 diff --git a/README.md b/README.md index 075a80b5..b918fef9 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ For Docker Deployment details, please refer to [relevant documentation](https:// The architecture of Crawlab is consisted of the Master Node and multiple Worker Nodes, and Redis and MongoDB databases which are mainly for nodes communication and data storage. -![](https://raw.githubusercontent.com/tikazyq/crawlab-docs/master/images/architecture.png) +![](http://static-docs.crawlab.cn/architecture.png) The frontend app makes requests to the Master Node, which assigns tasks and deploys spiders through MongoDB and Redis. When a Worker Node receives a task, it begins to execute the crawling task, and stores the results to MongoDB. The architecture is much more concise compared with versions before `v0.3.0`. It has removed unnecessary Flower module which offers node monitoring services. They are now done by Redis. From 09cef0a10ad0e57a7efaf40f085bab3e532df892 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Fri, 10 Jan 2020 18:42:37 +0800 Subject: [PATCH 06/65] fixed "nil returned" error --- backend/services/rpc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/rpc.go b/backend/services/rpc.go index 1743a8d2..b10b16d3 100644 --- a/backend/services/rpc.go +++ b/backend/services/rpc.go @@ -185,7 +185,7 @@ func InitRpcService() error { } // 获取获取消息队列信息 - dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0) + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), constants.Infinite) if err != nil { log.Errorf(err.Error()) debug.PrintStack() From c89a6f4bf2a27c731cc40c54b24c98fc48231e12 Mon Sep 17 00:00:00 2001 From: marvzhang Date: Sat, 11 Jan 2020 16:42:04 +0800 Subject: [PATCH 07/65] added pointer to node list --- backend/services/rpc.go | 9 ++++++--- frontend/src/views/node/NodeList.vue | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/backend/services/rpc.go b/backend/services/rpc.go index b10b16d3..66c04369 100644 --- a/backend/services/rpc.go +++ b/backend/services/rpc.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "github.com/apex/log" + "github.com/gomodule/redigo/redis" uuid "github.com/satori/go.uuid" "runtime/debug" ) @@ -185,10 +186,12 @@ func InitRpcService() error { } // 获取获取消息队列信息 - dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), constants.Infinite) + dataStr, err := database.RedisClient.BRPop(fmt.Sprintf("rpc:%s", node.Id.Hex()), 0) if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() + if err != redis.ErrNil { + log.Errorf(err.Error()) + debug.PrintStack() + } continue } diff --git a/frontend/src/views/node/NodeList.vue b/frontend/src/views/node/NodeList.vue index e032bcd4..7b1c50d5 100644 --- a/frontend/src/views/node/NodeList.vue +++ b/frontend/src/views/node/NodeList.vue @@ -26,6 +26,7 @@ class="table" :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}" border + @row-click="onRowClick" @expand-change="onRowExpand">