Merge remote-tracking branch 'origin/develop' into develop

2026-01-28 17:50:56 +01:00 · 2019-06-03 12:56:40 +08:00
parent 17b3329953 7db25ee47f
commit 02c2a17844
112 changed files with 560 additions and 161 deletions
--- a/frontend/src/App.vue
+++ b/frontend/src/App.vue
@@ -29,9 +29,11 @@ export default {
      localStorage.setItem('useStats', value)
      document.querySelector('.el-message__closeBtn').click()
      if (value === 1) {
-        window._hmt.push(['_trackPageview', '/allow_stats'])
+        this.$st.sendPv('/allow_stats')
+        this.$st.sendEv('全局', '允许/禁止统计', 'value', 'allow')
      } else {
-        window._hmt.push(['_trackPageview', '/disallow_stats'])
+        this.$st.sendPv('/disallow_stats')
+        this.$st.sendEv('全局', '允许/禁止统计', 'value', 'disallow')
      }
    }

--- a/frontend/src/api/request.js
+++ b/frontend/src/api/request.js
@@ -1,6 +1,6 @@
 import axios from 'axios'

-let baseUrl = process.env.VUE_APP_API_BASE_URL ? process.env.VUE_APP_API_BASE_URL : 'http://localhost:8000/api'
+let baseUrl = process.env.VUE_APP_BASE_URL ? process.env.VUE_APP_BASE_URL : 'http://localhost:8000/api'
 // console.log(process.env)
 // const baseUrl = process.env.API_BASE_URL || 'http://localhost:8000/api'

--- a/frontend/src/components/Config/ConfigList.vue
+++ b/frontend/src/components/Config/ConfigList.vue
@@ -163,6 +163,7 @@ export default {
      this.spiderForm.crawl_type = value
    },
    onSave () {
+      this.$st.sendEv('爬虫详情-配置', '保存')
      return new Promise((resolve, reject) => {
        this.saveLoading = true
        this.$store.dispatch('spider/updateSpiderFields')
@@ -207,6 +208,7 @@ export default {
            .finally(() => {
              this.previewLoading = false
            })
+          this.$st.sendEv('爬虫详情-配置', '预览')
        })
    },
    onCrawl () {
@@ -219,6 +221,7 @@ export default {
            .then(() => {
              this.$message.success(this.$t(`Spider task has been scheduled`))
            })
+          this.$st.sendEv('爬虫详情-配置', '运行')
        })
    },
    onExtractFields () {
@@ -239,6 +242,7 @@ export default {
            .finally(() => {
              this.extractFieldsLoading = false
            })
+          this.$st.sendEv('爬虫详情-配置', '提取字段')
        })
    },
    renderHeader (h, { column }) {
--- a/frontend/src/components/Environment/EnvironmentList.vue
+++ b/frontend/src/components/Environment/EnvironmentList.vue
@@ -49,10 +49,11 @@ export default {
        name: '',
        value: ''
      })
-      console.log(this.spiderForm)
+      this.$st.sendEv('爬虫详情-环境', '添加')
    },
    deleteEnv (index) {
      this.spiderForm.envs.splice(index, 1)
+      this.$st.sendEv('爬虫详情-环境', '删除')
    },
    save () {
      this.$store.dispatch('spider/updateSpiderEnvs')
@@ -62,6 +63,7 @@ export default {
        .catch(error => {
          this.$message.error(error)
        })
+      this.$st.sendEv('爬虫详情-环境', '保存')
    }
  }
 }
--- a/frontend/src/components/InfoView/NodeInfoView.vue
+++ b/frontend/src/components/InfoView/NodeInfoView.vue
@@ -55,6 +55,7 @@ export default {
            })
        }
      })
+      this.$st.sendEv('节点详情', '保存')
    }
  }
 }
--- a/frontend/src/components/InfoView/SpiderInfoView.vue
+++ b/frontend/src/components/InfoView/SpiderInfoView.vue
@@ -127,6 +127,7 @@ export default {
                .then(() => {
                  this.$message.success(this.$t(`Spider task has been scheduled`))
                })
+              this.$st.sendEv('爬虫详情-概览', '运行')
            })
        }
      })
@@ -149,6 +150,7 @@ export default {
                .then(() => {
                  this.$message.success(this.$t(`Spider has been deployed`))
                })
+              this.$st.sendEv('爬虫详情-概览', '部署')
            })
        }
      })
@@ -165,6 +167,7 @@ export default {
            })
        }
      })
+      this.$st.sendEv('爬虫详情-概览', '保存')
    },
    fetchSiteSuggestions (keyword, callback) {
      this.$request.get('/sites', {
--- a/frontend/src/components/Overview/TaskOverview.vue
+++ b/frontend/src/components/Overview/TaskOverview.vue
@@ -52,9 +52,11 @@ export default {
  methods: {
    onClickNodeTitle () {
      this.$router.push(`/nodes/${this.nodeForm._id}`)
+      this.$st.sendEv('任务详情-概览', '点击节点详情')
    },
    onClickSpiderTitle () {
      this.$router.push(`/spiders/${this.spiderForm._id}`)
+      this.$st.sendEv('任务详情-概览', '点击爬虫详情')
    }
  },
  created () {
--- a/frontend/src/components/TableView/FieldsTableView.vue
+++ b/frontend/src/components/TableView/FieldsTableView.vue
@@ -109,14 +109,17 @@ export default {
        type: 'css',
        extract_type: 'text'
      })
+      this.$st.sendEv('爬虫详情-配置', '添加字段')
    },
    deleteField (index) {
      this.fields.splice(index, 1)
+      this.$st.sendEv('爬虫详情-配置', '删除字段')
    },
    onNameChange (row) {
      if (this.fields.filter(d => d.name === row.name).length > 1) {
        this.$message.error(this.$t(`Duplicated field names for ${row.name}`))
      }
+      this.$st.sendEv('爬虫详情-配置', '更改字段')
    },
    onCheck (row) {
      this.fields.forEach(d => {
@@ -124,6 +127,7 @@ export default {
          this.$set(d, 'is_detail', false)
        }
      })
+      this.$st.sendEv('爬虫详情-配置', '设置详情页URL')
    }
  }
 }
--- a/frontend/src/components/TableView/GeneralTableView.vue
+++ b/frontend/src/components/TableView/GeneralTableView.vue
@@ -58,18 +58,18 @@ export default {
  computed: {
    filteredData () {
      return this.data
-        .map(d => d)
-        .filter((d, index) => {
-          // pagination
-          const pageNum = this.pageNum
-          const pageSize = this.pageSize
-          return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum)
-        })
+      // .map(d => d)
+      // .filter((d, index) => {
+      //   // pagination
+      //   const pageNum = this.pageNum
+      //   const pageSize = this.pageSize
+      //   return (pageSize * (pageNum - 1) <= index) && (index < pageSize * pageNum)
+      // })
    }
  },
  methods: {
    onPageChange () {
-      this.$emit('page-change')
+      this.$emit('page-change', { pageNum: this.pageNum, pageSize: this.pageSize })
    }
  }
 }
--- a/frontend/src/i18n/zh.js
+++ b/frontend/src/i18n/zh.js
@@ -56,6 +56,8 @@ export default {
  'Stop': '停止',
  'Preview': '预览',
  'Extract Fields': '提取字段',
+  'Download': '下载',
+  'Download CSV': '下载CSV',

  // 主页
  'Total Tasks': '总任务数',
--- a/frontend/src/main.js
+++ b/frontend/src/main.js
@@ -12,8 +12,6 @@ import 'font-awesome/scss/font-awesome.scss'// FontAwesome

 import 'codemirror/lib/codemirror.css'

-// import ba from 'vue-ba'
-
 import App from './App'
 import store from './store'
 import router from './router'
@@ -23,6 +21,7 @@ import '@/permission' // permission control

 import request from './api/request'
 import i18n from './i18n'
+import utils from './utils'

 Vue.use(ElementUI, { locale })

@@ -45,6 +44,12 @@ window._hmt = window._hmt || [];
 // inject request api
 Vue.prototype.$request = request

+// inject utils
+Vue.prototype.$utils = utils
+
+// inject stats
+Vue.prototype.$st = utils.stats
+
 const app = new Vue({
  el: '#app',
  i18n,
--- a/frontend/src/router/index.js
+++ b/frontend/src/router/index.js
@@ -1,6 +1,8 @@
 import Vue from 'vue'
 import Router from 'vue-router'

+import stats from '../utils/stats'
+
 /* Layout */
 import Layout from '../views/layout/Layout'

@@ -224,9 +226,7 @@ router.beforeEach((to, from, next) => {

 router.afterEach((to, from, next) => {
  if (to.path) {
-    if (localStorage.getItem('useStats') !== '0') {
-      window._hmt.push(['_trackPageview', to.path])
-    }
+    stats.sendPv(to.path)
  }
 })

--- a/frontend/src/store/modules/task.js
+++ b/frontend/src/store/modules/task.js
@@ -18,7 +18,7 @@ const state = {
  pageNum: 0,
  pageSize: 10,
  // results
-  resultsPageNum: 0,
+  resultsPageNum: 1,
  resultsPageSize: 10
 }

--- a/frontend/src/utils/index.js
+++ b/frontend/src/utils/index.js
@@ -0,0 +1,5 @@
+import stats from './stats'
+
+export default {
+  stats
+}
--- a/frontend/src/utils/stats.js
+++ b/frontend/src/utils/stats.js
@@ -0,0 +1,12 @@
+export default {
+  sendPv (page) {
+    if (localStorage.getItem('useStats') !== '0') {
+      window._hmt.push(['_trackPageview', page])
+    }
+  },
+  sendEv (category, eventName, optLabel, optValue) {
+    if (localStorage.getItem('useStats') !== '0') {
+      window._hmt.push(['_trackEvent', category, eventName, optLabel, optValue])
+    }
+  }
+}
--- a/frontend/src/views/deploy/DeployList.vue
+++ b/frontend/src/views/deploy/DeployList.vue
@@ -134,6 +134,7 @@ export default {
    },
    onRefresh () {
      this.$store.dispatch('deploy/getDeployList')
+      this.$st.sendEv('部署', '刷新')
    },
    onView (row) {
      this.$router.push(`/deploys/${row._id}`)
--- a/frontend/src/views/layout/components/Navbar.vue
+++ b/frontend/src/views/layout/components/Navbar.vue
@@ -57,6 +57,8 @@ export default {
      window.localStorage.setItem('lang', lang)
      this.$i18n.locale = lang
      this.$store.commit('lang/SET_LANG', lang)
+
+      this.$st.sendEv('全局', '切换中英文', 'lang', lang)
    }
  }
 }
--- a/frontend/src/views/node/NodeDetail.vue
+++ b/frontend/src/views/node/NodeDetail.vue
@@ -47,6 +47,7 @@ export default {
    },
    onNodeChange (id) {
      this.$router.push(`/nodes/${id}`)
+      this.$st.sendEv('节点详情', '切换节点')
    }
  },
  created () {
--- a/frontend/src/views/node/NodeList.vue
+++ b/frontend/src/views/node/NodeList.vue
@@ -123,8 +123,7 @@ export default {
    }
  },
  methods: {
-    onSearch (value) {
-      console.log(value)
+    onSearch () {
    },
    onAdd () {
      this.$store.commit('node/SET_NODE_FORM', [])
@@ -133,6 +132,7 @@ export default {
    },
    onRefresh () {
      this.$store.dispatch('node/getNodeList')
+      this.$st.sendEv('节点', '刷新')
    },
    onSubmit () {
      const vm = this
@@ -159,7 +159,6 @@ export default {
      this.dialogVisible = false
    },
    onEdit (row) {
-      console.log(row)
      this.isEditMode = true
      this.$store.commit('node/SET_NODE_FORM', row)
      this.dialogVisible = true
@@ -177,10 +176,13 @@ export default {
              message: 'Deleted successfully'
            })
          })
+        this.$st.sendEv('节点', '删除', 'id', row._id)
      })
    },
    onView (row) {
      this.$router.push(`/nodes/${row._id}`)
+
+      this.$st.sendEv('节点', '查看', 'id', row._id)
    },
    onPageChange () {
      this.$store.dispatch('node/getNodeList')
--- a/frontend/src/views/schedule/ScheduleList.vue
+++ b/frontend/src/views/schedule/ScheduleList.vue
@@ -160,6 +160,7 @@ export default {
      this.isEdit = false
      this.dialogVisible = true
      this.$store.commit('schedule/SET_SCHEDULE_FORM', {})
+      this.$st.sendEv('定时任务', '添加')
    },
    onAddSubmit () {
      this.$refs.scheduleForm.validate(res => {
@@ -179,6 +180,7 @@ export default {
            })
        }
      })
+      this.$st.sendEv('定时任务', '提交')
    },
    isShowRun () {
    },
@@ -186,6 +188,7 @@ export default {
      this.$store.commit('schedule/SET_SCHEDULE_FORM', row)
      this.dialogVisible = true
      this.isEdit = true
+      this.$st.sendEv('定时任务', '修改', 'id', row._id)
    },
    onRemove (row) {
      this.$store.dispatch('schedule/removeSchedule', row._id)
@@ -195,6 +198,7 @@ export default {
            this.$message.success(`Schedule "${row.name}" has been removed`)
          }, 100)
        })
+      this.$st.sendEv('定时任务', '删除', 'id', row._id)
    },
    onCrawl () {
    }
--- a/frontend/src/views/site/SiteList.vue
+++ b/frontend/src/views/site/SiteList.vue
@@ -8,11 +8,11 @@
                v-model="keyword">
      </el-input>
      <el-select v-model="filter.mainCategory" class="filter-category" :placeholder="$t('Select Main Category')"
-                 clearable filterable>
+                 clearable filterable @change="onSelectMainCategory">
        <el-option v-for="op in mainCategoryList" :key="op" :value="op" :label="op"></el-option>
      </el-select>
      <el-select v-model="filter.category" class="filter-category" :placeholder="$t('Select Category')"
-                 clearable filterable>
+                 clearable filterable @change="onSelectCategory">
        <el-option v-for="op in categoryList" :key="op" :value="op" :label="op"></el-option>
      </el-select>
      <el-button type="success"
@@ -53,7 +53,8 @@
                         :width="col.width"
                         :align="col.align">
          <template slot-scope="scope">
-            <a class="domain" :href="'http://' + scope.row[col.name]" target="_blank">
+            <a class="domain" :href="'http://' + scope.row[col.name]" target="_blank"
+               @click="onClickDomain(scope.row._id)">
              {{scope.row[col.name]}}
            </a>
          </template>
@@ -84,7 +85,7 @@
          <template slot-scope="scope">
            <div>
              <template v-if="scope.row[col.name]">
-                <a :href="`http://${scope.row._id}/robots.txt`" target="_blank">
+                <a :href="`http://${scope.row._id}/robots.txt`" target="_blank" @click="onClickRobots(scope.row._id)">
                  Y
                </a>
              </template>
@@ -242,6 +243,16 @@ export default {
      setTimeout(() => {
        this.$store.dispatch('site/getSiteList')
      }, 0)
+      this.$st.sendEv('网站', '搜索')
+    },
+    onSelectMainCategory () {
+      this.$st.sendEv('网站', '选择主类别')
+    },
+    onSelectCategory () {
+      this.$st.sendEv('网站', '选择类别')
+    },
+    onClickDomain (domain) {
+      this.$st.sendEv('网站', '点击域名', 'domain', domain)
    },
    onPageChange () {
      setTimeout(() => {
@@ -302,6 +313,10 @@ export default {
    },
    goToSpiders (domain) {
      this.$router.push({ name: 'SpiderList', params: { domain } })
+      this.$st.sendEv('网站', '点击爬虫数', 'domain', domain)
+    },
+    onClickRobots (domain) {
+      this.$st.sendEv('网站', '点击Robots协议', 'domain', domain)
    }
  },
  created () {
--- a/frontend/src/views/spider/SpiderDetail.vue
+++ b/frontend/src/views/spider/SpiderDetail.vue
@@ -72,15 +72,17 @@ export default {
    }
  },
  methods: {
-    onTabClick () {
+    onTabClick (tab) {
      if (this.activeTabName === 'analytics') {
        setTimeout(() => {
          this.$refs['spider-stats'].update()
        }, 0)
      }
+      this.$st.sendEv('爬虫详情', '切换标签', 'tabName', tab.name)
    },
    onSpiderChange (id) {
      this.$router.push(`/spiders/${id}`)
+      this.$st.sendEv('爬虫详情', '切换爬虫')
    }
  },
  created () {
--- a/frontend/src/views/spider/SpiderList.vue
+++ b/frontend/src/views/spider/SpiderList.vue
@@ -292,13 +292,16 @@ export default {
      this.$store.commit('spider/SET_SPIDER_FORM', {})
      this.addDialogVisible = false
      this.addConfigurableDialogVisible = true
+      this.$st.sendEv('爬虫', '添加爬虫-可配置爬虫')
    },
    onAddCustomized () {
      this.addDialogVisible = false
      this.addCustomizedDialogVisible = true
+      this.$st.sendEv('爬虫', '添加爬虫-自定义爬虫')
    },
    onRefresh () {
      this.$store.dispatch('spider/getSpiderList')
+      this.$st.sendEv('爬虫', '刷新')
    },
    onSubmit () {
      const vm = this
@@ -354,6 +357,7 @@ export default {
              message: 'Deleted successfully'
            })
          })
+        this.$st.sendEv('爬虫', '删除')
      })
    },
    onDeploy (row) {
@@ -369,6 +373,7 @@ export default {
              message: 'Deployed successfully'
            })
          })
+        this.$st.sendEv('爬虫', '部署')
      })
    },
    onCrawl (row) {
@@ -381,10 +386,12 @@ export default {
            .then(() => {
              this.$message.success(this.$t(`Spider task has been scheduled`))
            })
+          this.$st.sendEv('爬虫', '运行')
        })
    },
    onView (row) {
      this.$router.push(`/spiders/${row._id}`)
+      this.$st.sendEv('爬虫', '查看')
    },
    onPageChange () {
      this.$store.dispatch('spider/getSpiderList')
@@ -408,6 +415,7 @@ export default {
            })
        }
      })
+      this.$st.sendEv('爬虫', '导入爬虫')
    },
    openImportDialog () {
      this.dialogVisible = true
@@ -423,6 +431,7 @@ export default {
            .then(() => {
              this.$message.success(this.$t('Deployed all spiders successfully'))
            })
+          this.$st.sendEv('爬虫', '部署所有爬虫')
        })
    },
    isShowRun (row) {
@@ -455,6 +464,7 @@ export default {
    },
    onSiteSelect (item) {
      this.$store.commit('spider/SET_FILTER_SITE', item._id)
+      this.$st.sendEv('爬虫', '搜索网站')
    },
    onAddConfigurableSiteSelect (item) {
      this.spiderForm.site = item._id
--- a/frontend/src/views/task/TaskDetail.vue
+++ b/frontend/src/views/task/TaskDetail.vue
@@ -15,11 +15,17 @@
        </el-card>
      </el-tab-pane>
      <el-tab-pane :label="$t('Results')" name="results">
+        <div class="button-group">
+          <el-button type="primary" icon="el-icon-download" @click="downloadCSV">
+            {{$t('Download CSV')}}
+          </el-button>
+        </div>
        <general-table-view :data="taskResultsData"
                            :columns="taskResultsColumns"
                            :page-num="resultsPageNum"
                            :page-size="resultsPageSize"
-                            :total="taskResultsTotalCount"/>
+                            :total="taskResultsTotalCount"
+                            @page-change="onResultsPageChange"/>
      </el-tab-pane>
    </el-tabs>
  </div>
@@ -74,10 +80,21 @@ export default {
    }
  },
  methods: {
-    onTabClick () {
+    onTabClick (tab) {
+      this.$st.sendEv('任务详情', '切换标签', 'tabName', tab.name)
    },
    onSpiderChange (id) {
      this.$router.push(`/spiders/${id}`)
+    },
+    onResultsPageChange (payload) {
+      const { pageNum, pageSize } = payload
+      this.resultsPageNum = pageNum
+      this.resultsPageSize = pageSize
+      this.$store.dispatch('task/getTaskResults', this.$route.params.id)
+    },
+    downloadCSV () {
+      window.location.href = this.$request.baseUrl + '/tasks/' + this.$route.params.id + '/download_results'
+      this.$st.sendEv('任务详情-结果', '下载CSV')
    }
  },
  created () {
@@ -114,4 +131,9 @@ export default {
    overflow-x: auto;
    overflow-y: auto;
  }
+
+  .button-group {
+    margin-bottom: 10px;
+    text-align: right;
+  }
 </style>
--- a/frontend/src/views/task/TaskList.vue
+++ b/frontend/src/views/task/TaskList.vue
@@ -3,10 +3,20 @@
    <!--filter-->
    <div class="filter">
      <div class="left">
-        <el-select class="filter-select" v-model="filter.node_id" :placeholder="$t('Node')" filterable clearable>
+        <el-select class="filter-select"
+                   v-model="filter.node_id"
+                   :placeholder="$t('Node')"
+                   filterable
+                   clearable
+                   @change="onSelectNode">
          <el-option v-for="op in nodeList" :key="op._id" :value="op._id" :label="op.name"></el-option>
        </el-select>
-        <el-select class="filter-select" v-model="filter.spider_id" :placeholder="$t('Spider')" filterable clearable>
+        <el-select class="filter-select"
+                   v-model="filter.spider_id"
+                   :placeholder="$t('Spider')"
+                   filterable
+                   clearable
+                   @change="onSelectSpider">
          <el-option v-for="op in spiderList" :key="op._id" :value="op._id" :label="op.name"></el-option>
        </el-select>
        <el-button type="success"
@@ -183,6 +193,13 @@ export default {
    },
    onRefresh () {
      this.$store.dispatch('task/getTaskList')
+      this.$st.sendEv('任务', '搜索')
+    },
+    onSelectNode () {
+      this.$st.sendEv('任务', '选择节点')
+    },
+    onSelectSpider () {
+      this.$st.sendEv('任务', '选择爬虫')
    },
    onRemove (row) {
      this.$confirm(this.$t('Are you sure to delete this task?'), this.$t('Notification'), {
@@ -197,16 +214,20 @@ export default {
              message: 'Deleted successfully'
            })
          })
+        this.$st.sendEv('任务', '删除', 'id', row._id)
      })
    },
    onView (row) {
      this.$router.push(`/tasks/${row._id}`)
+      this.$st.sendEv('任务', '搜索', 'id', row._id)
    },
    onClickSpider (row) {
      this.$router.push(`/spiders/${row.spider_id}`)
+      this.$st.sendEv('任务', '点击爬虫详情', 'id', row.spider_id)
    },
    onClickNode (row) {
      this.$router.push(`/nodes/${row.node_id}`)
+      this.$st.sendEv('任务', '点击节点详情', 'id', row.node_id)
    },
    onPageChange () {
      setTimeout(() => {