added analytics for spider

crawlab/config/__init__.py

@@ -1,10 +1,3 @@
 # encoding: utf-8
 
-import os
-
-run_env = os.environ.get("RUNENV", "local")
-
-if run_env == "local":  # 加载本地配置
-    from config.config_local import *
-else:
-    from config.config import *
+from config.config import *

crawlab/config/config_local.py

@@ -1,55 +0,0 @@
-import os
-
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-# 爬虫源码路径
-PROJECT_SOURCE_FILE_FOLDER = os.path.join(BASE_DIR, "spiders")
-
-# 配置python虚拟环境的路径
-PYTHON_ENV_PATH = '/Users/yeqing/.pyenv/shims/python'
-
-# 爬虫部署路径
-# PROJECT_DEPLOY_FILE_FOLDER = '../deployfile'
-PROJECT_DEPLOY_FILE_FOLDER = '/var/crawlab'
-
-# 爬虫日志路径
-PROJECT_LOGS_FOLDER = '../deployfile/logs'
-
-# 打包临时文件夹
-PROJECT_TMP_FOLDER = '/tmp'
-
-# Celery中间者URL
-BROKER_URL = 'redis://127.0.0.1:6379/0'
-
-# Celery后台URL
-CELERY_RESULT_BACKEND = 'mongodb://127.0.0.1:27017/'
-
-# Celery MongoDB设置
-CELERY_MONGODB_BACKEND_SETTINGS = {
-    'database': 'crawlab_test',
-    'taskmeta_collection': 'tasks_celery',
-}
-
-# Celery时区
-CELERY_TIMEZONE = 'Asia/Shanghai'
-
-# 是否启用UTC
-CELERY_ENABLE_UTC = True
-
-# Celery Scheduler Redis URL
-CELERY_BEAT_SCHEDULER = 'utils.redisbeat.RedisScheduler'
-CELERY_REDIS_SCHEDULER_URL = 'redis://localhost:6379'
-CELERY_REDIS_SCHEDULER_KEY = 'celery:beat:order_tasks'
-
-# flower variables
-FLOWER_API_ENDPOINT = 'http://localhost:5555/api'
-
-# MongoDB 变量
-MONGO_HOST = '127.0.0.1'
-MONGO_PORT = 27017
-MONGO_DB = 'crawlab_test'
-
-# Flask 变量
-DEBUG = True
-FLASK_HOST = '127.0.0.1'
-FLASK_PORT = 8000

frontend/src/api/login 2.js

@@ -1,27 +0,0 @@
-import request from '@/utils/request'
-
-export function login (username, password) {
-  return request({
-    url: '/user/login',
-    method: 'post',
-    data: {
-      username,
-      password
-    }
-  })
-}
-
-export function getInfo (token) {
-  return request({
-    url: '/user/info',
-    method: 'get',
-    params: { token }
-  })
-}
-
-export function logout () {
-  return request({
-    url: '/user/logout',
-    method: 'post'
-  })
-}

frontend/src/api/request 2.js

@@ -1,46 +0,0 @@
-import axios from 'axios'
-
-let baseUrl = 'http://localhost:8000/api'
-if (process.env.NODE_ENV === 'production') {
-  baseUrl = 'http://139.129.230.98:8000/api'
-}
-// const baseUrl = process.env.API_BASE_URL || 'http://localhost:8000/api'
-
-const request = (method, path, params, data) => {
-  return new Promise((resolve, reject) => {
-    const url = `${baseUrl}${path}`
-    axios({
-      method,
-      url,
-      params,
-      data
-    })
-      .then(resolve)
-      .catch(reject)
-  })
-}
-
-const get = (path, params) => {
-  return request('GET', path, params)
-}
-
-const post = (path, data) => {
-  return request('POST', path, {}, data)
-}
-
-const put = (path, data) => {
-  return request('PUT', path, {}, data)
-}
-
-const del = (path, data) => {
-  return request('DELETE', path)
-}
-
-export default {
-  baseUrl,
-  request,
-  get,
-  post,
-  put,
-  delete: del
-}

frontend/src/api/request.js

@@ -4,6 +4,7 @@ let baseUrl = 'http://localhost:8000/api'
 if (process.env.NODE_ENV === 'production') {
   baseUrl = 'http://139.129.230.98:8000/api'
 }
+// console.log(process.env)
 // const baseUrl = process.env.API_BASE_URL || 'http://localhost:8000/api'
 
 const request = (method, path, params, data) => {

frontend/src/components/Stats/MetricCard.vue

@@ -0,0 +1,89 @@
+<template>
+  <el-card class="metric-card">
+    <el-col :span="6" class="icon-col">
+      <i :class="icon" :style="{color:color}"></i>
+    </el-col>
+    <el-col :span="18" class="text-col">
+      <el-row>
+        <label class="label">{{$t(label)}}</label>
+      </el-row>
+      <el-row>
+        <div class="value">{{value}}</div>
+      </el-row>
+    </el-col>
+  </el-card>
+</template>
+
+<script>
+export default {
+  name: 'MetricCard',
+  props: {
+    icon: {
+      type: String,
+      default: ''
+    },
+    label: {
+      type: String,
+      default: ''
+    },
+    value: {
+      type: String,
+      default: ''
+    },
+    type: {
+      type: String,
+      default: 'default'
+    }
+  },
+  computed: {
+    color () {
+      if (this.type === 'primary') {
+        return '#409EFF'
+      } else if (this.type === 'warning') {
+        return '#e6a23c'
+      } else if (this.type === 'success') {
+        return '#67c23a'
+      } else if (this.type === 'danger') {
+        return '#f56c6c'
+      } else {
+        return 'grey'
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+  .metric-card {
+    margin-right: 20px;
+  }
+
+  .metric-card:last-child {
+    margin-right: 0;
+  }
+
+  .metric-card .icon-col i {
+    margin-bottom: 15px;
+    font-size: 56px;
+  }
+
+  .metric-card .text-col {
+    padding-left: 10px;
+    height: 76px;
+    text-align: center;
+  }
+
+  .metric-card .text-col label {
+    font-size: 16px;
+    display: block;
+    height: 24px;
+    color: grey;
+    font-weight: 900;
+  }
+
+  .metric-card .text-col .value {
+    font-size: 24px;
+    display: block;
+    height: 32px;
+  }
+</style>

frontend/src/components/Stats/SpiderStats.vue

@@ -0,0 +1,175 @@
+<template>
+  <div class="spider-stats">
+    <!--overall stats-->
+    <el-row>
+      <div class="metric-list">
+        <metric-card :label="$t('30-Day Tasks')"
+                     icon="fa fa-play"
+                     :value="overviewStats.task_count"
+                     type="danger"/>
+        <metric-card :label="$t('30-Day Results')"
+                     icon="fa fa-table"
+                     :value="overviewStats.result_count"
+                     type="primary"/>
+        <metric-card :label="$t('Success Rate')"
+                     icon="fa fa-check"
+                     :value="getPercentStr(overviewStats.success_rate)"
+                     type="success"/>
+        <metric-card :label="$t('Avg Duration (sec)')"
+                     icon="fa fa-hourglass"
+                     :value="getDecimal(overviewStats.avg_duration)"
+                     type="warning"/>
+      </div>
+    </el-row>
+    <!--./overall stats-->
+
+    <el-row>
+      <el-col :span="8">
+        <el-card class="chart-wrapper" style="margin-right: 20px;">
+          <h4>{{$t('Tasks by Status')}}</h4>
+          <div id="task-pie" class="chart"></div>
+        </el-card>
+      </el-col>
+      <el-col :span="16">
+        <el-card class="chart-wrapper">
+          <h4>{{$t('Daily Tasks')}}</h4>
+          <div id="task-line" class="chart"></div>
+        </el-card>
+      </el-col>
+    </el-row>
+
+    <el-row>
+      <el-col :span="8">
+        <el-card class="chart-wrapper" style="margin-right: 20px;">
+          <h4>{{$t('Tasks by Status')}}</h4>
+          <el-table class="table"></el-table>
+        </el-card>
+      </el-col>
+      <el-col :span="16">
+        <el-card class="chart-wrapper">
+          <h4>{{$t('Tasks by Status')}}</h4>
+          <div id="duration-line" class="chart"></div>
+        </el-card>
+      </el-col>
+    </el-row>
+  </div>
+</template>
+
+<script>
+import {
+  mapState
+} from 'vuex'
+import MetricCard from './MetricCard'
+import echarts from 'echarts'
+
+export default {
+  name: 'SpiderStats',
+  components: { MetricCard },
+  methods: {
+    renderTaskPie () {
+      const chart = echarts.init(this.$el.querySelector('#task-pie'))
+      const option = {
+        series: [{
+          name: '',
+          type: 'pie',
+          radius: ['50%', '70%'],
+          data: this.statusStats.map(d => {
+            return {
+              name: this.$t(d.name),
+              value: d.value
+            }
+          })
+        }]
+      }
+      chart.setOption(option)
+    },
+
+    renderTaskLine () {
+      const chart = echarts.init(this.$el.querySelector('#task-line'))
+      const option = {
+        grid: {
+          top: 20,
+          bottom: 40
+        },
+        xAxis: {
+          type: 'category',
+          data: this.dailyStats.map(d => d.date)
+        },
+        yAxis: {
+          type: 'value'
+        },
+        series: [{
+          type: 'line',
+          data: this.dailyStats.map(d => d.count),
+          areaStyle: {},
+          smooth: true
+        }],
+        tooltip: {
+          trigger: 'axis',
+          show: true
+        }
+      }
+      chart.setOption(option)
+    },
+
+    render () {
+      this.renderTaskPie()
+      this.renderTaskLine()
+    },
+
+    update () {
+      this.$store.dispatch('spider/getSpiderStats')
+        .then(() => {
+          this.render()
+        })
+    },
+
+    getPercentStr (value) {
+      if (value === undefined) return 'NA'
+      return (value * 100).toFixed(2) + '%'
+    },
+
+    getDecimal (value) {
+      if (value === undefined) return 'NA'
+      return value.toFixed(2)
+    }
+  },
+  computed: {
+    ...mapState('spider', [
+      'overviewStats',
+      'statusStats',
+      'dailyStats'
+    ])
+  },
+  mounted () {
+  }
+}
+</script>
+
+<style scoped>
+  .metric-list {
+    display: flex;
+  }
+
+  .metric-list .metric-card {
+    flex-basis: 25%;
+  }
+
+  .chart-wrapper {
+    margin-top: 20px;
+  }
+
+  .chart {
+    width: 100%;
+    height: 240px;
+  }
+
+  .table {
+    height: 240px;
+  }
+
+  h4 {
+    display: inline-block;
+    margin: 0
+  }
+</style>

frontend/src/i18n/zh.js

@@ -18,6 +18,7 @@ export default {
   'Log': '日志',
   'Results': '结果',
   'Environment': '环境',
+  'Analytics': '分析',
 
   // 选择
   Spider: '爬虫',

frontend/src/store/modules/spider.js

@@ -14,7 +14,16 @@ const state = {
   importForm: {
     url: '',
     type: 'github'
-  }
+  },
+
+  // spider overview stats
+  overviewStats: {},
+
+  // spider status stats
+  statusStats: [],
+
+  // spider daily stats
+  dailyStats: []
 }
 
 const getters = {}
@@ -31,6 +40,15 @@ const mutations = {
   },
   SET_IMPORT_FORM (state, value) {
     state.importForm = value
+  },
+  SET_OVERVIEW_STATS (state, value) {
+    state.overviewStats = value
+  },
+  SET_STATUS_STATS (state, value) {
+    state.statusStats = value
+  },
+  SET_DAILY_STATS (state, value) {
+    state.dailyStats = value
   }
 }
 
@@ -138,6 +156,14 @@ const actions = {
       .then(response => {
         console.log(response)
       })
+  },
+  getSpiderStats ({ state, commit }) {
+    return request.get('/stats/get_spider_stats?spider_id=' + state.spiderForm._id)
+      .then(response => {
+        commit('SET_OVERVIEW_STATS', response.data.overview)
+        commit('SET_STATUS_STATS', response.data.task_count_by_status)
+        commit('SET_DAILY_STATS', response.data.daily_stats)
+      })
   }
 }
 

frontend/src/views/spider/SpiderDetail.vue

@@ -19,6 +19,9 @@
       <el-tab-pane :label="$t('Environment')" name="environment">
         <environment-list/>
       </el-tab-pane>
+      <el-tab-pane :label="$t('Analytics')" name="analytics">
+        <spider-stats ref="spider-stats"/>
+      </el-tab-pane>
     </el-tabs>
   </div>
 </template>
@@ -30,10 +33,12 @@ import {
 import FileList from '../../components/FileList/FileList'
 import SpiderOverview from '../../components/Overview/SpiderOverview'
 import EnvironmentList from '../../components/Environment/EnvironmentList'
+import SpiderStats from '../../components/Stats/SpiderStats'
 
 export default {
   name: 'NodeDetail',
   components: {
+    SpiderStats,
     EnvironmentList,
     FileList,
     SpiderOverview
@@ -57,6 +62,11 @@ export default {
   },
   methods: {
     onTabClick () {
+      if (this.activeTabName === 'analytics') {
+        setTimeout(() => {
+          this.$refs['spider-stats'].update()
+        }, 0)
+      }
     },
     onSpiderChange (id) {
       this.$router.push(`/spiders/${id}`)