mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
feat: added modules
This commit is contained in:
Marvin Zhang
5
core/stats/options.go
Normal file
5
core/stats/options.go
Normal file
@@ -0,0 +1,5 @@
|
||||
package stats
|
||||
|
||||
import "github.com/crawlab-team/crawlab/core/interfaces"
|
||||
|
||||
type Option func(svc interfaces.StatsService)
|
||||
323
core/stats/service.go
Normal file
323
core/stats/service.go
Normal file
@@ -0,0 +1,323 @@
|
||||
package stats
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab-db/mongo"
|
||||
"github.com/crawlab-team/crawlab/core/constants"
|
||||
"github.com/crawlab-team/crawlab/core/entity"
|
||||
"github.com/crawlab-team/crawlab/core/interfaces"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
mongo2 "go.mongodb.org/mongo-driver/mongo"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
}
|
||||
|
||||
func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) {
|
||||
stats := bson.M{}
|
||||
|
||||
// nodes
|
||||
stats["nodes"], err = mongo.GetMongoCol(interfaces.ModelColNameNode).Count(bson.M{"active": true})
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["nodes"] = 0
|
||||
}
|
||||
|
||||
// projects
|
||||
stats["projects"], err = mongo.GetMongoCol(interfaces.ModelColNameProject).Count(nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["projects"] = 0
|
||||
}
|
||||
|
||||
// spiders
|
||||
stats["spiders"], err = mongo.GetMongoCol(interfaces.ModelColNameSpider).Count(nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["spiders"] = 0
|
||||
}
|
||||
|
||||
// schedules
|
||||
stats["schedules"], err = mongo.GetMongoCol(interfaces.ModelColNameSchedule).Count(nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["schedules"] = 0
|
||||
}
|
||||
|
||||
// tasks
|
||||
stats["tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["tasks"] = 0
|
||||
}
|
||||
|
||||
// error tasks
|
||||
stats["error_tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(bson.M{"status": constants.TaskStatusError})
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["error_tasks"] = 0
|
||||
}
|
||||
|
||||
// results
|
||||
stats["results"], err = svc.getOverviewResults(query)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["results"] = 0
|
||||
}
|
||||
|
||||
// users
|
||||
stats["users"], err = mongo.GetMongoCol(interfaces.ModelColNameUser).Count(nil)
|
||||
if err != nil {
|
||||
if err.Error() != mongo2.ErrNoDocuments.Error() {
|
||||
return nil, err
|
||||
}
|
||||
stats["users"] = 0
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (svc *Service) GetDailyStats(query bson.M) (data interface{}, err error) {
|
||||
tasksStats, err := svc.getDailyTasksStats(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tasksStats, nil
|
||||
}
|
||||
|
||||
func (svc *Service) GetTaskStats(query bson.M) (data interface{}, err error) {
|
||||
stats := bson.M{}
|
||||
|
||||
// by status
|
||||
stats["by_status"], err = svc.getTaskStatsByStatus(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// by node
|
||||
stats["by_node"], err = svc.getTaskStatsByNode(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// by spider
|
||||
stats["by_spider"], err = svc.getTaskStatsBySpider(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{
|
||||
"$match", query,
|
||||
}},
|
||||
{{
|
||||
"$addFields",
|
||||
bson.M{
|
||||
"date": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"date": bson.M{"$toDate": "$_id"},
|
||||
"format": "%Y-%m-%d",
|
||||
"timezone": "Asia/Shanghai", // TODO: parameterization
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": "$date",
|
||||
"tasks": bson.M{"$sum": 1},
|
||||
"results": bson.M{"$sum": "$result_count"},
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$sort",
|
||||
bson.D{{"_id", 1}},
|
||||
}},
|
||||
}
|
||||
var results []entity.StatsDailyItem
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).All(&results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (svc *Service) getOverviewResults(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{"$match", query}},
|
||||
{{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": nil,
|
||||
"results": bson.M{"$sum": "$result_count"},
|
||||
},
|
||||
}},
|
||||
}
|
||||
var res bson.M
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).One(&res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res["results"], nil
|
||||
}
|
||||
|
||||
func (svc *Service) getTaskStatsByStatus(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{"$match", query}},
|
||||
{{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": "$status",
|
||||
"tasks": bson.M{"$sum": 1},
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$project",
|
||||
bson.M{
|
||||
"status": "$_id",
|
||||
"tasks": "$tasks",
|
||||
},
|
||||
}},
|
||||
}
|
||||
var results []bson.M
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{"$match", query}},
|
||||
{{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": "$node_id",
|
||||
"tasks": bson.M{"$sum": 1},
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$lookup",
|
||||
bson.M{
|
||||
"from": interfaces.ModelColNameNode,
|
||||
"localField": "_id",
|
||||
"foreignField": "_id",
|
||||
"as": "_n",
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$project",
|
||||
bson.M{
|
||||
"node_id": "$node_id",
|
||||
"node": bson.M{"$arrayElemAt": bson.A{"$_n", 0}},
|
||||
"node_name": bson.M{"$arrayElemAt": bson.A{"$_n.name", 0}},
|
||||
"tasks": "$tasks",
|
||||
},
|
||||
}},
|
||||
}
|
||||
var results []bson.M
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{"$match", query}},
|
||||
{{
|
||||
"$group",
|
||||
bson.M{
|
||||
"_id": "$spider_id",
|
||||
"tasks": bson.M{"$sum": 1},
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$lookup",
|
||||
bson.M{
|
||||
"from": interfaces.ModelColNameSpider,
|
||||
"localField": "_id",
|
||||
"foreignField": "_id",
|
||||
"as": "_s",
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$project",
|
||||
bson.M{
|
||||
"spider_id": "$spider_id",
|
||||
"spider": bson.M{"$arrayElemAt": bson.A{"$_s", 0}},
|
||||
"spider_name": bson.M{"$arrayElemAt": bson.A{"$_s.name", 0}},
|
||||
"tasks": "$tasks",
|
||||
},
|
||||
}},
|
||||
{{"$limit", 10}},
|
||||
}
|
||||
var results []bson.M
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err error) {
|
||||
pipeline := mongo2.Pipeline{
|
||||
{{"$match", query}},
|
||||
{{
|
||||
"$lookup",
|
||||
bson.M{
|
||||
"from": interfaces.ModelColNameTaskStat,
|
||||
"localField": "_id",
|
||||
"foreignField": "_id",
|
||||
"as": "_ts",
|
||||
},
|
||||
}},
|
||||
{{
|
||||
"$facet",
|
||||
bson.M{
|
||||
"total_duration": bson.A{
|
||||
bson.M{
|
||||
"$bucketAuto": bson.M{
|
||||
"groupBy": "$_ts.td",
|
||||
"buckets": 10,
|
||||
"granularity": "1-2-5",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
var res bson.M
|
||||
if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).One(&res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var svc interfaces.StatsService
|
||||
|
||||
func GetStatsService() interfaces.StatsService {
|
||||
if svc != nil {
|
||||
return svc
|
||||
}
|
||||
|
||||
// service
|
||||
svc = &Service{}
|
||||
|
||||
return svc
|
||||
}
|
||||
Reference in New Issue
Block a user