diff --git a/Jenkinsfile b/Jenkinsfile index 1188848d..16220039 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,7 +57,7 @@ pipeline { steps { echo 'Cleanup...' sh """ - docker image prune -f + docker rmi `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` """ } } diff --git a/README-zh.md b/README-zh.md index 24fef25e..6c2449f2 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,14 +1,14 @@ # Crawlab ![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/tikazyq/crawlab.svg) -![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) -![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/tikazyq/crawlab.svg) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) -中文 | [English](https://github.com/tikazyq/crawlab) +中文 | [English](https://github.com/crawlab-team/crawlab) [安装](#安装) | [运行](#运行) | [截图](#截图) | [架构](#架构) | [集成](#与其他框架的集成) | [比较](#与其他框架比较) | [相关文章](#相关文章) | [社区&赞助](#社区--赞助) @@ -202,7 +202,7 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 |框架 | 类型 | 分布式 | 前端 | 依赖于Scrapyd | |:---:|:---:|:---:|:---:|:---:| -| [Crawlab](https://github.com/tikazyq/crawlab) | 管理平台 | Y | Y | N +| [Crawlab](https://github.com/crawlab-team/crawlab) | 管理平台 | Y | Y | N | [ScrapydWeb](https://github.com/my8100/scrapydweb) | 管理平台 | Y | Y | Y | [SpiderKeeper](https://github.com/DormyMo/SpiderKeeper) | 管理平台 | Y | Y | Y | [Gerapy](https://github.com/Gerapy/Gerapy) | 管理平台 | Y | Y | Y diff --git a/README.md b/README.md index ca0df99a..9ae17a77 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # Crawlab ![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/tikazyq/crawlab.svg) -![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) -![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/tikazyq/crawlab.svg) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) -[中文](https://github.com/tikazyq/crawlab/blob/master/README-zh.md) | English +[中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English [Installation](#installation) | [Run](#run) | [Screenshot](#screenshot) | [Architecture](#architecture) | [Integration](#integration-with-other-frameworks) | [Compare](#comparison-with-other-frameworks) | [Community & Sponsorship](#community--sponsorship) @@ -199,7 +199,7 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any |Framework | Type | Distributed | Frontend | Scrapyd-Dependent | |:---:|:---:|:---:|:---:|:---:| -| [Crawlab](https://github.com/tikazyq/crawlab) | Admin Platform | Y | Y | N +| [Crawlab](https://github.com/crawlab-team/crawlab) | Admin Platform | Y | Y | N | [ScrapydWeb](https://github.com/my8100/scrapydweb) | Admin Platform | Y | Y | Y | [SpiderKeeper](https://github.com/DormyMo/SpiderKeeper) | Admin Platform | Y | Y | Y | [Gerapy](https://github.com/Gerapy/Gerapy) | Admin Platform | Y | Y | Y diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go index 9d7096b3..0cfd77ed 100644 --- a/backend/mock/node_test.go +++ b/backend/mock/node_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" "time" - "ucloudBilling/ucloud/log" ) var app *gin.Engine @@ -29,6 +28,7 @@ func init() { app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 //// 爬虫 + app.GET("/stats/home",GetHomeStats) // 首页统计数据 // 定时任务 app.GET("/schedules", GetScheduleList) // 定时任务列表 app.GET("/schedules/:id", GetSchedule) // 定时任务详情 @@ -44,7 +44,6 @@ func TestGetNodeList(t *testing.T) { req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) err := json.Unmarshal([]byte(w.Body.String()), &resp) - t.Log(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -148,7 +147,6 @@ func TestPostNode(t *testing.T) { var resp Response body, _ := json.Marshal(newItem) - log.Info(strings.NewReader(string(body))) var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() diff --git a/backend/mock/stats.go b/backend/mock/stats.go new file mode 100644 index 00000000..db2348c6 --- /dev/null +++ b/backend/mock/stats.go @@ -0,0 +1,64 @@ +package mock + +import ( + "crawlab/model" + "github.com/gin-gonic/gin" + "net/http" +) + + + +var taskDailyItems = []model.TaskDailyItem{ + { + Date: "2019/08/19", + TaskCount: 2, + AvgRuntimeDuration: 1000, + }, + { + Date: "2019/08/20", + TaskCount: 3, + AvgRuntimeDuration: 10130, + }, +} + +func GetHomeStats(c *gin.Context) { + type DataOverview struct { + TaskCount int `json:"task_count"` + SpiderCount int `json:"spider_count"` + ActiveNodeCount int `json:"active_node_count"` + ScheduleCount int `json:"schedule_count"` + } + + type Data struct { + Overview DataOverview `json:"overview"` + Daily []model.TaskDailyItem `json:"daily"` + } + + // 任务总数 + taskCount := 10 + + // 在线节点总数 + activeNodeCount := 4 + + // 爬虫总数 + spiderCount := 5 + // 定时任务数 + scheduleCount := 2 + + // 每日任务数 + items := taskDailyItems + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: Data{ + Overview: DataOverview{ + ActiveNodeCount: activeNodeCount, + TaskCount: taskCount, + SpiderCount: spiderCount, + ScheduleCount: scheduleCount, + }, + Daily: items, + }, + }) +} diff --git a/backend/mock/stats_test.go b/backend/mock/stats_test.go new file mode 100644 index 00000000..f2054f85 --- /dev/null +++ b/backend/mock/stats_test.go @@ -0,0 +1,29 @@ +package mock + +import ( + "encoding/json" + "fmt" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "testing" +) + +func TestGetHomeStats(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/stats/home", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + fmt.Println(resp.Data) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test API GetHomeStats", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} \ No newline at end of file