From 44e98a56a3d7c720deb4367e8912ea7a8e03c86f Mon Sep 17 00:00:00 2001 From: hantmac Date: Fri, 23 Aug 2019 20:28:11 +0800 Subject: [PATCH 001/122] unit test for stats.go --- backend/mock/node_test.go | 4 +-- backend/mock/stats.go | 64 ++++++++++++++++++++++++++++++++++++++ backend/mock/stats_test.go | 29 +++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 backend/mock/stats.go create mode 100644 backend/mock/stats_test.go diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go index 9d7096b3..0cfd77ed 100644 --- a/backend/mock/node_test.go +++ b/backend/mock/node_test.go @@ -11,7 +11,6 @@ import ( "strings" "testing" "time" - "ucloudBilling/ucloud/log" ) var app *gin.Engine @@ -29,6 +28,7 @@ func init() { app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 //// 爬虫 + app.GET("/stats/home",GetHomeStats) // 首页统计数据 // 定时任务 app.GET("/schedules", GetScheduleList) // 定时任务列表 app.GET("/schedules/:id", GetSchedule) // 定时任务详情 @@ -44,7 +44,6 @@ func TestGetNodeList(t *testing.T) { req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) err := json.Unmarshal([]byte(w.Body.String()), &resp) - t.Log(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -148,7 +147,6 @@ func TestPostNode(t *testing.T) { var resp Response body, _ := json.Marshal(newItem) - log.Info(strings.NewReader(string(body))) var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() diff --git a/backend/mock/stats.go b/backend/mock/stats.go new file mode 100644 index 00000000..db2348c6 --- /dev/null +++ b/backend/mock/stats.go @@ -0,0 +1,64 @@ +package mock + +import ( + "crawlab/model" + "github.com/gin-gonic/gin" + "net/http" +) + + + +var taskDailyItems = []model.TaskDailyItem{ + { + Date: "2019/08/19", + TaskCount: 2, + AvgRuntimeDuration: 1000, + }, + { + Date: "2019/08/20", + TaskCount: 3, + AvgRuntimeDuration: 10130, + }, +} + +func GetHomeStats(c *gin.Context) { + type DataOverview struct { + TaskCount int `json:"task_count"` + SpiderCount int `json:"spider_count"` + ActiveNodeCount int `json:"active_node_count"` + ScheduleCount int `json:"schedule_count"` + } + + type Data struct { + Overview DataOverview `json:"overview"` + Daily []model.TaskDailyItem `json:"daily"` + } + + // 任务总数 + taskCount := 10 + + // 在线节点总数 + activeNodeCount := 4 + + // 爬虫总数 + spiderCount := 5 + // 定时任务数 + scheduleCount := 2 + + // 每日任务数 + items := taskDailyItems + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: Data{ + Overview: DataOverview{ + ActiveNodeCount: activeNodeCount, + TaskCount: taskCount, + SpiderCount: spiderCount, + ScheduleCount: scheduleCount, + }, + Daily: items, + }, + }) +} diff --git a/backend/mock/stats_test.go b/backend/mock/stats_test.go new file mode 100644 index 00000000..f2054f85 --- /dev/null +++ b/backend/mock/stats_test.go @@ -0,0 +1,29 @@ +package mock + +import ( + "encoding/json" + "fmt" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "testing" +) + +func TestGetHomeStats(t *testing.T) { + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/stats/home", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + fmt.Println(resp.Data) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test API GetHomeStats", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} \ No newline at end of file From c41ef5105f21bbd974ca5ddb5c01871d6bdcf579 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 24 Aug 2019 12:53:00 +0800 Subject: [PATCH 002/122] updated Jenkinsfile --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1188848d..16220039 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,7 +57,7 @@ pipeline { steps { echo 'Cleanup...' sh """ - docker image prune -f + docker rmi `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` """ } } From efb5a653c8b0fe8c4047413f6c9ec781350e159a Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 24 Aug 2019 13:09:20 +0800 Subject: [PATCH 003/122] updated README --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ca0df99a..46bb3f2b 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ # Crawlab ![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/tikazyq/crawlab.svg) -![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) -![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) -![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/tikazyq/crawlab.svg) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) +![](https://img.shields.io/docker/pulls/crawlab-team/crawlab) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) -[中文](https://github.com/tikazyq/crawlab/blob/master/README-zh.md) | English +[中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English [Installation](#installation) | [Run](#run) | [Screenshot](#screenshot) | [Architecture](#architecture) | [Integration](#integration-with-other-frameworks) | [Compare](#comparison-with-other-frameworks) | [Community & Sponsorship](#community--sponsorship) @@ -199,7 +199,7 @@ Crawlab is easy to use, general enough to adapt spiders in any language and any |Framework | Type | Distributed | Frontend | Scrapyd-Dependent | |:---:|:---:|:---:|:---:|:---:| -| [Crawlab](https://github.com/tikazyq/crawlab) | Admin Platform | Y | Y | N +| [Crawlab](https://github.com/crawlab-team/crawlab) | Admin Platform | Y | Y | N | [ScrapydWeb](https://github.com/my8100/scrapydweb) | Admin Platform | Y | Y | Y | [SpiderKeeper](https://github.com/DormyMo/SpiderKeeper) | Admin Platform | Y | Y | Y | [Gerapy](https://github.com/Gerapy/Gerapy) | Admin Platform | Y | Y | Y From b4792680df7f0347f80d1cd4929cdf8270b53cf6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 24 Aug 2019 13:12:15 +0800 Subject: [PATCH 004/122] updated README --- README-zh.md | 14 +++++++------- README.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README-zh.md b/README-zh.md index 24fef25e..6c2449f2 100644 --- a/README-zh.md +++ b/README-zh.md @@ -1,14 +1,14 @@ # Crawlab ![](http://114.67.75.98:8082/buildStatus/icon?job=crawlab%2Fmaster) -![](https://img.shields.io/github/release/tikazyq/crawlab.svg) -![](https://img.shields.io/github/last-commit/tikazyq/crawlab.svg) -![](https://img.shields.io/github/issues/tikazyq/crawlab.svg) -![](https://img.shields.io/github/contributors/tikazyq/crawlab.svg) +![](https://img.shields.io/github/release/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) +![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) ![](https://img.shields.io/docker/pulls/tikazyq/crawlab) -![](https://img.shields.io/github/license/tikazyq/crawlab.svg) +![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) -中文 | [English](https://github.com/tikazyq/crawlab) +中文 | [English](https://github.com/crawlab-team/crawlab) [安装](#安装) | [运行](#运行) | [截图](#截图) | [架构](#架构) | [集成](#与其他框架的集成) | [比较](#与其他框架比较) | [相关文章](#相关文章) | [社区&赞助](#社区--赞助) @@ -202,7 +202,7 @@ Crawlab使用起来很方便，也很通用，可以适用于几乎任何主流 |框架 | 类型 | 分布式 | 前端 | 依赖于Scrapyd | |:---:|:---:|:---:|:---:|:---:| -| [Crawlab](https://github.com/tikazyq/crawlab) | 管理平台 | Y | Y | N +| [Crawlab](https://github.com/crawlab-team/crawlab) | 管理平台 | Y | Y | N | [ScrapydWeb](https://github.com/my8100/scrapydweb) | 管理平台 | Y | Y | Y | [SpiderKeeper](https://github.com/DormyMo/SpiderKeeper) | 管理平台 | Y | Y | Y | [Gerapy](https://github.com/Gerapy/Gerapy) | 管理平台 | Y | Y | Y diff --git a/README.md b/README.md index 46bb3f2b..9ae17a77 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![](https://img.shields.io/github/last-commit/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/issues/crawlab-team/crawlab.svg) ![](https://img.shields.io/github/contributors/crawlab-team/crawlab.svg) -![](https://img.shields.io/docker/pulls/crawlab-team/crawlab) +![](https://img.shields.io/docker/pulls/tikazyq/crawlab) ![](https://img.shields.io/github/license/crawlab-team/crawlab.svg) [中文](https://github.com/crawlab-team/crawlab/blob/master/README-zh.md) | English From 2c0ac66cb1da3f5102bcb5940114d7565429f8e3 Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 24 Aug 2019 13:39:06 +0800 Subject: [PATCH 005/122] Add unit test for task.go --- backend/mock/node.go | 4 +- backend/mock/node_test.go | 6 + backend/mock/schedule_test.go | 4 - backend/mock/task.go | 225 +++++++++++++++++++++++++++++++++- backend/mock/task_test.go | 138 +++++++++++++++++++++ 5 files changed, 370 insertions(+), 7 deletions(-) create mode 100644 backend/mock/task_test.go diff --git a/backend/mock/node.go b/backend/mock/node.go index 878dbcfa..4857a96d 100644 --- a/backend/mock/node.go +++ b/backend/mock/node.go @@ -42,7 +42,7 @@ var NodeList = []model.Node{ var TaskList = []model.Task{ { Id: "1234", - SpiderId: bson.ObjectId("xx429e6c19f7abede924fee2"), + SpiderId: bson.ObjectId("5d429e6c19f7abede924fee2"), StartTs: time.Now(), FinishTs: time.Now(), Status: "进行中", @@ -61,7 +61,7 @@ var TaskList = []model.Task{ }, { Id: "5678", - SpiderId: bson.ObjectId("xx429e6c19f7abede924fddf"), + SpiderId: bson.ObjectId("5d429e6c19f7abede924fee2"), StartTs: time.Now(), FinishTs: time.Now(), Status: "进行中", diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go index 0cfd77ed..cc2f94e5 100644 --- a/backend/mock/node_test.go +++ b/backend/mock/node_test.go @@ -35,6 +35,12 @@ func init() { app.PUT("/schedules", PutSchedule) // 创建定时任务 app.POST("/schedules/:id", PostSchedule) // 修改定时任务 app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 + app.GET("/tasks", GetTaskList) // 任务列表 + app.GET("/tasks/:id", GetTask) // 任务详情 + app.PUT("/tasks", PutTask) // 派发任务 + app.DELETE("/tasks/:id", DeleteTask) // 删除任务 + app.GET("/tasks/:id/results",GetTaskResults) // 任务结果 + app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果 } //mock test, test data in ./mock diff --git a/backend/mock/schedule_test.go b/backend/mock/schedule_test.go index d26a08d8..c24631b2 100644 --- a/backend/mock/schedule_test.go +++ b/backend/mock/schedule_test.go @@ -10,7 +10,6 @@ import ( "strings" "testing" "time" - "ucloudBilling/ucloud/log" ) func TestGetScheduleList(t *testing.T) { @@ -58,7 +57,6 @@ func TestDeleteSchedule(t *testing.T) { app.ServeHTTP(w, req) err := json.Unmarshal([]byte(w.Body.String()), &resp) - log.Info(w.Body.String()) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -89,7 +87,6 @@ func TestPostSchedule(t *testing.T) { var resp Response var mongoId = "5d429e6c19f7abede924fee2" body,_ := json.Marshal(newItem) - log.Info(strings.NewReader(string(body))) w := httptest.NewRecorder() req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body))) app.ServeHTTP(w, req) @@ -125,7 +122,6 @@ func TestPutSchedule(t *testing.T) { var resp Response body,_ := json.Marshal(newItem) - log.Info(strings.NewReader(string(body))) w := httptest.NewRecorder() req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body))) app.ServeHTTP(w, req) diff --git a/backend/mock/task.go b/backend/mock/task.go index c4807247..84dece09 100644 --- a/backend/mock/task.go +++ b/backend/mock/task.go @@ -1 +1,224 @@ -package mock \ No newline at end of file +package mock + +import ( + "bytes" + "crawlab/constants" + "crawlab/model" + "crawlab/utils" + "encoding/csv" + "fmt" + "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" + "github.com/satori/go.uuid" + "net/http" +) + +type TaskListRequestData struct { + PageNum int `form:"page_num"` + PageSize int `form:"page_size"` + NodeId string `form:"node_id"` + SpiderId string `form:"spider_id"` +} + +type TaskResultsRequestData struct { + PageNum int `form:"page_num"` + PageSize int `form:"page_size"` +} + +func GetTaskList(c *gin.Context) { + // 绑定数据 + data := TaskListRequestData{} + + if err := c.ShouldBindQuery(&data); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + if data.PageNum == 0 { + data.PageNum = 1 + } + if data.PageSize == 0 { + data.PageNum = 10 + } + + // 过滤条件 + query := bson.M{} + if data.NodeId != "" { + query["node_id"] = bson.ObjectIdHex(data.NodeId) + } + if data.SpiderId != "" { + query["spider_id"] = bson.ObjectIdHex(data.SpiderId) + } + + // 获取任务列表 + tasks := TaskList + + // 获取总任务数 + total := len(TaskList) + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Total: total, + Data: tasks, + }) +} + +func GetTask(c *gin.Context) { + id := c.Param("id") + + var result model.Task + for _, task := range TaskList { + if task.Id == id { + result = task + } + } + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + Data: result, + }) +} + +func PutTask(c *gin.Context) { + // 生成任务ID,generate task ID + id := uuid.NewV4() + + // 绑定数据 + var t model.Task + if err := c.ShouldBindJSON(&t); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + t.Id = id.String() + t.Status = constants.StatusPending + + // 如果没有传入node_id，则置为null + if t.NodeId.Hex() == "" { + t.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) + } + + // 将任务存入数据库,put the task into database + fmt.Println("put the task into database") + + // 加入任务队列, put the task into task queue + fmt.Println("put the task into task queue") + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func DeleteTask(c *gin.Context) { + id := c.Param("id") + + for _, task := range TaskList { + if task.Id == id { + fmt.Println("delete the task") + } + } + + c.JSON(http.StatusOK, Response{ + Status: "ok", + Message: "success", + }) +} + +func GetTaskResults(c *gin.Context) { + id := c.Param("id") + + // 绑定数据 + data := TaskResultsRequestData{} + if err := c.ShouldBindQuery(&data); err != nil { + HandleError(http.StatusBadRequest, c, err) + return + } + + // 获取任务 + var task model.Task + for _, ta := range TaskList { + if ta.Id == id { + task = ta + } + } + + fmt.Println(task) + // 获取结果 + var results interface{} + total := len(TaskList) + + c.JSON(http.StatusOK, ListResponse{ + Status: "ok", + Message: "success", + Data: results, + Total: total, + }) +} + +func DownloadTaskResultsCsv(c *gin.Context) { + id := c.Param("id") + + // 获取任务 + var task model.Task + for _, ta := range TaskList { + if ta.Id == id { + task = ta + } + } + fmt.Println(task) + + // 获取结果 + var results []interface { + } + + // 字段列表 + var columns []string + if len(results) == 0 { + columns = []string{} + } else { + item := results[0].(bson.M) + for key := range item { + columns = append(columns, key) + } + } + + // 缓冲 + bytesBuffer := &bytes.Buffer{} + + // 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码 + bytesBuffer.Write([]byte("\xEF\xBB\xBF")) + + writer := csv.NewWriter(bytesBuffer) + + // 写入表头 + if err := writer.Write(columns); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + + // 写入内容 + for _, result := range results { + // 将result转换为[]string + item := result.(bson.M) + var values []string + for _, col := range columns { + value := utils.InterfaceToString(item[col]) + values = append(values, value) + } + + // 写入数据 + if err := writer.Write(values); err != nil { + HandleError(http.StatusInternalServerError, c, err) + return + } + } + + // 此时才会将缓冲区数据写入 + writer.Flush() + + // 设置下载的文件名 + c.Writer.Header().Set("Content-Disposition", "attachment;filename=data.csv") + + // 设置文件类型以及输出数据 + c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes()) +} diff --git a/backend/mock/task_test.go b/backend/mock/task_test.go new file mode 100644 index 00000000..103ed643 --- /dev/null +++ b/backend/mock/task_test.go @@ -0,0 +1,138 @@ +package mock + +import ( + "crawlab/model" + "encoding/json" + "github.com/globalsign/mgo/bson" + . "github.com/smartystreets/goconvey/convey" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestGetTaskList(t *testing.T) { + //var teskListRequestFrom = TaskListRequestData{ + // PageNum: 2, + // PageSize: 10, + // NodeId: "434221grfsf", + // SpiderId: "fdfewqrftea", + //} + + var resp ListResponse + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test API GetNodeList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + So(resp.Total, ShouldEqual, 2) + }) + }) +} + +func TestGetTask(t *testing.T) { + var resp Response + var taskId = "1234" + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + Convey("Test API GetTask", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestPutTask(t *testing.T) { + var newItem = model.Task{ + Id: "1234", + SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + StartTs: time.Now(), + FinishTs: time.Now(), + Status: "online", + NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"), + LogPath: "./log", + Cmd: "scrapy crawl test", + Error: "", + ResultCount: 0, + WaitDuration: 10.0, + RuntimeDuration: 10, + TotalDuration: 20, + SpiderName: "test", + NodeName: "test", + CreateTs: time.Now(), + UpdateTs: time.Now(), + } + + var resp Response + body, _ := json.Marshal(&newItem) + w := httptest.NewRecorder() + req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body))) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API PutTask", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestDeleteTask(t *testing.T) { + taskId := "1234" + var resp Response + w := httptest.NewRecorder() + req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("unmarshal resp failed") + } + Convey("Test API DeleteTask", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + }) + }) +} + +func TestGetTaskResults(t *testing.T) { + //var teskListResultFrom = TaskResultsRequestData{ + // PageNum: 2, + // PageSize: 1, + //} + taskId := "1234" + + var resp ListResponse + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil) + app.ServeHTTP(w, req) + err := json.Unmarshal([]byte(w.Body.String()), &resp) + if err != nil { + t.Fatal("Unmarshal resp failed") + } + + Convey("Test API GetNodeList", t, func() { + Convey("Test response status", func() { + So(resp.Status, ShouldEqual, "ok") + So(resp.Message, ShouldEqual, "success") + So(resp.Total, ShouldEqual, 2) + }) + }) +} From ce8fb8670089aea2e6939dd3aee55fd28e4021f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 24 Aug 2019 15:11:52 +0800 Subject: [PATCH 006/122] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/services/node.go b/backend/services/node.go index 1fa2370c..09b49dbf 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -124,6 +124,7 @@ func IsMaster() bool { return viper.GetString("server.master") == Yes } +// 所有调用IsMasterNode的方法，都永远会在master节点执行，所以GetCurrentNode方法返回永远是master节点 // 该ID的节点是否为主节点 func IsMasterNode(id string) bool { curNode, _ := GetCurrentNode() From 3f249c8c6139aeabb772526776d926dd634c14ab Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 24 Aug 2019 15:16:27 +0800 Subject: [PATCH 007/122] bug fix:fix redis connection dead cycle --- backend/database/pubsub.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/backend/database/pubsub.go b/backend/database/pubsub.go index 4570e7b4..27b10687 100644 --- a/backend/database/pubsub.go +++ b/backend/database/pubsub.go @@ -1,9 +1,11 @@ package database import ( + "errors" "fmt" "github.com/apex/log" "github.com/gomodule/redigo/redis" + "time" "unsafe" ) @@ -23,7 +25,9 @@ func (c *Subscriber) Connect() { c.client = redis.PubSubConn{Conn: conn} c.cbMap = make(map[string]SubscribeCallback) - go func() { + //retry connect redis 5 times, or panic + index := 0 + go func(i int) { for { log.Debug("wait...") switch res := c.client.Receive().(type) { @@ -34,11 +38,24 @@ func (c *Subscriber) Connect() { case redis.Subscription: fmt.Printf("%s: %s %d\n", res.Channel, res.Kind, res.Count) case error: - log.Error("error handle...") + log.Error("error handle redis connection...") + con, err := GetRedisConn() + if err != nil { + log.Fatal("redis dial failed") + continue + } + c.client = redis.PubSubConn{Conn: con} + c.cbMap = make(map[string]SubscribeCallback) + time.Sleep(2 * time.Second) + if i > 5 { + panic(errors.New("redis connection failed too many times, panic")) + } + i += 1 + continue } } - }() + }(index) } From 561a9d260b970ad8f54620870009a340d3022f61 Mon Sep 17 00:00:00 2001 From: hantmac Date: Sat, 24 Aug 2019 15:50:53 +0800 Subject: [PATCH 008/122] remove redundant code --- backend/database/pubsub.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/database/pubsub.go b/backend/database/pubsub.go index 27b10687..b100535f 100644 --- a/backend/database/pubsub.go +++ b/backend/database/pubsub.go @@ -32,6 +32,7 @@ func (c *Subscriber) Connect() { log.Debug("wait...") switch res := c.client.Receive().(type) { case redis.Message: + i = 0 channel := (*string)(unsafe.Pointer(&res.Channel)) message := (*string)(unsafe.Pointer(&res.Data)) c.cbMap[*channel](*channel, *message) @@ -39,17 +40,17 @@ func (c *Subscriber) Connect() { fmt.Printf("%s: %s %d\n", res.Channel, res.Kind, res.Count) case error: log.Error("error handle redis connection...") - con, err := GetRedisConn() - if err != nil { - log.Fatal("redis dial failed") - continue - } - c.client = redis.PubSubConn{Conn: con} - c.cbMap = make(map[string]SubscribeCallback) + time.Sleep(2 * time.Second) if i > 5 { panic(errors.New("redis connection failed too many times, panic")) } + con, err := GetRedisConn() + if err != nil { + log.Error("redis dial failed") + continue + } + c.client = redis.PubSubConn{Conn: con} i += 1 continue From f6069a74fa255ec99a929e51465ace7db183fe24 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Sat, 24 Aug 2019 16:56:08 +0800 Subject: [PATCH 009/122] added language switch on login page --- frontend/src/assets/logo.svg | 14 +++++++ frontend/src/i18n/zh.js | 1 + .../src/views/layout/components/Navbar.vue | 2 +- frontend/src/views/login/index.vue | 38 ++++++++++++++++++- 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 frontend/src/assets/logo.svg diff --git a/frontend/src/assets/logo.svg b/frontend/src/assets/logo.svg new file mode 100644 index 00000000..b0e23910 --- /dev/null +++ b/frontend/src/assets/logo.svg @@ -0,0 +1,14 @@ + + + + + + + + + + diff --git a/frontend/src/i18n/zh.js b/frontend/src/i18n/zh.js index c8573bd4..b1de3b47 100644 --- a/frontend/src/i18n/zh.js +++ b/frontend/src/i18n/zh.js @@ -214,6 +214,7 @@ export default { // 下拉框 User: '用户', Logout: '退出登录', + Documentation: '文档', // 选择 'Yes': '是', diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 976d98d9..f60c0051 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -30,7 +30,7 @@ - 文档 + {{$t('Documentation')}} diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index 195ae1de..a21c0f42 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -4,7 +4,7 @@ diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 743a186e..c44d46e2 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -269,7 +269,7 @@ export default { }, created () { this.$store.dispatch('schedule/getScheduleList') - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') this.$store.dispatch('node/getNodeList') } } diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 69fdd770..916592f4 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -87,7 +87,7 @@ export default { }, created () { // get the list of the spiders - this.$store.dispatch('spider/getSpiderList') + // this.$store.dispatch('spider/getSpiderList') // get spider basic info this.$store.dispatch('spider/getSpiderData', this.$route.params.id) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 5b93afb4..743aabbe 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -111,12 +111,13 @@
- - + + - + @@ -156,8 +157,7 @@ align="left" :width="col.width"> { - console.log('resp', resp) + this.types = resp.data.data }) } }, From cacb12638576f1159dd6100396e81f85065fcc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 30 Sep 2019 16:44:35 +0800 Subject: [PATCH 100/122] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/package.json b/frontend/package.json index 20e40c7c..f5d170c8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.2.3", + "version": "0.3.2", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", From 0aa7d581b222bdba945b7eee7bb0c0daf8a070bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 30 Sep 2019 16:50:16 +0800 Subject: [PATCH 101/122] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/layout/components/Navbar.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 3b30c049..25d62e35 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -8,6 +8,9 @@ + + v0.3.2 + {{$t('Logout')}} From 6fbae93545fdfa919c4e311504d4f6dfd259f418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 30 Sep 2019 19:24:35 +0800 Subject: [PATCH 102/122] =?UTF-8?q?fix=20=E7=BC=96=E8=AF=91=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/mock/node.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/mock/node.go b/backend/mock/node.go index 4857a96d..789d0a9a 100644 --- a/backend/mock/node.go +++ b/backend/mock/node.go @@ -1,6 +1,7 @@ package mock import ( + "crawlab/entity" "crawlab/model" "crawlab/services" "github.com/apex/log" @@ -97,14 +98,14 @@ var dataList = []services.Data{ }, } -var executeble = []model.Executable{ +var executeble = []entity.Executable{ { Path: "/test", FileName: "test.py", DisplayName: "test.py", }, } -var systemInfo = model.SystemInfo{ARCH: "x86", +var systemInfo = entity.SystemInfo{ARCH: "x86", OS: "linux", Hostname: "test", NumCpu: 4, From 329b4a6470cfda7e82821ccbc2b336297c0aeb67 Mon Sep 17 00:00:00 2001 From: yaziming Date: Tue, 1 Oct 2019 09:58:24 +0800 Subject: [PATCH 103/122] fix(backend): fix mongo connect error when password has special characters --- backend/database/mongo.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/backend/database/mongo.go b/backend/database/mongo.go index 1c2d6433..e72baeaa 100644 --- a/backend/database/mongo.go +++ b/backend/database/mongo.go @@ -3,6 +3,7 @@ package database import ( "github.com/globalsign/mgo" "github.com/spf13/viper" + "net" "time" ) @@ -39,13 +40,28 @@ func InitMongo() error { var mongoAuth = viper.GetString("mongo.authSource") if Session == nil { - var uri string - if mongoUsername == "" { - uri = "mongodb://" + mongoHost + ":" + mongoPort + "/" + mongoDb - } else { - uri = "mongodb://" + mongoUsername + ":" + mongoPassword + "@" + mongoHost + ":" + mongoPort + "/" + mongoDb + "?authSource=" + mongoAuth + var dialInfo mgo.DialInfo + addr := net.JoinHostPort(mongoHost, mongoPort) + timeout := time.Second * 10 + dialInfo = mgo.DialInfo{ + Addrs: []string{addr}, + Timeout: timeout, + Database: mongoDb, + PoolLimit: 100, + PoolTimeout: timeout, + ReadTimeout: timeout, + WriteTimeout: timeout, + AppName: "crawlab", + FailFast: true, + MinPoolSize: 10, + MaxIdleTimeMS: 1000 * 30, } - sess, err := mgo.DialWithTimeout(uri, time.Second*5) + if mongoUsername != "" { + dialInfo.Username = mongoUsername + dialInfo.Password = mongoPassword + dialInfo.Source = mongoAuth + } + sess, err := mgo.DialWithInfo(&dialInfo) if err != nil { return err } From 17442fc89337012c8ea0e883e887325545b0c312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sun, 6 Oct 2019 18:46:51 +0800 Subject: [PATCH 104/122] add k8s deploy link --- README-zh.md | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README-zh.md b/README-zh.md index 6c2449f2..b4e2b469 100644 --- a/README-zh.md +++ b/README-zh.md @@ -21,6 +21,7 @@ 三种方式: 1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html)（推荐） 2. [直接部署](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html)（了解内核） +3. [Kubernetes](https://mp.weixin.qq.com/s/3Q1BQATUIEE_WXcHPqhYbA) ### 要求（Docker） - Docker 18.03+ diff --git a/README.md b/README.md index 9ae17a77..91a30b34 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Golang-based distributed web crawler management platform, supporting various lan Two methods: 1. [Docker](https://tikazyq.github.io/crawlab-docs/Installation/Docker.html) (Recommended) 2. [Direct Deploy](https://tikazyq.github.io/crawlab-docs/Installation/Direct.html) (Check Internal Kernel) +3. [Kubernetes](https://mp.weixin.qq.com/s/3Q1BQATUIEE_WXcHPqhYbA) ### Pre-requisite (Docker) - Docker 18.03+ From dabf5cacf156ebac3221340243db077d8a7eecde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 7 Oct 2019 12:21:32 +0800 Subject: [PATCH 105/122] =?UTF-8?q?fix=20=E5=88=9B=E5=BB=BA=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E9=94=99=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/spider.go | 2 +- backend/services/spider.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/routes/spider.go b/backend/routes/spider.go index addddd99..4c26fcee 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -135,7 +135,7 @@ func PutSpider(c *gin.Context) { // 以防tmp目录不存在 tmpPath := viper.GetString("other.tmppath") if !utils.Exists(tmpPath) { - if err := os.Mkdir(tmpPath, os.ModePerm); err != nil { + if err := os.MkdirAll(tmpPath, os.ModePerm); err != nil { log.Error("mkdir other.tmppath dir error:" + err.Error()) debug.PrintStack() HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error")) diff --git a/backend/services/spider.go b/backend/services/spider.go index a2e9a60f..c03ebe38 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -145,6 +145,7 @@ func PublishSpider(spider model.Spider) { // md5值不一样，则下载 md5Str := utils.ReadFileOneLine(md5) if gfFile.Md5 != md5Str { + log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str) spiderSync.RemoveSpiderFile() spiderSync.Download() spiderSync.CreateMd5File(gfFile.Md5) From 4a40d38844e88c27150babb4f6cba866bf9eddb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Mon, 7 Oct 2019 12:49:37 +0800 Subject: [PATCH 106/122] =?UTF-8?q?fix=20md5=E5=80=BC=E4=B8=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/services/spider.go b/backend/services/spider.go index c03ebe38..7aea456f 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -16,6 +16,7 @@ import ( "os" "path/filepath" "runtime/debug" + "strings" ) type SpiderFileData struct { @@ -144,6 +145,9 @@ func PublishSpider(spider model.Spider) { } // md5值不一样，则下载 md5Str := utils.ReadFileOneLine(md5) + // 去掉空格以及换行符 + md5Str = strings.Replace(md5Str, " ", "", -1) + md5Str = strings.Replace(md5Str, "\n", "", -1) if gfFile.Md5 != md5Str { log.Infof("md5 is different, gf-md5:%s, file-md5:%s", gfFile.Md5, md5Str) spiderSync.RemoveSpiderFile() From 10e8827dd3f716b7b954eb31d7a0f3820850e241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 14:46:03 +0800 Subject: [PATCH 107/122] =?UTF-8?q?fix=20=E9=9D=9E=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E7=88=AC=E8=99=AB=E7=9C=8B=E4=B8=8D=E5=88=B0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E7=9A=84=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/spider/SpiderDetail.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/spider/SpiderDetail.vue b/frontend/src/views/spider/SpiderDetail.vue index 916592f4..b42e750d 100644 --- a/frontend/src/views/spider/SpiderDetail.vue +++ b/frontend/src/views/spider/SpiderDetail.vue @@ -16,7 +16,7 @@ - + From 4c8b38f40bdc5b1376dd026d86a2f34b3d51dbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 19:41:29 +0800 Subject: [PATCH 108/122] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 0e8db964..6b0effe6 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -14,10 +14,12 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "runtime" "runtime/debug" "strconv" "sync" + "syscall" "time" ) @@ -142,7 +144,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e log.Infof("cancel process signal: %s", signal) if signal == constants.TaskCancel && cmd.Process != nil { // 取消进程 - if err := cmd.Process.Kill(); err != nil { + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { log.Errorf("process kill error: %s", err.Error()) debug.PrintStack() } @@ -152,6 +154,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e t.Status = constants.StatusFinished } t.FinishTs = time.Now() + t.Error = "user kill the process ..." if err := t.Save(); err != nil { log.Infof("save task error: %s", err.Error()) debug.PrintStack() @@ -159,6 +162,8 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e } }() + // 在选择所有节点执行的时候，实际就是随机一个节点执行的， + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} // 异步启动进程 if err := cmd.Start(); err != nil { log.Errorf("start spider error:{}", err.Error()) @@ -177,11 +182,12 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e log.Errorf("wait process finish error: %s", err.Error()) debug.PrintStack() + log.Infof("error type is : %s", reflect.TypeOf(err).String()) // 发生一次也需要保存 - t.Error = err.Error() - t.FinishTs = time.Now() - t.Status = constants.StatusFinished - _ = t.Save() + //t.Error = err.Error() + //t.FinishTs = time.Now() + //t.Status = constants.StatusError + //_ = t.Save() return err } ch <- constants.TaskFinish From a081b02c13440a8323e51f1b3ef7165cca5e1127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:16:49 +0800 Subject: [PATCH 109/122] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 6b0effe6..cf0f61a8 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -164,12 +164,14 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e // 在选择所有节点执行的时候，实际就是随机一个节点执行的， cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // 异步启动进程 if err := cmd.Start(); err != nil { log.Errorf("start spider error:{}", err.Error()) debug.PrintStack() return err } + // 保存pid到task t.Pid = cmd.Process.Pid if err := t.Save(); err != nil { @@ -183,11 +185,18 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e debug.PrintStack() log.Infof("error type is : %s", reflect.TypeOf(err).String()) - // 发生一次也需要保存 - //t.Error = err.Error() - //t.FinishTs = time.Now() - //t.Status = constants.StatusError - //_ = t.Save() + if exitError, ok := err.(*exec.ExitError); ok { + exitCode := exitError.ExitCode() + log.Errorf("exit error, exit code: %d", exitCode) + // 非kill 的错误类型 + if exitCode != 9 { + // 发生一次也需要保存 + t.Error = err.Error() + t.FinishTs = time.Now() + t.Status = constants.StatusError + _ = t.Save() + } + } return err } ch <- constants.TaskFinish From 6b90d86ea28b4559d3c02a328b718e0cb9f459d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:20:48 +0800 Subject: [PATCH 110/122] =?UTF-8?q?fix=20=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/services/task.go b/backend/services/task.go index cf0f61a8..9654e8d5 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -189,7 +189,7 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e exitCode := exitError.ExitCode() log.Errorf("exit error, exit code: %d", exitCode) // 非kill 的错误类型 - if exitCode != 9 { + if exitCode != -1 { // 发生一次也需要保存 t.Error = err.Error() t.FinishTs = time.Now() From f5af83dccee7d8b2662bbf741ffeca040f2e5602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 8 Oct 2019 20:26:08 +0800 Subject: [PATCH 111/122] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/task.go | 5 +---- frontend/package.json | 2 +- frontend/src/views/layout/components/Navbar.vue | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 9654e8d5..ce62a95e 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -14,7 +14,6 @@ import ( "os" "os/exec" "path/filepath" - "reflect" "runtime" "runtime/debug" "strconv" @@ -183,14 +182,12 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e if err := cmd.Wait(); err != nil { log.Errorf("wait process finish error: %s", err.Error()) debug.PrintStack() - - log.Infof("error type is : %s", reflect.TypeOf(err).String()) if exitError, ok := err.(*exec.ExitError); ok { exitCode := exitError.ExitCode() log.Errorf("exit error, exit code: %d", exitCode) // 非kill 的错误类型 if exitCode != -1 { - // 发生一次也需要保存 + // 非手动kill保存为错误状态 t.Error = err.Error() t.FinishTs = time.Now() t.Status = constants.StatusError diff --git a/frontend/package.json b/frontend/package.json index f5d170c8..60ac5cc8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "crawlab", - "version": "0.3.2", + "version": "0.3.4", "private": true, "scripts": { "serve": "vue-cli-service serve --ip=0.0.0.0 --mode=development", diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index 25d62e35..e294ad0c 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -9,7 +9,7 @@ - v0.3.2 + v0.3.4 {{$t('Logout')}} From 7fe2c9a42535fedada07e4efd9eed3cc73aec9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Thu, 10 Oct 2019 17:59:15 +0800 Subject: [PATCH 112/122] =?UTF-8?q?fix=20=E7=88=AC=E8=99=AB=E5=88=97?= =?UTF-8?q?=E8=A1=A8=E6=8C=89=E9=92=AE=E6=97=A0=E6=B3=95=E7=82=B9=E5=87=BB?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/spider/SpiderList.vue | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index 743aabbe..8c97339a 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -424,13 +424,18 @@ export default { this.dialogVisible = true }, isShowRun (row) { - if (this.isCustomized(row)) { - // customized spider - return !!row.cmd + if (row.cmd) { + return true } else { - // configurable spider - return !!row.fields + return false } + // if (this.isCustomized(row)) { + // // customized spider + // return !!row.cmd + // } else { + // // configurable spider + // return !!row.fields + // } }, isCustomized (row) { return row.type === 'customized' From 46d89c8cce2d417b089c8e80bb8f68e9059975e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Thu, 10 Oct 2019 19:56:04 +0800 Subject: [PATCH 113/122] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/msg_handler/handler.go | 2 ++ backend/services/msg_handler/msg_log.go | 3 +++ backend/services/node.go | 20 +++++++++++++------- backend/services/task.go | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/backend/services/msg_handler/handler.go b/backend/services/msg_handler/handler.go index 848e0c5d..b8b8e231 100644 --- a/backend/services/msg_handler/handler.go +++ b/backend/services/msg_handler/handler.go @@ -3,6 +3,7 @@ package msg_handler import ( "crawlab/constants" "crawlab/entity" + "github.com/apex/log" ) type Handler interface { @@ -10,6 +11,7 @@ type Handler interface { } func GetMsgHandler(msg entity.NodeMessage) Handler { + log.Infof("received msg , type is : %s", msg.Type) if msg.Type == constants.MsgTypeGetLog || msg.Type == constants.MsgTypeRemoveLog { // 日志相关 return &Log{ diff --git a/backend/services/msg_handler/msg_log.go b/backend/services/msg_handler/msg_log.go index 37080bd6..b865f4e3 100644 --- a/backend/services/msg_handler/msg_log.go +++ b/backend/services/msg_handler/msg_log.go @@ -40,8 +40,11 @@ func (g *Log) get() error { } // 发布消息给主节点 if err := utils.Pub(constants.ChannelMasterNode, msgSd); err != nil { + log.Errorf("pub log to master node error: %s", err.Error()) + debug.PrintStack() return err } + log.Infof(msgSd.Log) return nil } diff --git a/backend/services/node.go b/backend/services/node.go index 53af8d32..7fc134c5 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -110,13 +110,15 @@ func handleNodeInfo(key string, data Data) { if err := c.Find(bson.M{"key": key}).One(&node); err != nil { // 数据库不存在该节点 node = model.Node{ - Key: key, - Name: data.Ip, - Ip: data.Ip, - Port: "8000", - Mac: data.Mac, - Status: constants.StatusOnline, - IsMaster: data.Master, + Key: key, + Name: data.Ip, + Ip: data.Ip, + Port: "8000", + Mac: data.Mac, + Status: constants.StatusOnline, + IsMaster: data.Master, + UpdateTs: time.Now(), + UpdateTsUnix: time.Now().Unix(), } if err := node.Add(); err != nil { log.Errorf(err.Error()) @@ -125,6 +127,8 @@ func handleNodeInfo(key string, data Data) { } else { // 数据库存在该节点 node.Status = constants.StatusOnline + node.UpdateTs = time.Now() + node.UpdateTsUnix = time.Now().Unix() if err := node.Save(); err != nil { log.Errorf(err.Error()) return @@ -201,6 +205,8 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 反序列化 msg := utils.GetMessage(message) if err := msg_handler.GetMsgHandler(*msg).Handle(); err != nil { + log.Errorf("msg handler error: %s", err.Error()) + debug.PrintStack() return err } return nil diff --git a/backend/services/task.go b/backend/services/task.go index ce62a95e..f515f48d 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -148,12 +148,12 @@ func ExecuteShellCmd(cmdStr string, cwd string, t model.Task, s model.Spider) (e debug.PrintStack() } t.Status = constants.StatusCancelled + t.Error = "user kill the process ..." } else { // 保存任务 t.Status = constants.StatusFinished } t.FinishTs = time.Now() - t.Error = "user kill the process ..." if err := t.Save(); err != nil { log.Infof("save task error: %s", err.Error()) debug.PrintStack() From 973251a0fbe7a2184ac0da09e0404a17c736aee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 21:57:25 +0800 Subject: [PATCH 114/122] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/node.go | 6 ++++-- backend/model/schedule.go | 19 ++++++++++++++----- backend/model/spider.go | 11 ++++++++--- backend/model/task.go | 13 ++++--------- backend/routes/node.go | 10 +++++----- backend/services/node.go | 4 +++- 6 files changed, 38 insertions(+), 25 deletions(-) diff --git a/backend/model/node.go b/backend/model/node.go index 7af93dbe..1a1ebce5 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -157,10 +157,12 @@ func GetNodeList(filter interface{}) ([]Node, error) { } func GetNode(id bson.ObjectId) (Node, error) { + var node Node + if id.Hex() == "" { + return node, nil + } s, c := database.GetCol("nodes") defer s.Close() - - var node Node if err := c.FindId(id).One(&node); err != nil { if err != mgo.ErrNotFound { log.Errorf(err.Error()) diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 6415e22b..8ec065fb 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -38,6 +38,12 @@ func (sch *Schedule) Save() error { return nil } +func (sch *Schedule) Delete() error { + s, c := database.GetCol("schedules") + defer s.Close() + return c.RemoveId(sch.Id) +} + func GetScheduleList(filter interface{}) ([]Schedule, error) { s, c := database.GetCol("schedules") defer s.Close() @@ -47,11 +53,12 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { return schedules, err } - for i, schedule := range schedules { + var schs []Schedule + for _, schedule := range schedules { // 获取节点名称 if schedule.NodeId == bson.ObjectIdHex(constants.ObjectIdNull) { // 选择所有节点 - schedules[i].NodeName = "All Nodes" + schedule.NodeName = "All Nodes" } else { // 选择单一节点 node, err := GetNode(schedule.NodeId) @@ -59,7 +66,7 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { log.Errorf(err.Error()) continue } - schedules[i].NodeName = node.Name + schedule.NodeName = node.Name } // 获取爬虫名称 @@ -67,11 +74,13 @@ func GetScheduleList(filter interface{}) ([]Schedule, error) { if err != nil { log.Errorf("get spider by id: %s, error: %s", schedule.SpiderId.Hex(), err.Error()) debug.PrintStack() + _ = schedule.Delete() continue } - schedules[i].SpiderName = spider.Name + schedule.SpiderName = spider.Name + schs = append(schs, schedule) } - return schedules, nil + return schs, nil } func GetSchedule(id bson.ObjectId) (Schedule, error) { diff --git a/backend/model/spider.go b/backend/model/spider.go index 1f88acff..efd93c3d 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -98,6 +98,12 @@ func (spider *Spider) GetLastTask() (Task, error) { return tasks[0], nil } +func (spider *Spider) Delete() error { + s, c := database.GetCol("spiders") + defer s.Close() + return c.RemoveId(spider.Id) +} + // 爬虫列表 func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, int, error) { s, c := database.GetCol("spiders") @@ -256,15 +262,14 @@ func GetSpiderTypes() ([]*entity.SpiderType, error) { s, c := database.GetCol("spiders") defer s.Close() - group := bson.M{ "$group": bson.M{ - "_id": "$type", + "_id": "$type", "count": bson.M{"$sum": 1}, }, } var types []*entity.SpiderType - if err := c.Pipe([]bson.M{ group}).All(&types); err != nil { + if err := c.Pipe([]bson.M{group}).All(&types); err != nil { log.Errorf("get spider types error: %s", err.Error()) debug.PrintStack() return nil, err diff --git a/backend/model/task.go b/backend/model/task.go index f568b7fe..df046ecc 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -4,7 +4,6 @@ import ( "crawlab/constants" "crawlab/database" "github.com/apex/log" - "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" "runtime/debug" "time" @@ -118,20 +117,16 @@ func GetTaskList(filter interface{}, skip int, limit int, sortKey string) ([]Tas for i, task := range tasks { // 获取爬虫名称 spider, err := task.GetSpider() - if err == mgo.ErrNotFound { - // do nothing - } else if err != nil { - return tasks, err + if spider.Id.Hex() == "" || err != nil { + _ = spider.Delete() } else { tasks[i].SpiderName = spider.DisplayName } // 获取节点名称 node, err := task.GetNode() - if err == mgo.ErrNotFound { - // do nothing - } else if err != nil { - return tasks, err + if node.Id.Hex() == "" || err != nil { + _ = task.Delete() } else { tasks[i].NodeName = node.Name } diff --git a/backend/routes/node.go b/backend/routes/node.go index f86c152d..7d030773 100644 --- a/backend/routes/node.go +++ b/backend/routes/node.go @@ -15,9 +15,9 @@ func GetNodeList(c *gin.Context) { return } - for i, node := range nodes { - nodes[i].IsMaster = services.IsMasterNode(node.Id.Hex()) - } + //for i, node := range nodes { + // nodes[i].IsMaster = services.IsMasterNode(node.Id.Hex()) + //} c.JSON(http.StatusOK, Response{ Status: "ok", @@ -109,11 +109,11 @@ func GetSystemInfo(c *gin.Context) { }) } -func DeleteNode(c *gin.Context) { +func DeleteNode(c *gin.Context) { id := c.Param("id") node, err := model.GetNode(bson.ObjectIdHex(id)) if err != nil { - HandleError(http.StatusInternalServerError, c ,err) + HandleError(http.StatusInternalServerError, c, err) return } err = node.Delete() diff --git a/backend/services/node.go b/backend/services/node.go index 7fc134c5..144cdbd8 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -88,6 +88,8 @@ func UpdateNodeStatus() { handleNodeInfo(key, data) } + // 重新获取list + list, _ = database.RedisClient.HKeys("nodes") // 重置不在redis的key为offline model.ResetNodeStatusToOffline(list) } @@ -225,7 +227,7 @@ func InitNodeService() error { } // 首次更新节点数据（注册到Redis） - UpdateNodeData() + // UpdateNodeData() // 获取当前节点 node, err := model.GetCurrentNode() From 311f72da19094e3fa05ab4af49812f58843d8d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 21:57:59 +0800 Subject: [PATCH 115/122] =?UTF-8?q?fix=20=E7=9B=B4=E6=8E=A5=E7=82=B9?= =?UTF-8?q?=E5=87=BB=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E6=89=BE=E5=88=B0=E7=88=AC=E8=99=AB=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/schedule/ScheduleList.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index c44d46e2..743a186e 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -269,7 +269,7 @@ export default { }, created () { this.$store.dispatch('schedule/getScheduleList') - // this.$store.dispatch('spider/getSpiderList') + this.$store.dispatch('spider/getSpiderList') this.$store.dispatch('node/getNodeList') } } From 7dae91ab50a99901e03a72d52c673167ae4267de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 22:48:06 +0800 Subject: [PATCH 116/122] fix --- backend/model/schedule.go | 9 +++++++++ backend/routes/schedule.go | 6 +++--- backend/services/node.go | 14 +++++++------- backend/services/schedule.go | 18 +++++++++++++++--- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 8ec065fb..951cb043 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -16,6 +16,7 @@ type Schedule struct { Description string `json:"description" bson:"description"` SpiderId bson.ObjectId `json:"spider_id" bson:"spider_id"` NodeId bson.ObjectId `json:"node_id" bson:"node_id"` + NodeKey string `json:"node_key" bson:"node_key"` Cron string `json:"cron" bson:"cron"` EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` Param string `json:"param" bson:"param"` @@ -113,9 +114,17 @@ func AddSchedule(item Schedule) error { s, c := database.GetCol("schedules") defer s.Close() + node, err := GetNode(item.NodeId) + if err != nil { + log.Errorf("get node error: %s", err.Error()) + debug.PrintStack() + return nil + } + item.Id = bson.NewObjectId() item.CreateTs = time.Now() item.UpdateTs = time.Now() + item.NodeKey = node.Key if err := c.Insert(&item); err != nil { debug.PrintStack() diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index b447abb5..4ca245b3 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -81,9 +81,9 @@ func PutSchedule(c *gin.Context) { } // 如果node_id为空，则置为空ObjectId - if item.NodeId == "" { - item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) - } + //if item.NodeId == "" { + // item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) + //} // 更新数据库 if err := model.AddSchedule(item); err != nil { diff --git a/backend/services/node.go b/backend/services/node.go index 144cdbd8..04cbc0ef 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -100,13 +100,13 @@ func handleNodeInfo(key string, data Data) { defer s.Close() // 同个key可能因为并发，被注册多次 - var nodes []model.Node - _ = c.Find(bson.M{"key": key}).All(&nodes) - if nodes != nil && len(nodes) > 1 { - for _, node := range nodes { - _ = c.RemoveId(node.Id) - } - } + //var nodes []model.Node + //_ = c.Find(bson.M{"key": key}).All(&nodes) + //if nodes != nil && len(nodes) > 1 { + // for _, node := range nodes { + // _ = c.RemoveId(node.Id) + // } + //} var node model.Node if err := c.Find(bson.M{"key": key}).One(&node); err != nil { diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 58cdf628..f011f02a 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -17,7 +17,19 @@ type Scheduler struct { func AddTask(s model.Schedule) func() { return func() { - nodeId := s.NodeId + node, err := model.GetNodeByKey(s.NodeKey) + if err != nil || node.Id.Hex() == "" { + log.Errorf("get node by key error: %s", err.Error()) + debug.PrintStack() + return + } + + spider := model.GetSpiderByName(s.SpiderName) + if spider == nil || spider.Id.Hex() == "" { + log.Errorf("get spider by name error: %s", err.Error()) + debug.PrintStack() + return + } // 生成任务ID id := uuid.NewV4() @@ -25,8 +37,8 @@ func AddTask(s model.Schedule) func() { // 生成任务模型 t := model.Task{ Id: id.String(), - SpiderId: s.SpiderId, - NodeId: nodeId, + SpiderId: spider.Id, + NodeId: node.Id, Status: constants.StatusPending, Param: s.Param, } From 8eef98e082c9dcdd7423797cfcbb70cc100aa277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 22:48:22 +0800 Subject: [PATCH 117/122] fix --- frontend/src/views/schedule/ScheduleList.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index 743a186e..d16a6f69 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -16,7 +16,7 @@ - + Date: Fri, 11 Oct 2019 23:22:25 +0800 Subject: [PATCH 118/122] =?UTF-8?q?fix=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/log.go | 11 +++++++++-- backend/services/task.go | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/services/log.go b/backend/services/log.go index 485cb7dd..81140c0a 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "runtime/debug" + "time" ) // 任务日志频道映射 @@ -45,8 +46,14 @@ func GetRemoteLog(task model.Task) (logStr string, err error) { // 生成频道，等待获取log ch := TaskLogChanMap.ChanBlocked(task.Id) - // 此处阻塞，等待结果 - logStr = <-ch + select { + case logStr = <-ch: + log.Infof("get remote log") + break + case <-time.After(5 * time.Second): + logStr = "get remote log timeout" + break + } return logStr, nil } diff --git a/backend/services/task.go b/backend/services/task.go index f515f48d..12f0330e 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -285,6 +285,9 @@ func ExecuteTask(id int) { // 节点队列任务 var msg string msg, err = database.RedisClient.LPop(queueCur) + if msg != "" { + log.Infof("queue cur: %s", msg) + } if err != nil { if msg == "" { // 节点队列没有任务，获取公共队列任务 From d853948718b440a0f42a5c5dbf9f7e57df191459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 23:23:04 +0800 Subject: [PATCH 119/122] =?UTF-8?q?fix=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/schedule/ScheduleList.vue | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/schedule/ScheduleList.vue b/frontend/src/views/schedule/ScheduleList.vue index d16a6f69..4d283966 100644 --- a/frontend/src/views/schedule/ScheduleList.vue +++ b/frontend/src/views/schedule/ScheduleList.vue @@ -132,6 +132,7 @@ From 5bd30d8046ef5bafb483223a5a66f1f8531ed036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Fri, 11 Oct 2019 23:53:07 +0800 Subject: [PATCH 120/122] =?UTF-8?q?fix=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/schedule.go | 7 +++++++ backend/routes/schedule.go | 7 +++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 951cb043..bcd051e3 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -104,6 +104,13 @@ func UpdateSchedule(id bson.ObjectId, item Schedule) error { return err } + node, err := GetNode(item.NodeId) + if err != nil { + log.Errorf("get node error: %s", err.Error()) + debug.PrintStack() + return nil + } + item.NodeKey = node.Key if err := item.Save(); err != nil { return err } diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index 4ca245b3..24df0c0f 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -1,7 +1,6 @@ package routes import ( - "crawlab/constants" "crawlab/model" "crawlab/services" "github.com/gin-gonic/gin" @@ -49,9 +48,9 @@ func PostSchedule(c *gin.Context) { newItem.Id = bson.ObjectIdHex(id) // 如果node_id为空，则置为空ObjectId - if newItem.NodeId == "" { - newItem.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) - } + //if newItem.NodeId == "" { + // newItem.NodeId = bson.ObjectIdHex(constants.ObjectIdNull) + //} // 更新数据库 if err := model.UpdateSchedule(bson.ObjectIdHex(id), newItem); err != nil { From 9d8b0fd13767f60b4bf69ba69005b5df3acf4ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 12 Oct 2019 06:28:53 +0800 Subject: [PATCH 121/122] =?UTF-8?q?=E4=BF=AE=E6=94=B9Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 893cf6fe..52c668e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ADD ./frontend /app WORKDIR /app # install frontend +RUN npm config set unsafe-perm true RUN npm install -g yarn && yarn install RUN npm run build:prod @@ -56,4 +57,4 @@ EXPOSE 8080 EXPOSE 8000 # start backend -CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file +CMD ["/bin/sh", "/app/docker_init.sh"] From decb662c12361e4c6fc0290c8c885f319d1c7293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 12 Oct 2019 06:29:52 +0800 Subject: [PATCH 122/122] =?UTF-8?q?=E4=BF=AE=E6=94=B9Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 52c668e9..0809a0ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ WORKDIR /app # install frontend RUN npm config set unsafe-perm true -RUN npm install -g yarn && yarn install +RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org RUN npm run build:prod