From e10d8fd9964ba5ced4fddf693bf06561604326e6 Mon Sep 17 00:00:00 2001 From: yaziming Date: Tue, 3 Sep 2019 01:07:59 +0800 Subject: [PATCH 1/4] refactor(backend): Use more efficient bytes to string methods and remove unnecessary type conversions detail: 1. add utils.BytesToString function instead of string() convert bytes to string. 2. use bytes.NewReader instead of strings.NewReader(string(sb)). 3. use w.Body.Bytes() instead of []byte(w.Body.String()). --- backend/go.sum | 4 ++++ backend/mock/node_test.go | 33 +++++++++++++++++---------------- backend/mock/schedule_test.go | 21 ++++++++++++--------- backend/mock/spider_test.go | 18 +++++++++--------- backend/mock/stats_test.go | 4 ++-- backend/mock/task.go | 2 +- backend/mock/task_test.go | 14 +++++++------- backend/routes/file.go | 3 ++- backend/routes/spider.go | 2 +- backend/routes/task.go | 2 +- backend/services/log.go | 2 +- backend/services/log_test.go | 5 +++-- backend/services/node.go | 13 +++++++------ backend/services/spider.go | 2 +- backend/services/system.go | 2 +- backend/services/task.go | 10 +++++----- backend/utils/helpers.go | 7 +++++++ 17 files changed, 81 insertions(+), 63 deletions(-) create mode 100644 backend/utils/helpers.go diff --git a/backend/go.sum b/backend/go.sum index cc056d70..55a56852 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -57,6 +57,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -70,6 +71,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -126,8 +128,10 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go index eca321ad..669cafc5 100644 --- a/backend/mock/node_test.go +++ b/backend/mock/node_test.go @@ -1,6 +1,7 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/gin-gonic/gin" @@ -8,12 +9,12 @@ import ( . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) var app *gin.Engine + // 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务 func init() { app = gin.Default() @@ -28,18 +29,18 @@ func init() { app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 //// 爬虫 - app.GET("/stats/home",GetHomeStats) // 首页统计数据 + app.GET("/stats/home", GetHomeStats) // 首页统计数据 // 定时任务 - app.GET("/schedules", GetScheduleList) // 定时任务列表 - app.GET("/schedules/:id", GetSchedule) // 定时任务详情 - app.PUT("/schedules", PutSchedule) // 创建定时任务 - app.POST("/schedules/:id", PostSchedule) // 修改定时任务 - app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 + app.GET("/schedules", GetScheduleList) // 定时任务列表 + app.GET("/schedules/:id", GetSchedule) // 定时任务详情 + app.PUT("/schedules", PutSchedule) // 创建定时任务 + app.POST("/schedules/:id", PostSchedule) // 修改定时任务 + app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 app.GET("/tasks", GetTaskList) // 任务列表 app.GET("/tasks/:id", GetTask) // 任务详情 app.PUT("/tasks", PutTask) // 派发任务 app.DELETE("/tasks/:id", DeleteTask) // 删除任务 - app.GET("/tasks/:id/results",GetTaskResults) // 任务结果 + app.GET("/tasks/:id/results", GetTaskResults) // 任务结果 app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果 app.GET("/spiders", GetSpiderList) // 爬虫列表 app.GET("/spiders/:id", GetSpider) // 爬虫详情 @@ -55,7 +56,7 @@ func TestGetNodeList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -74,7 +75,7 @@ func TestGetNode(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -93,7 +94,7 @@ func TestPing(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/ping", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -111,7 +112,7 @@ func TestGetNodeTaskList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -130,7 +131,7 @@ func TestDeleteNode(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -162,10 +163,10 @@ func TestPostNode(t *testing.T) { var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "nodes/"+mongoId, bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("Unmarshal resp failed") @@ -184,7 +185,7 @@ func TestGetSystemInfo(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } diff --git a/backend/mock/schedule_test.go b/backend/mock/schedule_test.go index c24631b2..12843c75 100644 --- a/backend/mock/schedule_test.go +++ b/backend/mock/schedule_test.go @@ -1,7 +1,9 @@ package mock import ( + "bytes" "crawlab/model" + "crawlab/utils" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" @@ -17,7 +19,7 @@ func TestGetScheduleList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/schedules", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -36,7 +38,7 @@ func TestGetSchedule(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -56,7 +58,7 @@ func TestDeleteSchedule(t *testing.T) { req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -86,11 +88,12 @@ func TestPostSchedule(t *testing.T) { var resp Response var mongoId = "5d429e6c19f7abede924fee2" - body,_ := json.Marshal(newItem) + body, _ := json.Marshal(newItem) w := httptest.NewRecorder() - req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "/schedules/"+mongoId, strings.NewReader(utils.BytesToString(body))) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()),&resp) + + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("unmarshal resp failed") @@ -121,11 +124,11 @@ func TestPutSchedule(t *testing.T) { } var resp Response - body,_ := json.Marshal(newItem) + body, _ := json.Marshal(newItem) w := httptest.NewRecorder() - req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body))) + req, _ := http.NewRequest("PUT", "/schedules", bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()),&resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("unmarshal resp failed") diff --git a/backend/mock/spider_test.go b/backend/mock/spider_test.go index 87634ff7..f4dbea63 100644 --- a/backend/mock/spider_test.go +++ b/backend/mock/spider_test.go @@ -1,13 +1,13 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) @@ -17,7 +17,7 @@ func TestGetSpiderList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp faild") } @@ -35,7 +35,7 @@ func TestGetSpider(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -66,9 +66,9 @@ func TestPostSpider(t *testing.T) { var spiderId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() body, _ := json.Marshal(spider) - req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "/spiders/"+spiderId, bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -87,7 +87,7 @@ func TestGetSpiderDir(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -106,7 +106,7 @@ func TestGetSpiderTasks(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -124,7 +124,7 @@ func TestDeleteSpider(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -134,4 +134,4 @@ func TestDeleteSpider(t *testing.T) { So(resp.Message, ShouldEqual, "success") }) }) -} \ No newline at end of file +} diff --git a/backend/mock/stats_test.go b/backend/mock/stats_test.go index f2054f85..a94e52d4 100644 --- a/backend/mock/stats_test.go +++ b/backend/mock/stats_test.go @@ -14,7 +14,7 @@ func TestGetHomeStats(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/stats/home", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) fmt.Println(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") @@ -26,4 +26,4 @@ func TestGetHomeStats(t *testing.T) { So(resp.Message, ShouldEqual, "success") }) }) -} \ No newline at end of file +} diff --git a/backend/mock/task.go b/backend/mock/task.go index 84dece09..7b77d07e 100644 --- a/backend/mock/task.go +++ b/backend/mock/task.go @@ -186,7 +186,7 @@ func DownloadTaskResultsCsv(c *gin.Context) { bytesBuffer := &bytes.Buffer{} // 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码 - bytesBuffer.Write([]byte("\xEF\xBB\xBF")) + bytesBuffer.WriteString("\xEF\xBB\xBF") writer := csv.NewWriter(bytesBuffer) diff --git a/backend/mock/task_test.go b/backend/mock/task_test.go index 103ed643..1cd4ccfa 100644 --- a/backend/mock/task_test.go +++ b/backend/mock/task_test.go @@ -1,13 +1,13 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) @@ -24,7 +24,7 @@ func TestGetTaskList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -44,7 +44,7 @@ func TestGetTask(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -80,9 +80,9 @@ func TestPutTask(t *testing.T) { var resp Response body, _ := json.Marshal(&newItem) w := httptest.NewRecorder() - req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body))) + req, _ := http.NewRequest("PUT", "/tasks", bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -100,7 +100,7 @@ func TestDeleteTask(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -123,7 +123,7 @@ func TestGetTaskResults(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } diff --git a/backend/routes/file.go b/backend/routes/file.go index 435f1fba..eaf43ab5 100644 --- a/backend/routes/file.go +++ b/backend/routes/file.go @@ -1,6 +1,7 @@ package routes import ( + "crawlab/utils" "github.com/gin-gonic/gin" "io/ioutil" "net/http" @@ -15,6 +16,6 @@ func GetFile(c *gin.Context) { c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", - Data: string(fileBytes), + Data: utils.BytesToString(fileBytes), }) } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index f1a3c9e5..76e5c568 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -327,7 +327,7 @@ func GetSpiderFile(c *gin.Context) { c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", - Data: string(fileBytes), + Data: utils.BytesToString(fileBytes), }) } diff --git a/backend/routes/task.go b/backend/routes/task.go index e5efa425..4dc42f6f 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -215,7 +215,7 @@ func DownloadTaskResultsCsv(c *gin.Context) { bytesBuffer := &bytes.Buffer{} // 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码 - bytesBuffer.Write([]byte("\xEF\xBB\xBF")) + bytesBuffer.WriteString("\xEF\xBB\xBF") writer := csv.NewWriter(bytesBuffer) diff --git a/backend/services/log.go b/backend/services/log.go index 088a825a..1344b02f 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -71,7 +71,7 @@ func GetRemoteLog(task model.Task) (logStr string, err error) { // 发布获取日志消息 channel := "nodes:" + task.NodeId.Hex() - if _, err := database.RedisClient.Publish(channel, string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish(channel, utils.BytesToString(msgBytes)); err != nil { log.Errorf(err.Error()) return "", err } diff --git a/backend/services/log_test.go b/backend/services/log_test.go index a0b049c5..1e9a21c7 100644 --- a/backend/services/log_test.go +++ b/backend/services/log_test.go @@ -2,6 +2,7 @@ package services import ( "crawlab/config" + "crawlab/utils" "fmt" "github.com/apex/log" . "github.com/smartystreets/goconvey/convey" @@ -32,13 +33,13 @@ func TestGetLocalLog(t *testing.T) { fmt.Println(err.Error()) } else { - _, err = f.Write([]byte("This is for test")) + _, err = f.WriteString("This is for test") } Convey("Test GetLocalLog", t, func() { Convey("Test response", func() { logStr, err := GetLocalLog(logPath) - log.Info(string(logStr)) + log.Info(utils.BytesToString(logStr)) fmt.Println(err) So(err, ShouldEqual, nil) diff --git a/backend/services/node.go b/backend/services/node.go index eb24f759..63373be8 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -7,6 +7,7 @@ import ( "crawlab/lib/cron" "crawlab/model" "crawlab/services/register" + "crawlab/utils" "encoding/json" "fmt" "github.com/apex/log" @@ -254,7 +255,7 @@ func UpdateNodeData() { debug.PrintStack() return } - if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil { + if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil { log.Errorf(err.Error()) return } @@ -280,7 +281,7 @@ func MasterNodeCallback(message redis.Message) (err error) { time.Sleep(10 * time.Millisecond) ch := SystemInfoChanMap.ChanBlocked(msg.NodeId) sysInfoBytes, _ := json.Marshal(&msg.SysInfo) - ch <- string(sysInfoBytes) + ch <- utils.BytesToString(sysInfoBytes) } return nil } @@ -304,14 +305,14 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 获取本地日志 logStr, err := GetLocalLog(msg.LogPath) - log.Info(string(logStr)) + log.Info(utils.BytesToString(logStr)) if err != nil { log.Errorf(err.Error()) debug.PrintStack() msgSd.Error = err.Error() msgSd.Log = err.Error() } else { - msgSd.Log = string(logStr) + msgSd.Log = utils.BytesToString(logStr) } // 序列化 @@ -322,7 +323,7 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 发布消息给主节点 log.Info("publish get log msg to master") - if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil { return err } @@ -347,7 +348,7 @@ func WorkerNodeCallback(message redis.Message) (err error) { debug.PrintStack() return err } - if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil { log.Errorf(err.Error()) return err } diff --git a/backend/services/spider.go b/backend/services/spider.go index 87b4a1d5..5763b3de 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -307,7 +307,7 @@ func PublishSpider(spider model.Spider) (err error) { return } channel := "files:upload" - if _, err = database.RedisClient.Publish(channel, string(msgStr)); err != nil { + if _, err = database.RedisClient.Publish(channel, utils.BytesToString(msgStr)); err != nil { log.Errorf(err.Error()) debug.PrintStack() return diff --git a/backend/services/system.go b/backend/services/system.go index ff177aa0..b30b2bc7 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -112,7 +112,7 @@ func GetRemoteSystemInfo(id string) (sysInfo model.SystemInfo, err error) { // 序列化 msgBytes, _ := json.Marshal(&msg) - if _, err := database.RedisClient.Publish("nodes:"+id, string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil { return model.SystemInfo{}, err } diff --git a/backend/services/task.go b/backend/services/task.go index 6ba6b257..9c34394c 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -36,7 +36,7 @@ func (m *TaskMessage) ToString() (string, error) { if err != nil { return "", err } - return string(data), err + return utils.BytesToString(data), err } // 任务执行器 @@ -405,13 +405,13 @@ func GetTaskLog(id string) (logStr string, err error) { if IsMasterNode(task.NodeId.Hex()) { // 若为主节点，获取本机日志 logBytes, err := GetLocalLog(task.LogPath) - logStr = string(logBytes) + logStr = utils.BytesToString(logBytes) if err != nil { log.Errorf(err.Error()) - logStr = string(err.Error()) + logStr = err.Error() // return "", err } else { - logStr = string(logBytes) + logStr = utils.BytesToString(logBytes) } } else { @@ -466,7 +466,7 @@ func CancelTask(id string) (err error) { } // 发布消息 - if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), utils.BytesToString(msgBytes)); err != nil { return err } } diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go new file mode 100644 index 00000000..8e6de815 --- /dev/null +++ b/backend/utils/helpers.go @@ -0,0 +1,7 @@ +package utils + +import "unsafe" + +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} From 2fb1754e56df64ea1a9de30bb312cf8669ba0515 Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 4 Sep 2019 13:51:02 +0800 Subject: [PATCH 2/4] bug fix:fix data race caused by LockList --- backend/services/task.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/services/task.go b/backend/services/task.go index 9c34394c..06e54572 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -16,14 +16,15 @@ import ( "runtime" "runtime/debug" "strconv" + "sync" "time" ) var Exec *Executor // 任务执行锁 -var LockList []bool - +//Added by cloud: 2019/09/04,solve data race +var LockList sync.Map // 任务消息 type TaskMessage struct { Id string @@ -56,7 +57,7 @@ func (ex *Executor) Start() error { id := i // 初始化任务锁 - LockList = append(LockList, false) + LockList.Store(id, false) // 加入定时任务 _, err := ex.Cron.AddFunc(spec, GetExecuteTaskFunc(id)) @@ -220,17 +221,18 @@ func SaveTaskResultCount(id string) func() { // 执行任务 func ExecuteTask(id int) { - if LockList[id] { + if flag, _ := LockList.Load(id); flag.(bool) { log.Debugf(GetWorkerPrefix(id) + "正在执行任务...") return } // 上锁 - LockList[id] = true + LockList.Store(id, true) // 解锁（延迟执行） defer func() { - LockList[id] = false + LockList.Delete(id) + LockList.Store(id, false) }() // 开始计时 From 4893439ffb0d10a9e53499ce3614da5e3f57d9b6 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 4 Sep 2019 20:13:12 +0800 Subject: [PATCH 3/4] added param to task --- backend/model/schedule.go | 1 + backend/model/task.go | 1 + backend/services/schedule.go | 1 + backend/services/task.go | 6 +- .../dgrijalva/jwt-go/request/doc.go | 7 -- .../dgrijalva/jwt-go/request/extractor.go | 81 ------------------- .../dgrijalva/jwt-go/request/oauth2.go | 28 ------- .../dgrijalva/jwt-go/request/request.go | 68 ---------------- backend/vendor/modules.txt | 23 +++++- 9 files changed, 29 insertions(+), 187 deletions(-) delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go delete mode 100644 backend/vendor/github.com/dgrijalva/jwt-go/request/request.go diff --git a/backend/model/schedule.go b/backend/model/schedule.go index 9f77c452..1c8db0bd 100644 --- a/backend/model/schedule.go +++ b/backend/model/schedule.go @@ -18,6 +18,7 @@ type Schedule struct { NodeId bson.ObjectId `json:"node_id" bson:"node_id"` Cron string `json:"cron" bson:"cron"` EntryId cron.EntryID `json:"entry_id" bson:"entry_id"` + Param string `json:"param" bson:"param"` // 前端展示 SpiderName string `json:"spider_name" bson:"spider_name"` diff --git a/backend/model/task.go b/backend/model/task.go index 968055a6..4957b577 100644 --- a/backend/model/task.go +++ b/backend/model/task.go @@ -19,6 +19,7 @@ type Task struct { NodeId bson.ObjectId `json:"node_id" bson:"node_id"` LogPath string `json:"log_path" bson:"log_path"` Cmd string `json:"cmd" bson:"cmd"` + Param string `json:"param" bson:"param"` Error string `json:"error" bson:"error"` ResultCount int `json:"result_count" bson:"result_count"` WaitDuration float64 `json:"wait_duration" bson:"wait_duration"` diff --git a/backend/services/schedule.go b/backend/services/schedule.go index 916e42d0..1c08e0fd 100644 --- a/backend/services/schedule.go +++ b/backend/services/schedule.go @@ -28,6 +28,7 @@ func AddTask(s model.Schedule) func() { SpiderId: s.SpiderId, NodeId: nodeId, Status: constants.StatusPending, + Param: s.Param, } // 将任务存入数据库 diff --git a/backend/services/task.go b/backend/services/task.go index 06e54572..dbaa2800 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -325,8 +325,10 @@ func ExecuteTask(id int) { // 执行命令 cmd := spider.Cmd - if t.Cmd != "" { - cmd = t.Cmd + + // 加入参数 + if t.Param != "" { + cmd += " " + t.Param } // 任务赋值 diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go deleted file mode 100644 index c01069c9..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Utility package for extracting JWT tokens from -// HTTP requests. -// -// The main function is ParseFromRequest and it's WithClaims variant. -// See examples for how to use the various Extractor implementations -// or roll your own. -package request diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go deleted file mode 100644 index 14414fe2..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go +++ /dev/null @@ -1,81 +0,0 @@ -package request - -import ( - "errors" - "net/http" -) - -// Errors -var ( - ErrNoTokenInRequest = errors.New("no token present in request") -) - -// Interface for extracting a token from an HTTP request. -// The ExtractToken method should return a token string or an error. -// If no token is present, you must return ErrNoTokenInRequest. -type Extractor interface { - ExtractToken(*http.Request) (string, error) -} - -// Extractor for finding a token in a header. Looks at each specified -// header in order until there's a match -type HeaderExtractor []string - -func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) { - // loop over header names and return the first one that contains data - for _, header := range e { - if ah := req.Header.Get(header); ah != "" { - return ah, nil - } - } - return "", ErrNoTokenInRequest -} - -// Extract token from request arguments. This includes a POSTed form or -// GET URL arguments. Argument names are tried in order until there's a match. -// This extractor calls `ParseMultipartForm` on the request -type ArgumentExtractor []string - -func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) { - // Make sure form is parsed - req.ParseMultipartForm(10e6) - - // loop over arg names and return the first one that contains data - for _, arg := range e { - if ah := req.Form.Get(arg); ah != "" { - return ah, nil - } - } - - return "", ErrNoTokenInRequest -} - -// Tries Extractors in order until one returns a token string or an error occurs -type MultiExtractor []Extractor - -func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) { - // loop over header names and return the first one that contains data - for _, extractor := range e { - if tok, err := extractor.ExtractToken(req); tok != "" { - return tok, nil - } else if err != ErrNoTokenInRequest { - return "", err - } - } - return "", ErrNoTokenInRequest -} - -// Wrap an Extractor in this to post-process the value before it's handed off. -// See AuthorizationHeaderExtractor for an example -type PostExtractionFilter struct { - Extractor - Filter func(string) (string, error) -} - -func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) { - if tok, err := e.Extractor.ExtractToken(req); tok != "" { - return e.Filter(tok) - } else { - return "", err - } -} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go deleted file mode 100644 index 5948694a..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go +++ /dev/null @@ -1,28 +0,0 @@ -package request - -import ( - "strings" -) - -// Strips 'Bearer ' prefix from bearer token string -func stripBearerPrefixFromTokenString(tok string) (string, error) { - // Should be a bearer token - if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " { - return tok[7:], nil - } - return tok, nil -} - -// Extract bearer token from Authorization header -// Uses PostExtractionFilter to strip "Bearer " prefix from header -var AuthorizationHeaderExtractor = &PostExtractionFilter{ - HeaderExtractor{"Authorization"}, - stripBearerPrefixFromTokenString, -} - -// Extractor for OAuth2 access tokens. Looks in 'Authorization' -// header then 'access_token' argument for a token. -var OAuth2Extractor = &MultiExtractor{ - AuthorizationHeaderExtractor, - ArgumentExtractor{"access_token"}, -} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go deleted file mode 100644 index 70525cfa..00000000 --- a/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go +++ /dev/null @@ -1,68 +0,0 @@ -package request - -import ( - "github.com/dgrijalva/jwt-go" - "net/http" -) - -// Extract and parse a JWT token from an HTTP request. -// This behaves the same as Parse, but accepts a request and an extractor -// instead of a token string. The Extractor interface allows you to define -// the logic for extracting a token. Several useful implementations are provided. -// -// You can provide options to modify parsing behavior -func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) { - // Create basic parser struct - p := &fromRequestParser{req, extractor, nil, nil} - - // Handle options - for _, option := range options { - option(p) - } - - // Set defaults - if p.claims == nil { - p.claims = jwt.MapClaims{} - } - if p.parser == nil { - p.parser = &jwt.Parser{} - } - - // perform extract - tokenString, err := p.extractor.ExtractToken(req) - if err != nil { - return nil, err - } - - // perform parse - return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc) -} - -// ParseFromRequest but with custom Claims type -// DEPRECATED: use ParseFromRequest and the WithClaims option -func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) { - return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims)) -} - -type fromRequestParser struct { - req *http.Request - extractor Extractor - claims jwt.Claims - parser *jwt.Parser -} - -type ParseFromRequestOption func(*fromRequestParser) - -// Parse with custom claims -func WithClaims(claims jwt.Claims) ParseFromRequestOption { - return func(p *fromRequestParser) { - p.claims = claims - } -} - -// Parse using a custom parser -func WithParser(parser *jwt.Parser) ParseFromRequestOption { - return func(p *fromRequestParser) { - p.parser = parser - } -} diff --git a/backend/vendor/modules.txt b/backend/vendor/modules.txt index 57c7d3f1..634c337b 100644 --- a/backend/vendor/modules.txt +++ b/backend/vendor/modules.txt @@ -2,7 +2,6 @@ github.com/apex/log # github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dgrijalva/jwt-go -github.com/dgrijalva/jwt-go/request # github.com/fsnotify/fsnotify v1.4.7 github.com/fsnotify/fsnotify # github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 @@ -18,11 +17,18 @@ github.com/globalsign/mgo/bson github.com/globalsign/mgo/internal/sasl github.com/globalsign/mgo/internal/scram github.com/globalsign/mgo/internal/json +# github.com/go-playground/locales v0.12.1 +github.com/go-playground/locales +github.com/go-playground/locales/currency +# github.com/go-playground/universal-translator v0.16.0 +github.com/go-playground/universal-translator # github.com/golang/protobuf v1.3.1 github.com/golang/protobuf/proto # github.com/gomodule/redigo v2.0.0+incompatible github.com/gomodule/redigo/redis github.com/gomodule/redigo/internal +# github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 +github.com/gopherjs/gopherjs/js # github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl github.com/hashicorp/hcl/hcl/printer @@ -36,6 +42,10 @@ github.com/hashicorp/hcl/json/scanner github.com/hashicorp/hcl/json/token # github.com/json-iterator/go v1.1.6 github.com/json-iterator/go +# github.com/jtolds/gls v4.20.0+incompatible +github.com/jtolds/gls +# github.com/leodido/go-urn v1.1.0 +github.com/leodido/go-urn # github.com/magiconair/properties v1.8.0 github.com/magiconair/properties # github.com/mattn/go-isatty v0.0.8 @@ -52,6 +62,15 @@ github.com/pelletier/go-toml github.com/pkg/errors # github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid +# github.com/smartystreets/assertions v1.0.0 +github.com/smartystreets/assertions +github.com/smartystreets/assertions/internal/go-diff/diffmatchpatch +github.com/smartystreets/assertions/internal/go-render/render +github.com/smartystreets/assertions/internal/oglematchers +# github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 +github.com/smartystreets/goconvey/convey +github.com/smartystreets/goconvey/convey/reporting +github.com/smartystreets/goconvey/convey/gotest # github.com/spf13/afero v1.1.2 github.com/spf13/afero github.com/spf13/afero/mem @@ -72,5 +91,7 @@ golang.org/x/text/transform golang.org/x/text/unicode/norm # gopkg.in/go-playground/validator.v8 v8.18.2 gopkg.in/go-playground/validator.v8 +# gopkg.in/go-playground/validator.v9 v9.29.1 +gopkg.in/go-playground/validator.v9 # gopkg.in/yaml.v2 v2.2.2 gopkg.in/yaml.v2 From 50132334811d1a6c58422c346d2fb81806e12c25 Mon Sep 17 00:00:00 2001 From: Marvin Zhang Date: Wed, 4 Sep 2019 20:13:28 +0800 Subject: [PATCH 4/4] added param to task --- frontend/src/components/Common/CrawlConfirmDialog.vue | 8 ++++++-- frontend/src/components/InfoView/TaskInfoView.vue | 3 +++ frontend/src/i18n/zh.js | 1 + frontend/src/store/modules/spider.js | 5 +++-- frontend/src/views/schedule/ScheduleList.vue | 7 ++++--- frontend/src/views/spider/SpiderList.vue | 4 ++-- frontend/src/views/task/TaskList.vue | 3 ++- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/Common/CrawlConfirmDialog.vue b/frontend/src/components/Common/CrawlConfirmDialog.vue index 06ef1dba..266ef2eb 100644 --- a/frontend/src/components/Common/CrawlConfirmDialog.vue +++ b/frontend/src/components/Common/CrawlConfirmDialog.vue @@ -19,6 +19,9 @@ /> + + + - + - + - +