diff --git a/backend/go.sum b/backend/go.sum index cc056d70..55a56852 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -57,6 +57,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -70,6 +71,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -126,8 +128,10 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= diff --git a/backend/mock/node_test.go b/backend/mock/node_test.go index eca321ad..669cafc5 100644 --- a/backend/mock/node_test.go +++ b/backend/mock/node_test.go @@ -1,6 +1,7 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/gin-gonic/gin" @@ -8,12 +9,12 @@ import ( . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) var app *gin.Engine + // 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务 func init() { app = gin.Default() @@ -28,18 +29,18 @@ func init() { app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表 app.DELETE("/nodes/:id", DeleteNode) // 删除节点 //// 爬虫 - app.GET("/stats/home",GetHomeStats) // 首页统计数据 + app.GET("/stats/home", GetHomeStats) // 首页统计数据 // 定时任务 - app.GET("/schedules", GetScheduleList) // 定时任务列表 - app.GET("/schedules/:id", GetSchedule) // 定时任务详情 - app.PUT("/schedules", PutSchedule) // 创建定时任务 - app.POST("/schedules/:id", PostSchedule) // 修改定时任务 - app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 + app.GET("/schedules", GetScheduleList) // 定时任务列表 + app.GET("/schedules/:id", GetSchedule) // 定时任务详情 + app.PUT("/schedules", PutSchedule) // 创建定时任务 + app.POST("/schedules/:id", PostSchedule) // 修改定时任务 + app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务 app.GET("/tasks", GetTaskList) // 任务列表 app.GET("/tasks/:id", GetTask) // 任务详情 app.PUT("/tasks", PutTask) // 派发任务 app.DELETE("/tasks/:id", DeleteTask) // 删除任务 - app.GET("/tasks/:id/results",GetTaskResults) // 任务结果 + app.GET("/tasks/:id/results", GetTaskResults) // 任务结果 app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果 app.GET("/spiders", GetSpiderList) // 爬虫列表 app.GET("/spiders/:id", GetSpider) // 爬虫详情 @@ -55,7 +56,7 @@ func TestGetNodeList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -74,7 +75,7 @@ func TestGetNode(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -93,7 +94,7 @@ func TestPing(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/ping", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -111,7 +112,7 @@ func TestGetNodeTaskList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -130,7 +131,7 @@ func TestDeleteNode(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -162,10 +163,10 @@ func TestPostNode(t *testing.T) { var mongoId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() - req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "nodes/"+mongoId, bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("Unmarshal resp failed") @@ -184,7 +185,7 @@ func TestGetSystemInfo(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } diff --git a/backend/mock/schedule_test.go b/backend/mock/schedule_test.go index c24631b2..12843c75 100644 --- a/backend/mock/schedule_test.go +++ b/backend/mock/schedule_test.go @@ -1,7 +1,9 @@ package mock import ( + "bytes" "crawlab/model" + "crawlab/utils" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" @@ -17,7 +19,7 @@ func TestGetScheduleList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/schedules", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -36,7 +38,7 @@ func TestGetSchedule(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -56,7 +58,7 @@ func TestDeleteSchedule(t *testing.T) { req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -86,11 +88,12 @@ func TestPostSchedule(t *testing.T) { var resp Response var mongoId = "5d429e6c19f7abede924fee2" - body,_ := json.Marshal(newItem) + body, _ := json.Marshal(newItem) w := httptest.NewRecorder() - req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "/schedules/"+mongoId, strings.NewReader(utils.BytesToString(body))) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()),&resp) + + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("unmarshal resp failed") @@ -121,11 +124,11 @@ func TestPutSchedule(t *testing.T) { } var resp Response - body,_ := json.Marshal(newItem) + body, _ := json.Marshal(newItem) w := httptest.NewRecorder() - req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body))) + req, _ := http.NewRequest("PUT", "/schedules", bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()),&resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) t.Log(resp) if err != nil { t.Fatal("unmarshal resp failed") diff --git a/backend/mock/spider_test.go b/backend/mock/spider_test.go index 87634ff7..f4dbea63 100644 --- a/backend/mock/spider_test.go +++ b/backend/mock/spider_test.go @@ -1,13 +1,13 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) @@ -17,7 +17,7 @@ func TestGetSpiderList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp faild") } @@ -35,7 +35,7 @@ func TestGetSpider(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -66,9 +66,9 @@ func TestPostSpider(t *testing.T) { var spiderId = "5d429e6c19f7abede924fee2" w := httptest.NewRecorder() body, _ := json.Marshal(spider) - req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body))) + req, _ := http.NewRequest("POST", "/spiders/"+spiderId, bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -87,7 +87,7 @@ func TestGetSpiderDir(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -106,7 +106,7 @@ func TestGetSpiderTasks(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -124,7 +124,7 @@ func TestDeleteSpider(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -134,4 +134,4 @@ func TestDeleteSpider(t *testing.T) { So(resp.Message, ShouldEqual, "success") }) }) -} \ No newline at end of file +} diff --git a/backend/mock/stats_test.go b/backend/mock/stats_test.go index f2054f85..a94e52d4 100644 --- a/backend/mock/stats_test.go +++ b/backend/mock/stats_test.go @@ -14,7 +14,7 @@ func TestGetHomeStats(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/stats/home", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) fmt.Println(resp.Data) if err != nil { t.Fatal("Unmarshal resp failed") @@ -26,4 +26,4 @@ func TestGetHomeStats(t *testing.T) { So(resp.Message, ShouldEqual, "success") }) }) -} \ No newline at end of file +} diff --git a/backend/mock/task.go b/backend/mock/task.go index 84dece09..7b77d07e 100644 --- a/backend/mock/task.go +++ b/backend/mock/task.go @@ -186,7 +186,7 @@ func DownloadTaskResultsCsv(c *gin.Context) { bytesBuffer := &bytes.Buffer{} // 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码 - bytesBuffer.Write([]byte("\xEF\xBB\xBF")) + bytesBuffer.WriteString("\xEF\xBB\xBF") writer := csv.NewWriter(bytesBuffer) diff --git a/backend/mock/task_test.go b/backend/mock/task_test.go index 103ed643..1cd4ccfa 100644 --- a/backend/mock/task_test.go +++ b/backend/mock/task_test.go @@ -1,13 +1,13 @@ package mock import ( + "bytes" "crawlab/model" "encoding/json" "github.com/globalsign/mgo/bson" . "github.com/smartystreets/goconvey/convey" "net/http" "net/http/httptest" - "strings" "testing" "time" ) @@ -24,7 +24,7 @@ func TestGetTaskList(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -44,7 +44,7 @@ func TestGetTask(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } @@ -80,9 +80,9 @@ func TestPutTask(t *testing.T) { var resp Response body, _ := json.Marshal(&newItem) w := httptest.NewRecorder() - req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body))) + req, _ := http.NewRequest("PUT", "/tasks", bytes.NewReader(body)) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -100,7 +100,7 @@ func TestDeleteTask(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("unmarshal resp failed") } @@ -123,7 +123,7 @@ func TestGetTaskResults(t *testing.T) { w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil) app.ServeHTTP(w, req) - err := json.Unmarshal([]byte(w.Body.String()), &resp) + err := json.Unmarshal(w.Body.Bytes(), &resp) if err != nil { t.Fatal("Unmarshal resp failed") } diff --git a/backend/routes/file.go b/backend/routes/file.go index 435f1fba..eaf43ab5 100644 --- a/backend/routes/file.go +++ b/backend/routes/file.go @@ -1,6 +1,7 @@ package routes import ( + "crawlab/utils" "github.com/gin-gonic/gin" "io/ioutil" "net/http" @@ -15,6 +16,6 @@ func GetFile(c *gin.Context) { c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", - Data: string(fileBytes), + Data: utils.BytesToString(fileBytes), }) } diff --git a/backend/routes/spider.go b/backend/routes/spider.go index f1a3c9e5..76e5c568 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -327,7 +327,7 @@ func GetSpiderFile(c *gin.Context) { c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", - Data: string(fileBytes), + Data: utils.BytesToString(fileBytes), }) } diff --git a/backend/routes/task.go b/backend/routes/task.go index e5efa425..4dc42f6f 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -215,7 +215,7 @@ func DownloadTaskResultsCsv(c *gin.Context) { bytesBuffer := &bytes.Buffer{} // 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码 - bytesBuffer.Write([]byte("\xEF\xBB\xBF")) + bytesBuffer.WriteString("\xEF\xBB\xBF") writer := csv.NewWriter(bytesBuffer) diff --git a/backend/services/log.go b/backend/services/log.go index 088a825a..1344b02f 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -71,7 +71,7 @@ func GetRemoteLog(task model.Task) (logStr string, err error) { // 发布获取日志消息 channel := "nodes:" + task.NodeId.Hex() - if _, err := database.RedisClient.Publish(channel, string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish(channel, utils.BytesToString(msgBytes)); err != nil { log.Errorf(err.Error()) return "", err } diff --git a/backend/services/log_test.go b/backend/services/log_test.go index a0b049c5..1e9a21c7 100644 --- a/backend/services/log_test.go +++ b/backend/services/log_test.go @@ -2,6 +2,7 @@ package services import ( "crawlab/config" + "crawlab/utils" "fmt" "github.com/apex/log" . "github.com/smartystreets/goconvey/convey" @@ -32,13 +33,13 @@ func TestGetLocalLog(t *testing.T) { fmt.Println(err.Error()) } else { - _, err = f.Write([]byte("This is for test")) + _, err = f.WriteString("This is for test") } Convey("Test GetLocalLog", t, func() { Convey("Test response", func() { logStr, err := GetLocalLog(logPath) - log.Info(string(logStr)) + log.Info(utils.BytesToString(logStr)) fmt.Println(err) So(err, ShouldEqual, nil) diff --git a/backend/services/node.go b/backend/services/node.go index eb24f759..63373be8 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -7,6 +7,7 @@ import ( "crawlab/lib/cron" "crawlab/model" "crawlab/services/register" + "crawlab/utils" "encoding/json" "fmt" "github.com/apex/log" @@ -254,7 +255,7 @@ func UpdateNodeData() { debug.PrintStack() return } - if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil { + if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil { log.Errorf(err.Error()) return } @@ -280,7 +281,7 @@ func MasterNodeCallback(message redis.Message) (err error) { time.Sleep(10 * time.Millisecond) ch := SystemInfoChanMap.ChanBlocked(msg.NodeId) sysInfoBytes, _ := json.Marshal(&msg.SysInfo) - ch <- string(sysInfoBytes) + ch <- utils.BytesToString(sysInfoBytes) } return nil } @@ -304,14 +305,14 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 获取本地日志 logStr, err := GetLocalLog(msg.LogPath) - log.Info(string(logStr)) + log.Info(utils.BytesToString(logStr)) if err != nil { log.Errorf(err.Error()) debug.PrintStack() msgSd.Error = err.Error() msgSd.Log = err.Error() } else { - msgSd.Log = string(logStr) + msgSd.Log = utils.BytesToString(logStr) } // 序列化 @@ -322,7 +323,7 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 发布消息给主节点 log.Info("publish get log msg to master") - if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil { return err } @@ -347,7 +348,7 @@ func WorkerNodeCallback(message redis.Message) (err error) { debug.PrintStack() return err } - if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil { log.Errorf(err.Error()) return err } diff --git a/backend/services/spider.go b/backend/services/spider.go index 87b4a1d5..5763b3de 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -307,7 +307,7 @@ func PublishSpider(spider model.Spider) (err error) { return } channel := "files:upload" - if _, err = database.RedisClient.Publish(channel, string(msgStr)); err != nil { + if _, err = database.RedisClient.Publish(channel, utils.BytesToString(msgStr)); err != nil { log.Errorf(err.Error()) debug.PrintStack() return diff --git a/backend/services/system.go b/backend/services/system.go index ff177aa0..b30b2bc7 100644 --- a/backend/services/system.go +++ b/backend/services/system.go @@ -112,7 +112,7 @@ func GetRemoteSystemInfo(id string) (sysInfo model.SystemInfo, err error) { // 序列化 msgBytes, _ := json.Marshal(&msg) - if _, err := database.RedisClient.Publish("nodes:"+id, string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil { return model.SystemInfo{}, err } diff --git a/backend/services/task.go b/backend/services/task.go index 6ba6b257..9c34394c 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -36,7 +36,7 @@ func (m *TaskMessage) ToString() (string, error) { if err != nil { return "", err } - return string(data), err + return utils.BytesToString(data), err } // 任务执行器 @@ -405,13 +405,13 @@ func GetTaskLog(id string) (logStr string, err error) { if IsMasterNode(task.NodeId.Hex()) { // 若为主节点，获取本机日志 logBytes, err := GetLocalLog(task.LogPath) - logStr = string(logBytes) + logStr = utils.BytesToString(logBytes) if err != nil { log.Errorf(err.Error()) - logStr = string(err.Error()) + logStr = err.Error() // return "", err } else { - logStr = string(logBytes) + logStr = utils.BytesToString(logBytes) } } else { @@ -466,7 +466,7 @@ func CancelTask(id string) (err error) { } // 发布消息 - if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), string(msgBytes)); err != nil { + if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), utils.BytesToString(msgBytes)); err != nil { return err } } diff --git a/backend/utils/helpers.go b/backend/utils/helpers.go new file mode 100644 index 00000000..8e6de815 --- /dev/null +++ b/backend/utils/helpers.go @@ -0,0 +1,7 @@ +package utils + +import "unsafe" + +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +}