Merge pull request #206 from yaziming/v0.4.0-use-unsafe.pointer-convert-bytes-to-string

refactor(backend): Use more efficient bytes to string methods and remove unnecessary type conversions

backend/go.sum

@@ -57,6 +57,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -70,6 +71,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -126,8 +128,10 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=

backend/mock/node_test.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/gin-gonic/gin"
@@ -8,12 +9,12 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
 
 var app *gin.Engine
+
 // 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务
 func init() {
 	app = gin.Default()
@@ -28,18 +29,18 @@ func init() {
 	app.GET("/nodes/:id/system", GetSystemInfo)  // 节点任务列表
 	app.DELETE("/nodes/:id", DeleteNode)         // 删除节点
 	//// 爬虫
-	app.GET("/stats/home",GetHomeStats) // 首页统计数据
+	app.GET("/stats/home", GetHomeStats) // 首页统计数据
 	// 定时任务
-	app.GET("/schedules", GetScheduleList)       // 定时任务列表
-	app.GET("/schedules/:id", GetSchedule)       // 定时任务详情
-	app.PUT("/schedules", PutSchedule)           // 创建定时任务
-	app.POST("/schedules/:id", PostSchedule)     // 修改定时任务
-	app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务
+	app.GET("/schedules", GetScheduleList)                         // 定时任务列表
+	app.GET("/schedules/:id", GetSchedule)                         // 定时任务详情
+	app.PUT("/schedules", PutSchedule)                             // 创建定时任务
+	app.POST("/schedules/:id", PostSchedule)                       // 修改定时任务
+	app.DELETE("/schedules/:id", DeleteSchedule)                   // 删除定时任务
 	app.GET("/tasks", GetTaskList)                                 // 任务列表
 	app.GET("/tasks/:id", GetTask)                                 // 任务详情
 	app.PUT("/tasks", PutTask)                                     // 派发任务
 	app.DELETE("/tasks/:id", DeleteTask)                           // 删除任务
-	app.GET("/tasks/:id/results",GetTaskResults)                  // 任务结果
+	app.GET("/tasks/:id/results", GetTaskResults)                  // 任务结果
 	app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果
 	app.GET("/spiders", GetSpiderList)              // 爬虫列表
 	app.GET("/spiders/:id", GetSpider)              // 爬虫详情
@@ -55,7 +56,7 @@ func TestGetNodeList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/nodes", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -74,7 +75,7 @@ func TestGetNode(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -93,7 +94,7 @@ func TestPing(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/ping", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -111,7 +112,7 @@ func TestGetNodeTaskList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -130,7 +131,7 @@ func TestDeleteNode(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -162,10 +163,10 @@ func TestPostNode(t *testing.T) {
 
 	var mongoId = "5d429e6c19f7abede924fee2"
 	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "nodes/"+mongoId, bytes.NewReader(body))
 
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
@@ -184,7 +185,7 @@ func TestGetSystemInfo(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}

backend/mock/schedule_test.go

@@ -1,7 +1,9 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
+	"crawlab/utils"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
@@ -17,7 +19,7 @@ func TestGetScheduleList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/schedules", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -36,7 +38,7 @@ func TestGetSchedule(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -56,7 +58,7 @@ func TestDeleteSchedule(t *testing.T) {
 	req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil)
 	app.ServeHTTP(w, req)
 
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -86,11 +88,12 @@ func TestPostSchedule(t *testing.T) {
 
 	var resp Response
 	var mongoId = "5d429e6c19f7abede924fee2"
-	body,_ := json.Marshal(newItem)
+	body, _ := json.Marshal(newItem)
 	w := httptest.NewRecorder()
-	req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "/schedules/"+mongoId, strings.NewReader(utils.BytesToString(body)))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
@@ -121,11 +124,11 @@ func TestPutSchedule(t *testing.T) {
 	}
 
 	var resp Response
-	body,_ := json.Marshal(newItem)
+	body, _ := json.Marshal(newItem)
 	w := httptest.NewRecorder()
-	req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body)))
+	req, _ := http.NewRequest("PUT", "/schedules", bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")

backend/mock/spider_test.go

@@ -1,13 +1,13 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
@@ -17,7 +17,7 @@ func TestGetSpiderList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp faild")
 	}
@@ -35,7 +35,7 @@ func TestGetSpider(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -66,9 +66,9 @@ func TestPostSpider(t *testing.T) {
 	var spiderId = "5d429e6c19f7abede924fee2"
 	w := httptest.NewRecorder()
 	body, _ := json.Marshal(spider)
-	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -87,7 +87,7 @@ func TestGetSpiderDir(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -106,7 +106,7 @@ func TestGetSpiderTasks(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -124,7 +124,7 @@ func TestDeleteSpider(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -134,4 +134,4 @@ func TestDeleteSpider(t *testing.T) {
 			So(resp.Message, ShouldEqual, "success")
 		})
 	})
-}
+}

backend/mock/stats_test.go

@@ -14,7 +14,7 @@ func TestGetHomeStats(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/stats/home", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	fmt.Println(resp.Data)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
@@ -26,4 +26,4 @@ func TestGetHomeStats(t *testing.T) {
 			So(resp.Message, ShouldEqual, "success")
 		})
 	})
-}
+}

backend/mock/task.go

@@ -186,7 +186,7 @@ func DownloadTaskResultsCsv(c *gin.Context) {
 	bytesBuffer := &bytes.Buffer{}
 
 	// 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码
-	bytesBuffer.Write([]byte("\xEF\xBB\xBF"))
+	bytesBuffer.WriteString("\xEF\xBB\xBF")
 
 	writer := csv.NewWriter(bytesBuffer)
 

backend/mock/task_test.go

@@ -1,13 +1,13 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
@@ -24,7 +24,7 @@ func TestGetTaskList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -44,7 +44,7 @@ func TestGetTask(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -80,9 +80,9 @@ func TestPutTask(t *testing.T) {
 	var resp Response
 	body, _ := json.Marshal(&newItem)
 	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body)))
+	req, _ := http.NewRequest("PUT", "/tasks", bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -100,7 +100,7 @@ func TestDeleteTask(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -123,7 +123,7 @@ func TestGetTaskResults(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}

backend/routes/file.go

@@ -1,6 +1,7 @@
 package routes
 
 import (
+	"crawlab/utils"
 	"github.com/gin-gonic/gin"
 	"io/ioutil"
 	"net/http"
@@ -15,6 +16,6 @@ func GetFile(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    string(fileBytes),
+		Data:    utils.BytesToString(fileBytes),
 	})
 }

backend/routes/spider.go

@@ -327,7 +327,7 @@ func GetSpiderFile(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    string(fileBytes),
+		Data:    utils.BytesToString(fileBytes),
 	})
 }
 

backend/routes/task.go

@@ -215,7 +215,7 @@ func DownloadTaskResultsCsv(c *gin.Context) {
 	bytesBuffer := &bytes.Buffer{}
 
 	// 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码
-	bytesBuffer.Write([]byte("\xEF\xBB\xBF"))
+	bytesBuffer.WriteString("\xEF\xBB\xBF")
 
 	writer := csv.NewWriter(bytesBuffer)
 

backend/services/log.go

@@ -71,7 +71,7 @@ func GetRemoteLog(task model.Task) (logStr string, err error) {
 
 	// 发布获取日志消息
 	channel := "nodes:" + task.NodeId.Hex()
-	if _, err := database.RedisClient.Publish(channel, string(msgBytes)); err != nil {
+	if _, err := database.RedisClient.Publish(channel, utils.BytesToString(msgBytes)); err != nil {
 		log.Errorf(err.Error())
 		return "", err
 	}

backend/services/log_test.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"crawlab/config"
+	"crawlab/utils"
 	"fmt"
 	"github.com/apex/log"
 	. "github.com/smartystreets/goconvey/convey"
@@ -32,13 +33,13 @@ func TestGetLocalLog(t *testing.T) {
 		fmt.Println(err.Error())
 
 	} else {
-		_, err = f.Write([]byte("This is for test"))
+		_, err = f.WriteString("This is for test")
 	}
 
 	Convey("Test GetLocalLog", t, func() {
 		Convey("Test response", func() {
 			logStr, err := GetLocalLog(logPath)
-			log.Info(string(logStr))
+			log.Info(utils.BytesToString(logStr))
 			fmt.Println(err)
 			So(err, ShouldEqual, nil)
 

backend/services/node.go

@@ -7,6 +7,7 @@ import (
 	"crawlab/lib/cron"
 	"crawlab/model"
 	"crawlab/services/register"
+	"crawlab/utils"
 	"encoding/json"
 	"fmt"
 	"github.com/apex/log"
@@ -254,7 +255,7 @@ func UpdateNodeData() {
 		debug.PrintStack()
 		return
 	}
-	if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil {
+	if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil {
 		log.Errorf(err.Error())
 		return
 	}
@@ -280,7 +281,7 @@ func MasterNodeCallback(message redis.Message) (err error) {
 		time.Sleep(10 * time.Millisecond)
 		ch := SystemInfoChanMap.ChanBlocked(msg.NodeId)
 		sysInfoBytes, _ := json.Marshal(&msg.SysInfo)
-		ch <- string(sysInfoBytes)
+		ch <- utils.BytesToString(sysInfoBytes)
 	}
 	return nil
 }
@@ -304,14 +305,14 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 
 		// 获取本地日志
 		logStr, err := GetLocalLog(msg.LogPath)
-		log.Info(string(logStr))
+		log.Info(utils.BytesToString(logStr))
 		if err != nil {
 			log.Errorf(err.Error())
 			debug.PrintStack()
 			msgSd.Error = err.Error()
 			msgSd.Log = err.Error()
 		} else {
-			msgSd.Log = string(logStr)
+			msgSd.Log = utils.BytesToString(logStr)
 		}
 
 		// 序列化
@@ -322,7 +323,7 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 
 		// 发布消息给主节点
 		log.Info("publish get log msg to master")
-		if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil {
 
 			return err
 		}
@@ -347,7 +348,7 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 			debug.PrintStack()
 			return err
 		}
-		if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil {
 			log.Errorf(err.Error())
 			return err
 		}

backend/services/spider.go

@@ -307,7 +307,7 @@ func PublishSpider(spider model.Spider) (err error) {
 		return
 	}
 	channel := "files:upload"
-	if _, err = database.RedisClient.Publish(channel, string(msgStr)); err != nil {
+	if _, err = database.RedisClient.Publish(channel, utils.BytesToString(msgStr)); err != nil {
 		log.Errorf(err.Error())
 		debug.PrintStack()
 		return

backend/services/system.go

@@ -112,7 +112,7 @@ func GetRemoteSystemInfo(id string) (sysInfo model.SystemInfo, err error) {
 
 	// 序列化
 	msgBytes, _ := json.Marshal(&msg)
-	if _, err := database.RedisClient.Publish("nodes:"+id, string(msgBytes)); err != nil {
+	if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil {
 		return model.SystemInfo{}, err
 	}
 

backend/services/task.go

@@ -36,7 +36,7 @@ func (m *TaskMessage) ToString() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return string(data), err
+	return utils.BytesToString(data), err
 }
 
 // 任务执行器
@@ -405,13 +405,13 @@ func GetTaskLog(id string) (logStr string, err error) {
 	if IsMasterNode(task.NodeId.Hex()) {
 		// 若为主节点，获取本机日志
 		logBytes, err := GetLocalLog(task.LogPath)
-		logStr = string(logBytes)
+		logStr = utils.BytesToString(logBytes)
 		if err != nil {
 			log.Errorf(err.Error())
-			logStr = string(err.Error())
+			logStr = err.Error()
 			// return "", err
 		} else {
-			logStr = string(logBytes)
+			logStr = utils.BytesToString(logBytes)
 		}
 
 	} else {
@@ -466,7 +466,7 @@ func CancelTask(id string) (err error) {
 		}
 
 		// 发布消息
-		if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), string(msgBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), utils.BytesToString(msgBytes)); err != nil {
 			return err
 		}
 	}

backend/utils/helpers.go

@@ -0,0 +1,7 @@
+package utils
+
+import "unsafe"
+
+func BytesToString(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}