Merge remote-tracking branch 'forked/develop' into forked_master

backend/go.sum

@@ -57,6 +57,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -70,6 +71,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -126,8 +128,10 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
+github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=

backend/mock/node_test.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/gin-gonic/gin"
@@ -8,12 +9,12 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
 
 var app *gin.Engine
+
 // 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务
 func init() {
 	app = gin.Default()
@@ -28,18 +29,18 @@ func init() {
 	app.GET("/nodes/:id/system", GetSystemInfo)  // 节点任务列表
 	app.DELETE("/nodes/:id", DeleteNode)         // 删除节点
 	//// 爬虫
-	app.GET("/stats/home",GetHomeStats) // 首页统计数据
+	app.GET("/stats/home", GetHomeStats) // 首页统计数据
 	// 定时任务
-	app.GET("/schedules", GetScheduleList)       // 定时任务列表
-	app.GET("/schedules/:id", GetSchedule)       // 定时任务详情
-	app.PUT("/schedules", PutSchedule)           // 创建定时任务
-	app.POST("/schedules/:id", PostSchedule)     // 修改定时任务
-	app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务
+	app.GET("/schedules", GetScheduleList)                         // 定时任务列表
+	app.GET("/schedules/:id", GetSchedule)                         // 定时任务详情
+	app.PUT("/schedules", PutSchedule)                             // 创建定时任务
+	app.POST("/schedules/:id", PostSchedule)                       // 修改定时任务
+	app.DELETE("/schedules/:id", DeleteSchedule)                   // 删除定时任务
 	app.GET("/tasks", GetTaskList)                                 // 任务列表
 	app.GET("/tasks/:id", GetTask)                                 // 任务详情
 	app.PUT("/tasks", PutTask)                                     // 派发任务
 	app.DELETE("/tasks/:id", DeleteTask)                           // 删除任务
-	app.GET("/tasks/:id/results",GetTaskResults)                  // 任务结果
+	app.GET("/tasks/:id/results", GetTaskResults)                  // 任务结果
 	app.GET("/tasks/:id/results/download", DownloadTaskResultsCsv) // 下载任务结果
 	app.GET("/spiders", GetSpiderList)              // 爬虫列表
 	app.GET("/spiders/:id", GetSpider)              // 爬虫详情
@@ -55,7 +56,7 @@ func TestGetNodeList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/nodes", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -74,7 +75,7 @@ func TestGetNode(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -93,7 +94,7 @@ func TestPing(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/ping", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -111,7 +112,7 @@ func TestGetNodeTaskList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -130,7 +131,7 @@ func TestDeleteNode(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -162,10 +163,10 @@ func TestPostNode(t *testing.T) {
 
 	var mongoId = "5d429e6c19f7abede924fee2"
 	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "nodes/"+mongoId, bytes.NewReader(body))
 
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
@@ -184,7 +185,7 @@ func TestGetSystemInfo(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}

backend/mock/schedule_test.go

@@ -1,7 +1,9 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
+	"crawlab/utils"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
@@ -17,7 +19,7 @@ func TestGetScheduleList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/schedules", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -36,7 +38,7 @@ func TestGetSchedule(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -56,7 +58,7 @@ func TestDeleteSchedule(t *testing.T) {
 	req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil)
 	app.ServeHTTP(w, req)
 
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -86,11 +88,12 @@ func TestPostSchedule(t *testing.T) {
 
 	var resp Response
 	var mongoId = "5d429e6c19f7abede924fee2"
-	body,_ := json.Marshal(newItem)
+	body, _ := json.Marshal(newItem)
 	w := httptest.NewRecorder()
-	req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "/schedules/"+mongoId, strings.NewReader(utils.BytesToString(body)))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
@@ -121,11 +124,11 @@ func TestPutSchedule(t *testing.T) {
 	}
 
 	var resp Response
-	body,_ := json.Marshal(newItem)
+	body, _ := json.Marshal(newItem)
 	w := httptest.NewRecorder()
-	req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body)))
+	req, _ := http.NewRequest("PUT", "/schedules", bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()),&resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	t.Log(resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")

backend/mock/spider_test.go

@@ -1,13 +1,13 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
@@ -17,7 +17,7 @@ func TestGetSpiderList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp faild")
 	}
@@ -35,7 +35,7 @@ func TestGetSpider(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -66,9 +66,9 @@ func TestPostSpider(t *testing.T) {
 	var spiderId = "5d429e6c19f7abede924fee2"
 	w := httptest.NewRecorder()
 	body, _ := json.Marshal(spider)
-	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, strings.NewReader(string(body)))
+	req, _ := http.NewRequest("POST", "/spiders/"+spiderId, bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -87,7 +87,7 @@ func TestGetSpiderDir(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/dir", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -106,7 +106,7 @@ func TestGetSpiderTasks(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/spiders/"+spiderId+"/tasks", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -124,7 +124,7 @@ func TestDeleteSpider(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "/spiders/"+spiderId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -134,4 +134,4 @@ func TestDeleteSpider(t *testing.T) {
 			So(resp.Message, ShouldEqual, "success")
 		})
 	})
-}
+}

backend/mock/stats_test.go

@@ -14,7 +14,7 @@ func TestGetHomeStats(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/stats/home", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	fmt.Println(resp.Data)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
@@ -26,4 +26,4 @@ func TestGetHomeStats(t *testing.T) {
 			So(resp.Message, ShouldEqual, "success")
 		})
 	})
-}
+}

backend/mock/task.go

@@ -186,7 +186,7 @@ func DownloadTaskResultsCsv(c *gin.Context) {
 	bytesBuffer := &bytes.Buffer{}
 
 	// 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码
-	bytesBuffer.Write([]byte("\xEF\xBB\xBF"))
+	bytesBuffer.WriteString("\xEF\xBB\xBF")
 
 	writer := csv.NewWriter(bytesBuffer)
 

backend/mock/task_test.go

@@ -1,13 +1,13 @@
 package mock
 
 import (
+	"bytes"
 	"crawlab/model"
 	"encoding/json"
 	"github.com/globalsign/mgo/bson"
 	. "github.com/smartystreets/goconvey/convey"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 	"time"
 )
@@ -24,7 +24,7 @@ func TestGetTaskList(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks?PageNum=2&PageSize=10&NodeId=342dfsff&SpiderId=f8dsf", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -44,7 +44,7 @@ func TestGetTask(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks/"+taskId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}
@@ -80,9 +80,9 @@ func TestPutTask(t *testing.T) {
 	var resp Response
 	body, _ := json.Marshal(&newItem)
 	w := httptest.NewRecorder()
-	req, _ := http.NewRequest("PUT", "/tasks", strings.NewReader(string(body)))
+	req, _ := http.NewRequest("PUT", "/tasks", bytes.NewReader(body))
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -100,7 +100,7 @@ func TestDeleteTask(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("DELETE", "/tasks/"+taskId, nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("unmarshal resp failed")
 	}
@@ -123,7 +123,7 @@ func TestGetTaskResults(t *testing.T) {
 	w := httptest.NewRecorder()
 	req, _ := http.NewRequest("GET", "/tasks/"+taskId+"/results?PageNum=2&PageSize=1", nil)
 	app.ServeHTTP(w, req)
-	err := json.Unmarshal([]byte(w.Body.String()), &resp)
+	err := json.Unmarshal(w.Body.Bytes(), &resp)
 	if err != nil {
 		t.Fatal("Unmarshal resp failed")
 	}

backend/model/schedule.go

@@ -18,6 +18,7 @@ type Schedule struct {
 	NodeId      bson.ObjectId `json:"node_id" bson:"node_id"`
 	Cron        string        `json:"cron" bson:"cron"`
 	EntryId     cron.EntryID  `json:"entry_id" bson:"entry_id"`
+	Param       string        `json:"param" bson:"param"`
 
 	// 前端展示
 	SpiderName string `json:"spider_name" bson:"spider_name"`

backend/model/task.go

@@ -19,6 +19,7 @@ type Task struct {
 	NodeId          bson.ObjectId `json:"node_id" bson:"node_id"`
 	LogPath         string        `json:"log_path" bson:"log_path"`
 	Cmd             string        `json:"cmd" bson:"cmd"`
+	Param           string        `json:"param" bson:"param"`
 	Error           string        `json:"error" bson:"error"`
 	ResultCount     int           `json:"result_count" bson:"result_count"`
 	WaitDuration    float64       `json:"wait_duration" bson:"wait_duration"`

backend/routes/file.go

@@ -1,6 +1,7 @@
 package routes
 
 import (
+	"crawlab/utils"
 	"github.com/gin-gonic/gin"
 	"io/ioutil"
 	"net/http"
@@ -15,6 +16,6 @@ func GetFile(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    string(fileBytes),
+		Data:    utils.BytesToString(fileBytes),
 	})
 }

backend/routes/spider.go

@@ -327,7 +327,7 @@ func GetSpiderFile(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{
 		Status:  "ok",
 		Message: "success",
-		Data:    string(fileBytes),
+		Data:    utils.BytesToString(fileBytes),
 	})
 }
 

backend/routes/task.go

@@ -215,7 +215,7 @@ func DownloadTaskResultsCsv(c *gin.Context) {
 	bytesBuffer := &bytes.Buffer{}
 
 	// 写入UTF-8 BOM，避免使用Microsoft Excel打开乱码
-	bytesBuffer.Write([]byte("\xEF\xBB\xBF"))
+	bytesBuffer.WriteString("\xEF\xBB\xBF")
 
 	writer := csv.NewWriter(bytesBuffer)
 

backend/services/log.go

@@ -71,7 +71,7 @@ func GetRemoteLog(task model.Task) (logStr string, err error) {
 
 	// 发布获取日志消息
 	channel := "nodes:" + task.NodeId.Hex()
-	if _, err := database.RedisClient.Publish(channel, string(msgBytes)); err != nil {
+	if _, err := database.RedisClient.Publish(channel, utils.BytesToString(msgBytes)); err != nil {
 		log.Errorf(err.Error())
 		return "", err
 	}

backend/services/log_test.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"crawlab/config"
+	"crawlab/utils"
 	"fmt"
 	"github.com/apex/log"
 	. "github.com/smartystreets/goconvey/convey"
@@ -32,13 +33,13 @@ func TestGetLocalLog(t *testing.T) {
 		fmt.Println(err.Error())
 
 	} else {
-		_, err = f.Write([]byte("This is for test"))
+		_, err = f.WriteString("This is for test")
 	}
 
 	Convey("Test GetLocalLog", t, func() {
 		Convey("Test response", func() {
 			logStr, err := GetLocalLog(logPath)
-			log.Info(string(logStr))
+			log.Info(utils.BytesToString(logStr))
 			fmt.Println(err)
 			So(err, ShouldEqual, nil)
 

backend/services/node.go

@@ -7,6 +7,7 @@ import (
 	"crawlab/lib/cron"
 	"crawlab/model"
 	"crawlab/services/register"
+	"crawlab/utils"
 	"encoding/json"
 	"fmt"
 	"github.com/apex/log"
@@ -254,7 +255,7 @@ func UpdateNodeData() {
 		debug.PrintStack()
 		return
 	}
-	if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil {
+	if err := database.RedisClient.HSet("nodes", key, utils.BytesToString(dataBytes)); err != nil {
 		log.Errorf(err.Error())
 		return
 	}
@@ -280,7 +281,7 @@ func MasterNodeCallback(message redis.Message) (err error) {
 		time.Sleep(10 * time.Millisecond)
 		ch := SystemInfoChanMap.ChanBlocked(msg.NodeId)
 		sysInfoBytes, _ := json.Marshal(&msg.SysInfo)
-		ch <- string(sysInfoBytes)
+		ch <- utils.BytesToString(sysInfoBytes)
 	}
 	return nil
 }
@@ -304,14 +305,14 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 
 		// 获取本地日志
 		logStr, err := GetLocalLog(msg.LogPath)
-		log.Info(string(logStr))
+		log.Info(utils.BytesToString(logStr))
 		if err != nil {
 			log.Errorf(err.Error())
 			debug.PrintStack()
 			msgSd.Error = err.Error()
 			msgSd.Log = err.Error()
 		} else {
-			msgSd.Log = string(logStr)
+			msgSd.Log = utils.BytesToString(logStr)
 		}
 
 		// 序列化
@@ -322,7 +323,7 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 
 		// 发布消息给主节点
 		log.Info("publish get log msg to master")
-		if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil {
 
 			return err
 		}
@@ -347,7 +348,7 @@ func WorkerNodeCallback(message redis.Message) (err error) {
 			debug.PrintStack()
 			return err
 		}
-		if _, err := database.RedisClient.Publish("nodes:master", string(msgSdBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:master", utils.BytesToString(msgSdBytes)); err != nil {
 			log.Errorf(err.Error())
 			return err
 		}

backend/services/schedule.go

@@ -28,6 +28,7 @@ func AddTask(s model.Schedule) func() {
 			SpiderId: s.SpiderId,
 			NodeId:   nodeId,
 			Status:   constants.StatusPending,
+			Param:    s.Param,
 		}
 
 		// 将任务存入数据库

backend/services/spider.go

@@ -307,7 +307,7 @@ func PublishSpider(spider model.Spider) (err error) {
 		return
 	}
 	channel := "files:upload"
-	if _, err = database.RedisClient.Publish(channel, string(msgStr)); err != nil {
+	if _, err = database.RedisClient.Publish(channel, utils.BytesToString(msgStr)); err != nil {
 		log.Errorf(err.Error())
 		debug.PrintStack()
 		return

backend/services/system.go

@@ -112,7 +112,7 @@ func GetRemoteSystemInfo(id string) (sysInfo model.SystemInfo, err error) {
 
 	// 序列化
 	msgBytes, _ := json.Marshal(&msg)
-	if _, err := database.RedisClient.Publish("nodes:"+id, string(msgBytes)); err != nil {
+	if _, err := database.RedisClient.Publish("nodes:"+id, utils.BytesToString(msgBytes)); err != nil {
 		return model.SystemInfo{}, err
 	}
 

backend/services/task.go

@@ -16,14 +16,15 @@ import (
 	"runtime"
 	"runtime/debug"
 	"strconv"
+	"sync"
 	"time"
 )
 
 var Exec *Executor
 
 // 任务执行锁
-var LockList []bool
-
+//Added by cloud: 2019/09/04,solve data race
+var LockList sync.Map
 // 任务消息
 type TaskMessage struct {
 	Id  string
@@ -36,7 +37,7 @@ func (m *TaskMessage) ToString() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	return string(data), err
+	return utils.BytesToString(data), err
 }
 
 // 任务执行器
@@ -56,7 +57,7 @@ func (ex *Executor) Start() error {
 		id := i
 
 		// 初始化任务锁
-		LockList = append(LockList, false)
+		LockList.Store(id, false)
 
 		// 加入定时任务
 		_, err := ex.Cron.AddFunc(spec, GetExecuteTaskFunc(id))
@@ -220,17 +221,18 @@ func SaveTaskResultCount(id string) func() {
 
 // 执行任务
 func ExecuteTask(id int) {
-	if LockList[id] {
+	if flag, _ := LockList.Load(id); flag.(bool) {
 		log.Debugf(GetWorkerPrefix(id) + "正在执行任务...")
 		return
 	}
 
 	// 上锁
-	LockList[id] = true
+	LockList.Store(id, true)
 
 	// 解锁（延迟执行）
 	defer func() {
-		LockList[id] = false
+		LockList.Delete(id)
+		LockList.Store(id, false)
 	}()
 
 	// 开始计时
@@ -323,8 +325,10 @@ func ExecuteTask(id int) {
 
 	// 执行命令
 	cmd := spider.Cmd
-	if t.Cmd != "" {
-		cmd = t.Cmd
+
+	// 加入参数
+	if t.Param != "" {
+		cmd += " " + t.Param
 	}
 
 	// 任务赋值
@@ -405,13 +409,13 @@ func GetTaskLog(id string) (logStr string, err error) {
 	if IsMasterNode(task.NodeId.Hex()) {
 		// 若为主节点，获取本机日志
 		logBytes, err := GetLocalLog(task.LogPath)
-		logStr = string(logBytes)
+		logStr = utils.BytesToString(logBytes)
 		if err != nil {
 			log.Errorf(err.Error())
-			logStr = string(err.Error())
+			logStr = err.Error()
 			// return "", err
 		} else {
-			logStr = string(logBytes)
+			logStr = utils.BytesToString(logBytes)
 		}
 
 	} else {
@@ -466,7 +470,7 @@ func CancelTask(id string) (err error) {
 		}
 
 		// 发布消息
-		if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), string(msgBytes)); err != nil {
+		if _, err := database.RedisClient.Publish("nodes:"+task.NodeId.Hex(), utils.BytesToString(msgBytes)); err != nil {
 			return err
 		}
 	}

backend/utils/helpers.go

@@ -0,0 +1,7 @@
+package utils
+
+import "unsafe"
+
+func BytesToString(b []byte) string {
+	return *(*string)(unsafe.Pointer(&b))
+}

backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go

@@ -1,7 +0,0 @@
-// Utility package for extracting JWT tokens from
-// HTTP requests.
-//
-// The main function is ParseFromRequest and it's WithClaims variant.
-// See examples for how to use the various Extractor implementations
-// or roll your own.
-package request

backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go

@@ -1,81 +0,0 @@
-package request
-
-import (
-	"errors"
-	"net/http"
-)
-
-// Errors
-var (
-	ErrNoTokenInRequest = errors.New("no token present in request")
-)
-
-// Interface for extracting a token from an HTTP request.
-// The ExtractToken method should return a token string or an error.
-// If no token is present, you must return ErrNoTokenInRequest.
-type Extractor interface {
-	ExtractToken(*http.Request) (string, error)
-}
-
-// Extractor for finding a token in a header.  Looks at each specified
-// header in order until there's a match
-type HeaderExtractor []string
-
-func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) {
-	// loop over header names and return the first one that contains data
-	for _, header := range e {
-		if ah := req.Header.Get(header); ah != "" {
-			return ah, nil
-		}
-	}
-	return "", ErrNoTokenInRequest
-}
-
-// Extract token from request arguments.  This includes a POSTed form or
-// GET URL arguments.  Argument names are tried in order until there's a match.
-// This extractor calls `ParseMultipartForm` on the request
-type ArgumentExtractor []string
-
-func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) {
-	// Make sure form is parsed
-	req.ParseMultipartForm(10e6)
-
-	// loop over arg names and return the first one that contains data
-	for _, arg := range e {
-		if ah := req.Form.Get(arg); ah != "" {
-			return ah, nil
-		}
-	}
-
-	return "", ErrNoTokenInRequest
-}
-
-// Tries Extractors in order until one returns a token string or an error occurs
-type MultiExtractor []Extractor
-
-func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) {
-	// loop over header names and return the first one that contains data
-	for _, extractor := range e {
-		if tok, err := extractor.ExtractToken(req); tok != "" {
-			return tok, nil
-		} else if err != ErrNoTokenInRequest {
-			return "", err
-		}
-	}
-	return "", ErrNoTokenInRequest
-}
-
-// Wrap an Extractor in this to post-process the value before it's handed off.
-// See AuthorizationHeaderExtractor for an example
-type PostExtractionFilter struct {
-	Extractor
-	Filter func(string) (string, error)
-}
-
-func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) {
-	if tok, err := e.Extractor.ExtractToken(req); tok != "" {
-		return e.Filter(tok)
-	} else {
-		return "", err
-	}
-}

backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go

@@ -1,28 +0,0 @@
-package request
-
-import (
-	"strings"
-)
-
-// Strips 'Bearer ' prefix from bearer token string
-func stripBearerPrefixFromTokenString(tok string) (string, error) {
-	// Should be a bearer token
-	if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " {
-		return tok[7:], nil
-	}
-	return tok, nil
-}
-
-// Extract bearer token from Authorization header
-// Uses PostExtractionFilter to strip "Bearer " prefix from header
-var AuthorizationHeaderExtractor = &PostExtractionFilter{
-	HeaderExtractor{"Authorization"},
-	stripBearerPrefixFromTokenString,
-}
-
-// Extractor for OAuth2 access tokens.  Looks in 'Authorization'
-// header then 'access_token' argument for a token.
-var OAuth2Extractor = &MultiExtractor{
-	AuthorizationHeaderExtractor,
-	ArgumentExtractor{"access_token"},
-}

backend/vendor/github.com/dgrijalva/jwt-go/request/request.go

@@ -1,68 +0,0 @@
-package request
-
-import (
-	"github.com/dgrijalva/jwt-go"
-	"net/http"
-)
-
-// Extract and parse a JWT token from an HTTP request.
-// This behaves the same as Parse, but accepts a request and an extractor
-// instead of a token string.  The Extractor interface allows you to define
-// the logic for extracting a token.  Several useful implementations are provided.
-//
-// You can provide options to modify parsing behavior
-func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) {
-	// Create basic parser struct
-	p := &fromRequestParser{req, extractor, nil, nil}
-
-	// Handle options
-	for _, option := range options {
-		option(p)
-	}
-
-	// Set defaults
-	if p.claims == nil {
-		p.claims = jwt.MapClaims{}
-	}
-	if p.parser == nil {
-		p.parser = &jwt.Parser{}
-	}
-
-	// perform extract
-	tokenString, err := p.extractor.ExtractToken(req)
-	if err != nil {
-		return nil, err
-	}
-
-	// perform parse
-	return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc)
-}
-
-// ParseFromRequest but with custom Claims type
-// DEPRECATED: use ParseFromRequest and the WithClaims option
-func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) {
-	return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims))
-}
-
-type fromRequestParser struct {
-	req       *http.Request
-	extractor Extractor
-	claims    jwt.Claims
-	parser    *jwt.Parser
-}
-
-type ParseFromRequestOption func(*fromRequestParser)
-
-// Parse with custom claims
-func WithClaims(claims jwt.Claims) ParseFromRequestOption {
-	return func(p *fromRequestParser) {
-		p.claims = claims
-	}
-}
-
-// Parse using a custom parser
-func WithParser(parser *jwt.Parser) ParseFromRequestOption {
-	return func(p *fromRequestParser) {
-		p.parser = parser
-	}
-}

backend/vendor/modules.txt

@@ -2,7 +2,6 @@
 github.com/apex/log
 # github.com/dgrijalva/jwt-go v3.2.0+incompatible
 github.com/dgrijalva/jwt-go
-github.com/dgrijalva/jwt-go/request
 # github.com/fsnotify/fsnotify v1.4.7
 github.com/fsnotify/fsnotify
 # github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3
@@ -18,11 +17,18 @@ github.com/globalsign/mgo/bson
 github.com/globalsign/mgo/internal/sasl
 github.com/globalsign/mgo/internal/scram
 github.com/globalsign/mgo/internal/json
+# github.com/go-playground/locales v0.12.1
+github.com/go-playground/locales
+github.com/go-playground/locales/currency
+# github.com/go-playground/universal-translator v0.16.0
+github.com/go-playground/universal-translator
 # github.com/golang/protobuf v1.3.1
 github.com/golang/protobuf/proto
 # github.com/gomodule/redigo v2.0.0+incompatible
 github.com/gomodule/redigo/redis
 github.com/gomodule/redigo/internal
+# github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1
+github.com/gopherjs/gopherjs/js
 # github.com/hashicorp/hcl v1.0.0
 github.com/hashicorp/hcl
 github.com/hashicorp/hcl/hcl/printer
@@ -36,6 +42,10 @@ github.com/hashicorp/hcl/json/scanner
 github.com/hashicorp/hcl/json/token
 # github.com/json-iterator/go v1.1.6
 github.com/json-iterator/go
+# github.com/jtolds/gls v4.20.0+incompatible
+github.com/jtolds/gls
+# github.com/leodido/go-urn v1.1.0
+github.com/leodido/go-urn
 # github.com/magiconair/properties v1.8.0
 github.com/magiconair/properties
 # github.com/mattn/go-isatty v0.0.8
@@ -52,6 +62,15 @@ github.com/pelletier/go-toml
 github.com/pkg/errors
 # github.com/satori/go.uuid v1.2.0
 github.com/satori/go.uuid
+# github.com/smartystreets/assertions v1.0.0
+github.com/smartystreets/assertions
+github.com/smartystreets/assertions/internal/go-diff/diffmatchpatch
+github.com/smartystreets/assertions/internal/go-render/render
+github.com/smartystreets/assertions/internal/oglematchers
+# github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337
+github.com/smartystreets/goconvey/convey
+github.com/smartystreets/goconvey/convey/reporting
+github.com/smartystreets/goconvey/convey/gotest
 # github.com/spf13/afero v1.1.2
 github.com/spf13/afero
 github.com/spf13/afero/mem
@@ -72,5 +91,7 @@ golang.org/x/text/transform
 golang.org/x/text/unicode/norm
 # gopkg.in/go-playground/validator.v8 v8.18.2
 gopkg.in/go-playground/validator.v8
+# gopkg.in/go-playground/validator.v9 v9.29.1
+gopkg.in/go-playground/validator.v9
 # gopkg.in/yaml.v2 v2.2.2
 gopkg.in/yaml.v2

frontend/src/components/Common/CrawlConfirmDialog.vue

@@ -19,6 +19,9 @@
           />
         </el-select>
       </el-form-item>
+      <el-form-item :label="$t('Parameters')">
+        <el-input v-model="param" :placeholder="$t('Parameters')"></el-input>
+      </el-form-item>
     </el-form>
     <template slot="footer">
       <el-button type="plain" size="small" @click="$emit('close')">{{$t('Cancel')}}</el-button>
@@ -42,7 +45,8 @@ export default {
   },
   data () {
     return {
-      nodeId: ''
+      nodeId: '',
+      param: ''
     }
   },
   methods: {
@@ -50,7 +54,7 @@ export default {
       this.$emit('close')
     },
     onConfirm () {
-      this.$store.dispatch('spider/crawlSpider', { id: this.spiderId, nodeId: this.nodeId })
+      this.$store.dispatch('spider/crawlSpider', { id: this.spiderId, nodeId: this.nodeId, param: this.param })
         .then(() => {
           this.$message.success(this.$t('A task has been scheduled successfully'))
         })

frontend/src/components/InfoView/TaskInfoView.vue

@@ -15,6 +15,9 @@
         <el-form-item :label="$t('Log File Path')">
           <el-input v-model="taskForm.log_path" placeholder="Log File Path" disabled></el-input>
         </el-form-item>
+        <el-form-item :label="$t('Parameters')">
+          <el-input v-model="taskForm.param" placeholder="Parameters" disabled></el-input>
+        </el-form-item>
         <el-form-item :label="$t('Create Time')">
           <el-input :value="getTime(taskForm.create_ts)" placeholder="Create Time" disabled></el-input>
         </el-form-item>

frontend/src/i18n/zh.js

@@ -247,6 +247,7 @@ export default {
   'username already exists': '用户名已存在',
   'Deleted successfully': '成功删除',
   'Saved successfully': '成功保存',
+  'Please zip your spider files from the root directory': '爬虫文件请从根目录下开始压缩。',
   'English': 'English',
   // 登录
   'Sign in': '登录',

frontend/src/store/modules/spider.js

@@ -101,10 +101,11 @@ const actions = {
       })
   },
   crawlSpider ({ state, dispatch }, payload) {
-    const { id, nodeId } = payload
+    const { id, nodeId, param } = payload
     return request.put(`/tasks`, {
       spider_id: id,
-      node_id: nodeId
+      node_id: nodeId,
+      param: param
     })
   },
   getTaskList ({ state, commit }, id) {

frontend/src/views/schedule/ScheduleList.vue

@@ -60,8 +60,8 @@
                     disabled>
           </el-input>
         </el-form-item>
-        <el-form-item :label="$t('Parameters')" prop="params">
-          <el-input v-model="scheduleForm.params"
+        <el-form-item :label="$t('Parameters')" prop="param">
+          <el-input v-model="scheduleForm.param"
                     :placeholder="$t('Parameters')"></el-input>
         </el-form-item>
         <el-form-item :label="$t('Schedule Description')" prop="description">
@@ -111,7 +111,7 @@
             </template>
           </el-table-column>
         </template>
-        <el-table-column :label="$t('Action')" align="left" width="250" fixed="right">
+        <el-table-column :label="$t('Action')" align="left" width="150px" fixed="right">
           <template slot-scope="scope">
             <el-tooltip :content="$t('Edit')" placement="top">
               <el-button type="warning" icon="el-icon-edit" size="mini" @click="onEdit(scope.row)"></el-button>
@@ -159,6 +159,7 @@ export default {
         { name: 'cron', label: 'schedules.cron', width: '120' },
         { name: 'node_name', label: 'Node', width: '150' },
         { name: 'spider_name', label: 'Spider', width: '150' },
+        { name: 'param', label: 'Parameters', width: '150' },
         { name: 'description', label: 'Description', width: 'auto' }
       ],
       isEdit: false,

frontend/src/views/spider/SpiderList.vue

@@ -93,7 +93,7 @@
           </el-upload>
         </el-form-item>
       </el-form>
-      <el-alert type="error" title="爬虫文件请从根目录下开始压缩。" :closable="false"></el-alert>
+      <el-alert type="error" :title="$t('Please zip your spider files from the root directory')" :closable="false"></el-alert>
     </el-dialog>
     <!--./customized spider dialog-->
 
@@ -207,7 +207,7 @@
                            :width="col.width">
           </el-table-column>
         </template>
-        <el-table-column :label="$t('Action')" align="left" width="auto" fixed="right">
+        <el-table-column :label="$t('Action')" align="left" width="150px" fixed="right">
           <template slot-scope="scope">
             <el-tooltip :content="$t('View')" placement="top">
               <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>

frontend/src/views/task/TaskList.vue

@@ -119,7 +119,7 @@
                            :width="col.width">
           </el-table-column>
         </template>
-        <el-table-column :label="$t('Action')" align="left" fixed="right">
+        <el-table-column :label="$t('Action')" align="left" fixed="right" width="150px">
           <template slot-scope="scope">
             <el-tooltip :content="$t('View')" placement="top">
               <el-button type="primary" icon="el-icon-search" size="mini" @click="onView(scope.row)"></el-button>
@@ -172,6 +172,7 @@ export default {
         { name: 'node_name', label: 'Node', width: '120' },
         { name: 'spider_name', label: 'Spider', width: '120' },
         { name: 'status', label: 'Status', width: '120' },
+        { name: 'param', label: 'Parameters', width: '120' },
         // { name: 'create_ts', label: 'Create Time', width: '100' },
         { name: 'start_ts', label: 'Start Time', width: '100' },
         { name: 'finish_ts', label: 'Finish Time', width: '100' },