git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:tikazyq/crawlab into develop

Browse Source
This commit is contained in:
Marvin Zhang
2019-08-22 19:28:26 +08:00
parent 54fa4ac81e 00a61ed7dc
commit 67959f40a0
24 changed files with 1144 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -119,4 +119,6 @@ logs/
tmp/
_book/
.idea
*.lock
*.lock
backend/spiders

55
backend/database/mongo_test.go
View File

@@ -1,19 +1,70 @@
package database
import (
"crawlab/config"
"github.com/apex/log"
"github.com/globalsign/mgo"
. "github.com/smartystreets/goconvey/convey"
"github.com/spf13/viper"
"reflect"
"testing"
)
func init() {
if err := config.InitConfig("../conf/config.yml"); err != nil {
log.Fatal("Init config failed")
}
log.Infof("初始化配置成功")
err := InitMongo()
if err != nil {
log.Fatal("Init mongodb failed")
}
}
func TestGetDb(t *testing.T) {
Convey("Test GetDb", t, func() {
if err := config.InitConfig("../conf/config.yml"); err != nil {
t.Fatal("Init config failed")
}
t.Log("初始化配置成功")
err := InitMongo()
if err != nil {
t.Fatal("Init mongodb failed")
}
s, db := GetDb()
Convey("The value should be Session.Copy", func() {
So(s, ShouldEqual, Session.Copy())
So(s, ShouldResemble, Session.Copy())
})
Convey("The value should be reference of database", func() {
So(db, ShouldEqual, s.DB(viper.GetString("mongo.db")))
So(db, ShouldResemble, s.DB(viper.GetString("mongo.db")))
})
})
}
func TestGetCol(t *testing.T) {
var c = "nodes"
var colActual *mgo.Collection
Convey("Test GetCol", t, func() {
s, col := GetCol(c)
Convey("s should resemble Session.Copy", func() {
So(s, ShouldResemble, Session.Copy())
So(reflect.TypeOf(col), ShouldResemble, reflect.TypeOf(colActual))
})
})
}
func TestGetGridFs(t *testing.T) {
var prefix = "files"
var gfActual *mgo.GridFS
Convey("Test GetGridFs", t, func() {
s, gf := GetGridFs(prefix)
Convey("s should be session.copy", func() {
So(s, ShouldResemble, Session.Copy())
})
Convey("gf should be *mgo.GridFS", func() {
So(reflect.TypeOf(gf), ShouldResemble, reflect.TypeOf(gfActual))
})
})
}

5
backend/go.sum
View File

@@ -53,6 +53,7 @@ github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -65,6 +66,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -117,8 +119,10 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -184,6 +188,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

16
backend/mock/base.go Normal file
View File

@@ -0,0 +1,16 @@
package mock
type Response struct {
Status string `json:"status"`
Message string `json:"message"`
Data interface{} `json:"data"`
Error string `json:"error"`
}
type ListResponse struct {
Status string `json:"status"`
Message string `json:"message"`
Total int `json:"total"`
Data interface{} `json:"data"`
Error string `json:"error"`
}

8
backend/mock/file.go Normal file
View File

@@ -0,0 +1,8 @@
package mock
type File struct {
Name string `json:"name"`
Path string `json:"path"`
IsDir bool `json:"is_dir"`
Size int64 `json:"size"`
}

210
backend/mock/node.go Normal file
View File

@@ -0,0 +1,210 @@
package mock
import (
"crawlab/model"
"crawlab/services"
"github.com/apex/log"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
"time"
)
var NodeList = []model.Node{
{
Id: bson.ObjectId("5d429e6c19f7abede924fee2"),
Ip: "10.32.35.15",
Name: "test1",
Status: "online",
Port: "8081",
Mac: "ac:12:df:12:fd",
Description: "For test1",
IsMaster: true,
UpdateTs: time.Now(),
CreateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
},
{
Id: bson.ObjectId("5d429e6c19f7abede924fe22"),
Ip: "10.32.35.12",
Name: "test2",
Status: "online",
Port: "8082",
Mac: "ac:12:df:12:vh",
Description: "For test2",
IsMaster: true,
UpdateTs: time.Now(),
CreateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
},
}
var TaskList = []model.Task{
{
Id: "1234",
SpiderId: bson.ObjectId("xx429e6c19f7abede924fee2"),
StartTs: time.Now(),
FinishTs: time.Now(),
Status: "进行中",
NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"),
LogPath: "./log",
Cmd: "scrapy crawl test",
Error: "",
ResultCount: 0,
WaitDuration: 10.0,
RuntimeDuration: 10,
TotalDuration: 20,
SpiderName: "test",
NodeName: "test",
CreateTs: time.Now(),
UpdateTs: time.Now(),
},
{
Id: "5678",
SpiderId: bson.ObjectId("xx429e6c19f7abede924fddf"),
StartTs: time.Now(),
FinishTs: time.Now(),
Status: "进行中",
NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"),
LogPath: "./log",
Cmd: "scrapy crawl test2",
Error: "",
ResultCount: 0,
WaitDuration: 10.0,
RuntimeDuration: 10,
TotalDuration: 20,
SpiderName: "test",
NodeName: "test",
CreateTs: time.Now(),
UpdateTs: time.Now(),
},
}
var dataList = []services.Data{
{
Mac: "ac:12:fc:fd:ds:dd",
Ip: "192.10.2.1",
Master: true,
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
},
{
Mac: "22:12:fc:fd:ds:dd",
Ip: "182.10.2.2",
Master: true,
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
},
}
var executeble = []model.Executable{
{
Path: "/test",
FileName: "test.py",
DisplayName: "test.py",
},
}
var systemInfo = model.SystemInfo{ARCH: "x86",
OS: "linux",
Hostname: "test",
NumCpu: 4,
Executables: executeble,
}
func GetNodeList(c *gin.Context) {
nodes := NodeList
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: nodes,
})
}
func GetNode(c *gin.Context) {
var result model.Node
id := c.Param("id")
for _, node := range NodeList {
if node.Id == bson.ObjectId(id) {
result = node
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: result,
})
}
func Ping(c *gin.Context) {
data := dataList[0]
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: data,
})
}
func PostNode(c *gin.Context) {
id := c.Param("id")
var oldItem model.Node
for _, node := range NodeList {
if node.Id == bson.ObjectId(id) {
oldItem = node
}
}
log.Info(id)
var newItem model.Node
if err := c.ShouldBindJSON(&newItem); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
newItem.Id = oldItem.Id
log.Info("Post Node success")
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetNodeTaskList(c *gin.Context) {
tasks := TaskList
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: tasks,
})
}
func DeleteNode(c *gin.Context) {
id := bson.ObjectId("5d429e6c19f7abede924fee2")
for _, node := range NodeList {
if node.Id == bson.ObjectId(id) {
log.Infof("Delete a node")
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func GetSystemInfo(c *gin.Context) {
id := c.Param("id")
log.Info(id)
sysInfo := systemInfo
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: sysInfo,
})
}

187
backend/mock/node_test.go Normal file
View File

@@ -0,0 +1,187 @@
package mock
import (
"crawlab/model"
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
. "github.com/smartystreets/goconvey/convey"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"ucloudBilling/ucloud/log"
)
var app *gin.Engine
// 本测试依赖MongoDB的服务，所以在测试之前需要启动MongoDB及相关服务
func init() {
app = gin.Default()
// mock Test
// 节点相关的API
app.GET("/ping", Ping)
app.GET("/nodes", GetNodeList) // 节点列表
app.GET("/nodes/:id", GetNode) // 节点详情
app.POST("/nodes/:id", PostNode) // 修改节点
app.GET("/nodes/:id/tasks", GetNodeTaskList) // 节点任务列表
app.GET("/nodes/:id/system", GetSystemInfo) // 节点任务列表
app.DELETE("/nodes/:id", DeleteNode) // 删除节点
//// 爬虫
// 定时任务
app.GET("/schedules", GetScheduleList) // 定时任务列表
app.GET("/schedules/:id", GetSchedule) // 定时任务详情
app.PUT("/schedules", PutSchedule) // 创建定时任务
app.POST("/schedules/:id", PostSchedule) // 修改定时任务
app.DELETE("/schedules/:id", DeleteSchedule) // 删除定时任务
}
//mock test, test data in ./mock
func TestGetNodeList(t *testing.T) {
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/nodes", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
t.Log(resp.Data)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetNodeList", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestGetNode(t *testing.T) {
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/nodes/"+mongoId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
t.Log(resp.Data)
Convey("Test API GetNode", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
})
})
}
func TestPing(t *testing.T) {
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/ping", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API ping", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestGetNodeTaskList(t *testing.T) {
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/tasks", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetNodeTaskList", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestDeleteNode(t *testing.T) {
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
w := httptest.NewRecorder()
req, _ := http.NewRequest("DELETE", "nodes/"+mongoId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API DeleteNode", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestPostNode(t *testing.T) {
var newItem = model.Node{
Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
Ip: "10.32.35.15",
Name: "test1",
Status: "online",
Port: "8081",
Mac: "ac:12:df:12:fd",
Description: "For test1",
IsMaster: true,
UpdateTs: time.Now(),
CreateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
}
var resp Response
body, _ := json.Marshal(newItem)
log.Info(strings.NewReader(string(body)))
var mongoId = "5d429e6c19f7abede924fee2"
w := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "nodes/"+mongoId, strings.NewReader(string(body)))
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
t.Log(resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API PostNode", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestGetSystemInfo(t *testing.T) {
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "nodes/"+mongoId+"/system", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetSystemInfo", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}

126
backend/mock/schedule.go Normal file
View File

@@ -0,0 +1,126 @@
package mock
import (
"crawlab/constants"
"crawlab/model"
"fmt"
"github.com/gin-gonic/gin"
"github.com/globalsign/mgo/bson"
"net/http"
"time"
)
var scheduleList = []model.Schedule{
{
Id: bson.ObjectId("5d429e6c19f7abede924fee2"),
Name: "test schedule",
SpiderId: "123",
NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"),
Cron: "***1*",
EntryId: 10,
// 前端展示
SpiderName: "test scedule",
NodeName: "测试节点",
CreateTs: time.Now(),
UpdateTs: time.Now(),
},
{
Id: bson.ObjectId("xx429e6c19f7abede924fee2"),
Name: "test schedule2",
SpiderId: "234",
NodeId: bson.ObjectId("5d429e6c19f7abede924fee2"),
Cron: "***1*",
EntryId: 10,
// 前端展示
SpiderName: "test scedule2",
NodeName: "测试节点",
CreateTs: time.Now(),
UpdateTs: time.Now(),
},
}
func GetScheduleList(c *gin.Context) {
results := scheduleList
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: results,
})
}
func GetSchedule(c *gin.Context) {
id := c.Param("id")
var result model.Schedule
for _, sch := range scheduleList {
if sch.Id == bson.ObjectId(id) {
result = sch
}
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
Data: result,
})
}
func PostSchedule(c *gin.Context) {
id := c.Param("id")
var oldItem model.Schedule
for _, sch := range scheduleList {
if sch.Id == bson.ObjectId(id) {
oldItem = sch
}
}
var newItem model.Schedule
if err := c.ShouldBindJSON(&newItem); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
newItem.Id = oldItem.Id
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func PutSchedule(c *gin.Context) {
var item model.Schedule
// 绑定数据模型
if err := c.ShouldBindJSON(&item); err != nil {
HandleError(http.StatusBadRequest, c, err)
return
}
// 如果node_id为空，则置为空ObjectId
if item.NodeId == "" {
item.NodeId = bson.ObjectIdHex(constants.ObjectIdNull)
}
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}
func DeleteSchedule(c *gin.Context) {
id := bson.ObjectIdHex("5d429e6c19f7abede924fee2")
for _, sch := range scheduleList {
if sch.Id == bson.ObjectId(id) {
fmt.Println("delete a schedule")
}
}
fmt.Println(id)
fmt.Println("update schedule")
c.JSON(http.StatusOK, Response{
Status: "ok",
Message: "success",
})
}

144
backend/mock/schedule_test.go Normal file
View File

@@ -0,0 +1,144 @@
package mock
import (
"crawlab/model"
"encoding/json"
"github.com/globalsign/mgo/bson"
. "github.com/smartystreets/goconvey/convey"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"ucloudBilling/ucloud/log"
)
func TestGetScheduleList(t *testing.T) {
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/schedules", nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
t.Log(resp.Data)
Convey("Test API GetScheduleList", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestGetSchedule(t *testing.T) {
var mongoId = "5d429e6c19f7abede924fee2"
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/schedules/"+mongoId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test API GetSchedule", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
So(resp.Data.(map[string]interface{})["_id"], ShouldEqual, bson.ObjectId(mongoId).Hex())
})
})
}
func TestDeleteSchedule(t *testing.T) {
var mongoId = "5d429e6c19f7abede924fee2"
var resp Response
w := httptest.NewRecorder()
req, _ := http.NewRequest("DELETE", "/schedules/"+mongoId, nil)
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()), &resp)
log.Info(w.Body.String())
if err != nil {
t.Fatal("Unmarshal resp failed")
}
Convey("Test DeleteSchedule", t, func() {
Convey("Test resp status", func() {
So(resp.Status, ShouldEqual, "ok")
})
})
}
func TestPostSchedule(t *testing.T) {
var newItem = model.Schedule{
Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
Name: "test schedule",
SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
Cron: "***1*",
EntryId: 10,
// 前端展示
SpiderName: "test scedule",
NodeName: "测试节点",
CreateTs: time.Now(),
UpdateTs: time.Now(),
}
var resp Response
var mongoId = "5d429e6c19f7abede924fee2"
body,_ := json.Marshal(newItem)
log.Info(strings.NewReader(string(body)))
w := httptest.NewRecorder()
req,_ := http.NewRequest("POST", "/schedules/"+mongoId,strings.NewReader(string(body)))
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()),&resp)
t.Log(resp)
if err != nil {
t.Fatal("unmarshal resp failed")
}
Convey("Test API PostSchedule", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}
func TestPutSchedule(t *testing.T) {
var newItem = model.Schedule{
Id: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
Name: "test schedule",
SpiderId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
NodeId: bson.ObjectIdHex("5d429e6c19f7abede924fee2"),
Cron: "***1*",
EntryId: 10,
// 前端展示
SpiderName: "test scedule",
NodeName: "测试节点",
CreateTs: time.Now(),
UpdateTs: time.Now(),
}
var resp Response
body,_ := json.Marshal(newItem)
log.Info(strings.NewReader(string(body)))
w := httptest.NewRecorder()
req,_ := http.NewRequest("PUT", "/schedules",strings.NewReader(string(body)))
app.ServeHTTP(w, req)
err := json.Unmarshal([]byte(w.Body.String()),&resp)
t.Log(resp)
if err != nil {
t.Fatal("unmarshal resp failed")
}
Convey("Test API PutSchedule", t, func() {
Convey("Test response status", func() {
So(resp.Status, ShouldEqual, "ok")
So(resp.Message, ShouldEqual, "success")
})
})
}

1
backend/mock/spider.go Normal file
View File

@@ -0,0 +1 @@
package mock

1
backend/mock/system.go Normal file
View File

@@ -0,0 +1 @@
package mock

1
backend/mock/task.go Normal file
View File

@@ -0,0 +1 @@
package mock

1
backend/mock/user.go Normal file
View File

@@ -0,0 +1 @@
package mock

24
backend/mock/utils.go Normal file
View File

@@ -0,0 +1,24 @@
package mock
import (
"github.com/gin-gonic/gin"
"runtime/debug"
)
func HandleError(statusCode int, c *gin.Context, err error) {
debug.PrintStack()
c.JSON(statusCode, Response{
Status: "ok",
Message: "error",
Error: err.Error(),
})
}
func HandleErrorF(statusCode int, c *gin.Context, err string) {
debug.PrintStack()
c.JSON(statusCode, Response{
Status: "ok",
Message: "error",
Error: err,
})
}

8
backend/model/node.go
View File

@@ -17,13 +17,15 @@ type Node struct {
Port string `json:"port" bson:"port"`
Mac string `json:"mac" bson:"mac"`
Description string `json:"description" bson:"description"`
// 用于唯一标识节点，可能是mac地址，可能是ip地址
Key string `json:"key" bson:"key"`
// 前端展示
IsMaster bool `json:"is_master"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"`
UpdateTsUnix int64 `json:"update_ts_unix" bson:"update_ts_unix"`
}
func (n *Node) Save() error {
@@ -98,12 +100,12 @@ func GetNode(id bson.ObjectId) (Node, error) {
return node, nil
}
func GetNodeByMac(mac string) (Node, error) {
func GetNodeByKey(key string) (Node, error) {
s, c := database.GetCol("nodes")
defer s.Close()
var node Node
if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
if err != mgo.ErrNotFound {
log.Errorf(err.Error())
debug.PrintStack()

14
backend/routes/spider.go
View File

@@ -131,10 +131,22 @@ func PutSpider(c *gin.Context) {
return
}
// 以防tmp目录不存在
tmpPath := viper.GetString("other.tmppath")
if !utils.Exists(tmpPath) {
if err := os.Mkdir(tmpPath, os.ModePerm); err != nil {
log.Error("mkdir other.tmppath dir error:" + err.Error())
debug.PrintStack()
HandleError(http.StatusBadRequest, c, errors.New("Mkdir other.tmppath dir error"))
return
}
}
// 保存到本地临时文件
randomId := uuid.NewV4()
tmpFilePath := filepath.Join(viper.GetString("other.tmppath"), randomId.String()+".zip")
tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip")
if err := c.SaveUploadedFile(file, tmpFilePath); err != nil {
log.Error("save upload file error: " + err.Error())
debug.PrintStack()
HandleError(http.StatusInternalServerError, c, err)
return

70
backend/services/node.go
View File

@@ -16,6 +16,7 @@ import (
)
type Data struct {
Key string `json:"key"`
Mac string `json:"mac"`
Ip string `json:"ip"`
Master bool `json:"master"`
@@ -51,10 +52,9 @@ const (
// 获取本机节点
func GetCurrentNode() (model.Node, error) {
// 获取本机MAC地址
value, err := register.GetRegister().GetValue()
// 获得注册的key值
key, err := register.GetRegister().GetKey()
if err != nil {
debug.PrintStack()
return model.Node{}, err
}
@@ -68,8 +68,7 @@ func GetCurrentNode() (model.Node, error) {
}
// 尝试获取节点
node, err = model.GetNodeByMac(value)
node, err = model.GetNodeByKey(key)
// 如果获取失败
if err != nil {
// 如果为主节点，表示为第一次注册，插入节点信息
@@ -80,12 +79,26 @@ func GetCurrentNode() (model.Node, error) {
debug.PrintStack()
return model.Node{}, err
}
mac, err := register.GetRegister().GetMac()
if err != nil {
debug.PrintStack()
return model.Node{}, err
}
key, err := register.GetRegister().GetKey()
if err != nil {
debug.PrintStack()
return model.Node{}, err
}
// 生成节点
node = model.Node{
Key: key,
Id: bson.NewObjectId(),
Ip: ip,
Name: value,
Mac: value,
Name: key,
Mac: mac,
IsMaster: true,
}
if err := node.Add(); err != nil {
@@ -100,11 +113,9 @@ func GetCurrentNode() (model.Node, error) {
time.Sleep(5 * time.Second)
continue
}
// 跳出循环
break
}
return node, nil
}
@@ -122,12 +133,12 @@ func IsMasterNode(id string) bool {
// 获取节点数据
func GetNodeData() (Data, error) {
val, err := register.GetRegister().GetValue()
if err != nil {
key, err := register.GetRegister().GetKey()
if key == "" {
return Data{}, err
}
value, err := database.RedisClient.HGet("nodes", val)
value, err := database.RedisClient.HGet("nodes", key)
data := Data{}
if err := json.Unmarshal([]byte(value), &data); err != nil {
return data, err
@@ -145,9 +156,9 @@ func UpdateNodeStatus() {
}
// 遍历节点keys
for _, mac := range list {
for _, key := range list {
// 获取节点数据
value, err := database.RedisClient.HGet("nodes", mac)
value, err := database.RedisClient.HGet("nodes", key)
if err != nil {
log.Errorf(err.Error())
return
@@ -163,7 +174,7 @@ func UpdateNodeStatus() {
// 如果记录的更新时间超过60秒，该节点被认为离线
if time.Now().Unix()-data.UpdateTsUnix > 60 {
// 在Redis中删除该节点
if err := database.RedisClient.HDel("nodes", data.Mac); err != nil {
if err := database.RedisClient.HDel("nodes", data.Key); err != nil {
log.Errorf(err.Error())
return
}
@@ -172,7 +183,7 @@ func UpdateNodeStatus() {
s, c := database.GetCol("nodes")
defer s.Close()
var node model.Node
if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
@@ -189,14 +200,16 @@ func UpdateNodeStatus() {
s, c := database.GetCol("nodes")
defer s.Close()
var node model.Node
if err := c.Find(bson.M{"mac": mac}).One(&node); err != nil {
if err := c.Find(bson.M{"key": key}).One(&node); err != nil {
// 数据库不存在该节点
node = model.Node{
Name: data.Mac,
Ip: data.Ip,
Port: "8000",
Mac: data.Mac,
Status: constants.StatusOnline,
Key: key,
Name: key,
Ip: data.Ip,
Port: "8000",
Mac: data.Mac,
Status: constants.StatusOnline,
IsMaster: data.Master,
}
if err := node.Add(); err != nil {
log.Errorf(err.Error())
@@ -216,8 +229,8 @@ func UpdateNodeStatus() {
nodes, err := model.GetNodeList(nil)
for _, node := range nodes {
hasNode := false
for _, mac := range list {
if mac == node.Mac {
for _, key := range list {
if key == node.Key {
hasNode = true
break
}
@@ -236,7 +249,7 @@ func UpdateNodeStatus() {
// 更新节点数据
func UpdateNodeData() {
// 获取MAC地址
val, err := register.GetRegister().GetValue()
mac, err := register.GetRegister().GetMac()
if err != nil {
log.Errorf(err.Error())
return
@@ -248,10 +261,13 @@ func UpdateNodeData() {
log.Errorf(err.Error())
return
}
// 获取redis的key
key, err := register.GetRegister().GetKey()
// 构造节点数据
data := Data{
Mac: val,
Key: key,
Mac: mac,
Ip: ip,
Master: IsMaster(),
UpdateTs: time.Now(),
@@ -265,7 +281,7 @@ func UpdateNodeData() {
debug.PrintStack()
return
}
if err := database.RedisClient.HSet("nodes", val, string(dataBytes)); err != nil {
if err := database.RedisClient.HSet("nodes", key, string(dataBytes)); err != nil {
log.Errorf(err.Error())
return
}

58
backend/services/register/register.go
View File

@@ -10,50 +10,44 @@ import (
type Register interface {
// 注册的key类型
GetKey() string
GetType() string
// 注册的key的值，唯一标识节点
GetValue() (string, error)
GetKey() (string, error)
// 注册的节点IP
GetIp() (string, error)
// 注册节点的mac地址
GetMac() (string, error)
}
// mac 地址注册
// ===================== mac 地址注册 =====================
type MacRegister struct{}
func (mac *MacRegister) GetKey() string {
func (mac *MacRegister) GetType() string {
return "mac"
}
func (mac *MacRegister) GetValue() (string, error) {
interfaces, err := net.Interfaces()
if err != nil {
log.Errorf("get interfaces error:" + err.Error())
debug.PrintStack()
return "", err
}
for _, inter := range interfaces {
if inter.HardwareAddr != nil {
mac := inter.HardwareAddr.String()
return mac, nil
}
}
return "", nil
func (mac *MacRegister) GetKey() (string, error) {
return mac.GetMac()
}
func (mac *MacRegister) GetMac() (string, error) {
return getMac()
}
func (mac *MacRegister) GetIp() (string, error) {
return getIp()
}
// ip 注册
// ===================== ip 地址注册 =====================
type IpRegister struct {
Ip string
}
func (ip *IpRegister) GetKey() string {
func (ip *IpRegister) GetType() string {
return "ip"
}
func (ip *IpRegister) GetValue() (string, error) {
func (ip *IpRegister) GetKey() (string, error) {
return ip.Ip, nil
}
@@ -61,6 +55,11 @@ func (ip *IpRegister) GetIp() (string, error) {
return ip.Ip, nil
}
func (ip *IpRegister) GetMac() (string, error) {
return getMac()
}
// ===================== 公共方法 =====================
// 获取本机的IP地址
// TODO: 考虑多个IP地址的情况
func getIp() (string, error) {
@@ -78,6 +77,23 @@ func getIp() (string, error) {
return "", nil
}
func getMac() (string, error) {
interfaces, err := net.Interfaces()
if err != nil {
log.Errorf("get interfaces error:" + err.Error())
debug.PrintStack()
return "", err
}
for _, inter := range interfaces {
if inter.HardwareAddr != nil {
mac := inter.HardwareAddr.String()
return mac, nil
}
}
return "", nil
}
// ===================== 获得注册简单工厂 =====================
var register Register
// 获得注册器

65
backend/services/spider.go
View File

@@ -7,10 +7,12 @@ import (
"crawlab/model"
"crawlab/utils"
"encoding/json"
"fmt"
"github.com/apex/log"
"github.com/globalsign/mgo"
"github.com/globalsign/mgo/bson"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"github.com/satori/go.uuid"
"github.com/spf13/viper"
"io"
"io/ioutil"
@@ -162,24 +164,6 @@ func ZipSpider(spider model.Spider) (filePath string, err error) {
func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid bson.ObjectId, err error) {
fid = ""
// 读取zip文件
file, err := os.OpenFile(filePath, os.O_RDONLY, 0777)
fileBytes, err := ioutil.ReadAll(file)
if err != nil {
debug.PrintStack()
return
}
if err = file.Close(); err != nil {
debug.PrintStack()
return
}
// 删除zip文件
if err = os.Remove(filePath); err != nil {
debug.PrintStack()
return
}
// 获取MongoDB GridFS连接
s, gf := database.GetGridFs("files")
defer s.Close()
@@ -198,23 +182,58 @@ func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid
return
}
// 将文件写入到GridFS
if _, err = f.Write(fileBytes); err != nil {
//分片读取爬虫zip文件
err = ReadFileByStep(filePath, WriteToGridFS, f)
if err != nil {
debug.PrintStack()
return "", err
}
// 删除zip文件
if err = os.Remove(filePath); err != nil {
debug.PrintStack()
return
}
// 关闭文件，提交写入
if err = f.Close(); err != nil {
return "", err
}
// 文件ID
fid = f.Id().(bson.ObjectId)
return fid, nil
}
func WriteToGridFS(content []byte, f *mgo.GridFile) {
if _, err := f.Write(content); err != nil {
debug.PrintStack()
return
}
}
//分片读取大文件
func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCreate *mgo.GridFile) error {
f, err := os.OpenFile(filePath, os.O_RDONLY, 0777)
if err != nil {
log.Infof("can't opened this file")
return err
}
defer f.Close()
s := make([]byte, 4096)
for {
switch nr, err := f.Read(s[:]); true {
case nr < 0:
fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error())
debug.PrintStack()
case nr == 0: // EOF
return nil
case nr > 0:
handle(s[0:nr], fileCreate)
}
}
return nil
}
// 发布所有爬虫
func PublishAllSpiders() error {
// 获取爬虫列表

76
backend/utils/chan_test.go Normal file
View File

@@ -0,0 +1,76 @@
package utils
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestNewChanMap(t *testing.T) {
mapTest := make(map[string]chan string)
chanTest := make(chan string)
test := "test"
Convey("Call NewChanMap to generate ChanMap", t, func() {
mapTest[test] = chanTest
chanMapTest := ChanMap{mapTest}
chanMap := NewChanMap()
chanMap.m[test] = chanTest
Convey(test, func() {
So(chanMap, ShouldResemble, &chanMapTest)
})
})
}
func TestChan(t *testing.T) {
mapTest := make(map[string]chan string)
chanTest := make(chan string)
mapTest["test"] = chanTest
chanMapTest := ChanMap{mapTest}
Convey("Test Chan use exist key", t, func() {
ch1 := chanMapTest.Chan(
"test")
Convey("ch1 should equal chanTest", func() {
So(ch1, ShouldEqual, chanTest)
})
})
Convey("Test Chan use no-exist key", t, func() {
ch2 := chanMapTest.Chan("test2")
Convey("ch2 should equal chanMapTest.m[test2]", func() {
So(chanMapTest.m["test2"], ShouldEqual, ch2)
})
Convey("Cap of chanMapTest.m[test2] should equal 10", func() {
So(10, ShouldEqual, cap(chanMapTest.m["test2"]))
})
})
}
func TestChanBlocked(t *testing.T) {
mapTest := make(map[string]chan string)
chanTest := make(chan string)
mapTest["test"] = chanTest
chanMapTest := ChanMap{mapTest}
Convey("Test Chan use exist key", t, func() {
ch1 := chanMapTest.ChanBlocked(
"test")
Convey("ch1 should equal chanTest", func() {
So(ch1, ShouldEqual, chanTest)
})
})
Convey("Test Chan use no-exist key", t, func() {
ch2 := chanMapTest.ChanBlocked("test2")
Convey("ch2 should equal chanMapTest.m[test2]", func() {
So(chanMapTest.m["test2"], ShouldEqual, ch2)
})
Convey("Cap of chanMapTest.m[test2] should equal 10", func() {
So(0, ShouldEqual, cap(chanMapTest.m["test2"]))
})
})
}

72
backend/utils/file_test.go Normal file
View File

@@ -0,0 +1,72 @@
package utils
import (
. "github.com/smartystreets/goconvey/convey"
"os"
"testing"
)
func TestExists(t *testing.T) {
var pathString = "../config"
var wrongPathString = "test"
Convey("Test path or file is Exists or not", t, func() {
res := Exists(pathString)
Convey("The result should be true", func() {
So(res, ShouldEqual, true)
})
wrongRes := Exists(wrongPathString)
Convey("The result should be false", func() {
So(wrongRes, ShouldEqual, false)
})
})
}
func TestIsDir(t *testing.T) {
var pathString = "../config"
var fileString = "../config/config.go"
var wrongString = "test"
Convey("Test path is folder or not", t, func() {
res := IsDir(pathString)
So(res, ShouldEqual, true)
fileRes := IsDir(fileString)
So(fileRes, ShouldEqual, false)
wrongRes := IsDir(wrongString)
So(wrongRes, ShouldEqual, false)
})
}
func TestCompress(t *testing.T) {
var pathString = "../utils"
var files []*os.File
var disPath = "../utils/test"
file, err := os.Open(pathString)
if err != nil {
t.Error("open source path failed")
}
files = append(files, file)
Convey("Verify dispath is valid path", t, func() {
er := Compress(files, disPath)
Convey("err should be nil", func() {
So(er, ShouldEqual, nil)
})
})
}
// 测试之前需存在有效的test(.zip)文件
func TestDeCompress(t *testing.T) {
var tmpFilePath = "./test"
tmpFile, err := os.OpenFile(tmpFilePath, os.O_RDONLY, 0777)
if err != nil {
t.Fatal("open zip file failed")
}
var dstPath = "./testDeCompress"
Convey("Test DeCopmress func", t, func() {
err := DeCompress(tmpFile, dstPath)
So(err, ShouldEqual, nil)
})
}

49
backend/utils/model_test.go Normal file
View File

@@ -0,0 +1,49 @@
package utils
import (
"github.com/globalsign/mgo/bson"
. "github.com/smartystreets/goconvey/convey"
"strconv"
"testing"
"time"
)
func TestIsObjectIdNull(t *testing.T) {
var id bson.ObjectId = "123455"
Convey("Test Object ID is null or not", t, func() {
res := IsObjectIdNull(id)
So(res, ShouldEqual, false)
})
}
func TestInterfaceToString(t *testing.T) {
var valueBson bson.ObjectId = "12345"
var valueString = "12345"
var valueInt = 12345
var valueTime = time.Now().Add(60 * time.Second)
var valueOther = []string{"a", "b"}
Convey("Test InterfaceToString", t, func() {
resBson := InterfaceToString(valueBson)
Convey("resBson should be string value", func() {
So(resBson, ShouldEqual, valueBson.Hex())
})
resString := InterfaceToString(valueString)
Convey("resString should be string value", func() {
So(resString, ShouldEqual, valueString)
})
resInt := InterfaceToString(valueInt)
Convey("resInt should be string value", func() {
So(resInt, ShouldEqual, strconv.Itoa(valueInt))
})
resTime := InterfaceToString(valueTime)
Convey("resTime should be string value", func() {
So(resTime, ShouldEqual, valueTime.String())
})
resOther := InterfaceToString(valueOther)
Convey("resOther should be empty string", func() {
So(resOther, ShouldEqual, "")
})
})
}

14
backend/utils/user_test.go Normal file
View File

@@ -0,0 +1,14 @@
package utils
import (
. "github.com/smartystreets/goconvey/convey"
"testing"
)
func TestEncryptPassword(t *testing.T) {
var passwd = "test"
Convey("Test EncryptPassword", t, func() {
res := EncryptPassword(passwd)
t.Log(res)
})
}

24
frontend/src/views/layout/components/TagsView.vue
View File

@@ -110,20 +110,22 @@ export default {
},
moveToCurrentTag () {
const tags = this.$refs.tag
this.$nextTick(() => {
for (const tag of tags) {
if (tag.to.path === this.$route.path) {
this.$refs.scrollPane.moveToTarget(tag)
if (tags) {
this.$nextTick(() => {
for (const tag of tags) {
if (tag.to.path === this.$route.path) {
this.$refs.scrollPane.moveToTarget(tag)
// when query is different then update
if (tag.to.fullPath !== this.$route.fullPath) {
this.$store.dispatch('updateVisitedView', this.$route)
// when query is different then update
if (tag.to.fullPath !== this.$route.fullPath) {
this.$store.dispatch('updateVisitedView', this.$route)
}
break
}
break
}
}
})
})
}
},
refreshSelectedTag (view) {
this.$store.dispatch('delCachedView', view).then(() => {