From 0ad2a06d369cfbce9d09928be98681c3a59ee56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 24 Aug 2019 15:11:52 +0800 Subject: [PATCH 01/19] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/services/node.go b/backend/services/node.go index 1fa2370c..09b49dbf 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -124,6 +124,7 @@ func IsMaster() bool { return viper.GetString("server.master") == Yes } +// 所有调用IsMasterNode的方法，都永远会在master节点执行，所以GetCurrentNode方法返回永远是master节点 // 该ID的节点是否为主节点 func IsMasterNode(id string) bool { curNode, _ := GetCurrentNode() From b11f31ec18a9d7b2983cfa2919fb613f7fd90ade Mon Sep 17 00:00:00 2001 From: yaziming Date: Mon, 26 Aug 2019 23:13:26 +0800 Subject: [PATCH 02/19] JSON -> AbortWithStatusJSON --- backend/routes/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/routes/utils.go b/backend/routes/utils.go index 14c5853e..2b45c5f1 100644 --- a/backend/routes/utils.go +++ b/backend/routes/utils.go @@ -7,7 +7,7 @@ import ( func HandleError(statusCode int, c *gin.Context, err error) { debug.PrintStack() - c.JSON(statusCode, Response{ + c.AbortWithStatusJSON(statusCode, Response{ Status: "ok", Message: "error", Error: err.Error(), @@ -16,7 +16,7 @@ func HandleError(statusCode int, c *gin.Context, err error) { func HandleErrorF(statusCode int, c *gin.Context, err string) { debug.PrintStack() - c.JSON(statusCode, Response{ + c.AbortWithStatusJSON(statusCode, Response{ Status: "ok", Message: "error", Error: err, From 0186d081124f59aaafb13758b5570d037da13a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Tue, 27 Aug 2019 09:39:16 +0800 Subject: [PATCH 03/19] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E9=A1=BA=E5=B8=A6=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 11 +++++++++-- backend/routes/utils.go | 2 ++ backend/services/spider.go | 2 +- backend/services/task.go | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index c4c94edf..65782fe8 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -92,8 +92,6 @@ func (spider *Spider) GetLastTask() (Task, error) { return tasks[0], nil } - - func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { s, c := database.GetCol("spiders") defer s.Close() @@ -165,6 +163,15 @@ func RemoveSpider(id bson.ObjectId) error { return err } + // gf上的文件 + s, gf := database.GetGridFs("files") + defer s.Close() + + if err := gf.RemoveId(result.FileId); err != nil { + log.Error("remove file error, id:" + result.FileId.Hex()) + return err + } + return nil } diff --git a/backend/routes/utils.go b/backend/routes/utils.go index 14c5853e..8044ffc9 100644 --- a/backend/routes/utils.go +++ b/backend/routes/utils.go @@ -1,11 +1,13 @@ package routes import ( + "github.com/apex/log" "github.com/gin-gonic/gin" "runtime/debug" ) func HandleError(statusCode int, c *gin.Context, err error) { + log.Errorf("handle error:" + err.Error()) debug.PrintStack() c.JSON(statusCode, Response{ Status: "ok", diff --git a/backend/services/spider.go b/backend/services/spider.go index f4f856e6..5c5c6e6e 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -320,7 +320,7 @@ func OnFileUpload(channel string, msgStr string) { // 从GridFS获取该文件 f, err := gf.OpenId(bson.ObjectIdHex(msg.FileId)) if err != nil { - log.Errorf(err.Error()) + log.Errorf("open file id" + msg.FileId + ", error: " + err.Error()) debug.PrintStack() return } diff --git a/backend/services/task.go b/backend/services/task.go index 8c0ff8a1..8c3c3407 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -472,6 +472,7 @@ func CancelTask(id string) (err error) { } func HandleTaskError(t model.Task, err error) { + log.Error("handle task error:" + err.Error()) t.Status = constants.StatusError t.Error = err.Error() t.FinishTs = time.Now() From 1736d00161db53fcede2314748e391b886232097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Wed, 28 Aug 2019 16:00:09 +0800 Subject: [PATCH 04/19] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=89=93=E5=8D=B0?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index 5c5c6e6e..ce416a79 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -295,6 +295,7 @@ func PublishSpider(spider model.Spider) (err error) { return } channel := "files:upload" + log.Info("publish files.upload event, file id:" + msg.FileId) if err = database.Publish(channel, string(msgStr)); err != nil { log.Errorf(err.Error()) debug.PrintStack() @@ -306,6 +307,7 @@ func PublishSpider(spider model.Spider) (err error) { // 上传爬虫回调 func OnFileUpload(channel string, msgStr string) { + log.Info("received files.upload event, msgStr:" + msgStr) s, gf := database.GetGridFs("files") defer s.Close() @@ -320,7 +322,7 @@ func OnFileUpload(channel string, msgStr string) { // 从GridFS获取该文件 f, err := gf.OpenId(bson.ObjectIdHex(msg.FileId)) if err != nil { - log.Errorf("open file id" + msg.FileId + ", error: " + err.Error()) + log.Errorf("open file id: " + msg.FileId + ", error: " + err.Error()) debug.PrintStack() return } From 9db9365a2c0a4bd23ca32541673c586300679d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 10:08:37 +0800 Subject: [PATCH 05/19] fix #184 --- frontend/src/components/InfoView/TaskInfoView.vue | 6 +++--- frontend/src/views/task/TaskDetail.vue | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/InfoView/TaskInfoView.vue b/frontend/src/components/InfoView/TaskInfoView.vue index bfe6419a..e902e959 100644 --- a/frontend/src/components/InfoView/TaskInfoView.vue +++ b/frontend/src/components/InfoView/TaskInfoView.vue @@ -86,15 +86,15 @@ export default { return dayjs(str).format('YYYY-MM-DD HH:mm:ss') }, getWaitDuration (row) { - if (row.start_ts.match('^0001')) return 'NA' + if (!row.start_ts || row.start_ts.match('^0001')) return 'NA' return dayjs(row.start_ts).diff(row.create_ts, 'second') }, getRuntimeDuration (row) { - if (row.finish_ts.match('^0001')) return 'NA' + if (!row.finish_ts || row.finish_ts.match('^0001')) return 'NA' return dayjs(row.finish_ts).diff(row.start_ts, 'second') }, getTotalDuration (row) { - if (row.finish_ts.match('^0001')) return 'NA' + if (!row.finish_ts || row.finish_ts.match('^0001')) return 'NA' return dayjs(row.finish_ts).diff(row.create_ts, 'second') } } diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index a1edb497..89309ea8 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -99,12 +99,12 @@ export default { this.$st.sendEv('任务详情-结果', '下载CSV') } }, - created () { - this.$store.dispatch('task/getTaskData', this.$route.params.id) + async created () { + await this.$store.dispatch('task/getTaskData', this.$route.params.id) this.$store.dispatch('task/getTaskLog', this.$route.params.id) this.$store.dispatch('task/getTaskResults', this.$route.params.id) - if (['running'].includes(this.taskForm.status)) { + if (this.taskForm && ['running'].includes(this.taskForm.status)) { this.handle = setInterval(() => { this.$store.dispatch('task/getTaskLog', this.$route.params.id) }, 5000) From c634e2b5cf1d8882f5cd7a3918f9d58c89583b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 12:04:12 +0800 Subject: [PATCH 06/19] =?UTF-8?q?fix=20=E5=89=8D=E7=AB=AF=E6=8E=A7?= =?UTF-8?q?=E5=88=B6=E5=8F=B0=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20\=20fix=20=E6=97=A0=E6=B3=95=E6=89=93=E5=8D=B0=E4=B8=AD?= =?UTF-8?q?=E6=96=87=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 ++++ docker/Dockerfile.master.alpine | 2 +- docker/Dockerfile.worker.alpine | 2 +- frontend/src/components/InfoView/NodeInfoView.vue | 2 +- frontend/src/views/layout/components/Navbar.vue | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index 1397e335..ad0c0ae5 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -20,6 +20,7 @@ import ( "path/filepath" "runtime/debug" "strings" + "syscall" ) type SpiderFileData struct { @@ -133,6 +134,8 @@ func ZipSpider(spider model.Spider) (filePath string, err error) { // 如果源文件夹不存在，抛错 if !utils.Exists(spider.Src) { debug.PrintStack() + // 删除该爬虫，否则会一直报错 + _ = model.RemoveSpider(spider.Id) return "", errors.New("source path does not exist") } @@ -173,6 +176,7 @@ func UploadToGridFs(spider model.Spider, fileName string, filePath string) (fid // 如果存在FileId删除GridFS上的老文件 if !utils.IsObjectIdNull(spider.FileId) { if err = gf.RemoveId(spider.FileId); err != nil { + log.Error("remove gf file:" + err.Error()) debug.PrintStack() } } diff --git a/docker/Dockerfile.master.alpine b/docker/Dockerfile.master.alpine index 6979861b..b9dbb742 100644 --- a/docker/Dockerfile.master.alpine +++ b/docker/Dockerfile.master.alpine @@ -75,7 +75,7 @@ RUN sed -i 's/#rc_sys=""/rc_sys="lxc"/g' /etc/rc.conf && \ # working directory WORKDIR /app/backend - +ENV PYTHONIOENCODING utf-8 # frontend port EXPOSE 8080 diff --git a/docker/Dockerfile.worker.alpine b/docker/Dockerfile.worker.alpine index e7a66776..388125a2 100644 --- a/docker/Dockerfile.worker.alpine +++ b/docker/Dockerfile.worker.alpine @@ -35,7 +35,7 @@ RUN apk del .build-deps # working directory WORKDIR /app/backend - +ENV PYTHONIOENCODING utf-8 # backend port EXPOSE 8000 diff --git a/frontend/src/components/InfoView/NodeInfoView.vue b/frontend/src/components/InfoView/NodeInfoView.vue index 8e350448..e6ffb58a 100644 --- a/frontend/src/components/InfoView/NodeInfoView.vue +++ b/frontend/src/components/InfoView/NodeInfoView.vue @@ -22,7 +22,7 @@ - {{$t('Save')}} + {{$t('Save')}} diff --git a/frontend/src/views/layout/components/Navbar.vue b/frontend/src/views/layout/components/Navbar.vue index f60c0051..3b30c049 100644 --- a/frontend/src/views/layout/components/Navbar.vue +++ b/frontend/src/views/layout/components/Navbar.vue @@ -32,6 +32,7 @@ {{$t('Documentation')}} + From e43332b34f9ed6ccb55d3551f4635b6421c6394b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 12:42:53 +0800 Subject: [PATCH 07/19] =?UTF-8?q?FIX=20#178=20FIX=20=E5=BD=93=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=96=87=E4=BB=B6=E5=B0=8F=E4=BA=8E2048=E7=9A=84?= =?UTF-8?q?=E6=97=B6=E5=80=99=E6=97=A0=E6=B3=95=E6=AD=A3=E5=B8=B8=E6=8B=89?= =?UTF-8?q?=E5=8F=96=E6=97=A5=E5=BF=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/log.go | 9 +++++++-- backend/services/node.go | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/services/log.go b/backend/services/log.go index c6e4f090..a248c176 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -35,8 +35,13 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } defer f.Close() logBuf := make([]byte, 2048) - n, err := f.ReadAt(logBuf, fi.Size()-int64(len(logBuf))) - if err != nil { + off := int64(0) + if fi.Size() > int64(len(logBuf)) { + off = fi.Size() - int64(len(logBuf)) + } + n, err := f.ReadAt(logBuf, off) + // 到文件结尾会有EOF的报错 + if err.Error() != "EOF" && err != nil { log.Error(err.Error()) debug.PrintStack() return nil, err diff --git a/backend/services/node.go b/backend/services/node.go index 09b49dbf..083fdc3d 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -97,7 +97,7 @@ func GetCurrentNode() (model.Node, error) { Key: key, Id: bson.NewObjectId(), Ip: ip, - Name: key, + Name: ip, Mac: mac, IsMaster: true, } @@ -205,7 +205,7 @@ func UpdateNodeStatus() { // 数据库不存在该节点 node = model.Node{ Key: key, - Name: key, + Name: data.Ip, Ip: data.Ip, Port: "8000", Mac: data.Mac, From 494601ab625a53aa91e9cc3ce7c4cb5d45e84741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 13:49:34 +0800 Subject: [PATCH 08/19] =?UTF-8?q?fix=20=E8=8A=82=E7=82=B9=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E5=BC=82=E5=B8=B8=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/node.go | 47 +++++++++++++++ backend/model/node_test.go | 50 ++++++++++++++++ backend/services/node.go | 116 ++++++++++++++----------------------- backend/services/spider.go | 4 +- 4 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 backend/model/node_test.go diff --git a/backend/model/node.go b/backend/model/node.go index 61c20473..6211115c 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -1,7 +1,9 @@ package model import ( + "crawlab/constants" "crawlab/database" + "crawlab/services/register" "github.com/apex/log" "github.com/globalsign/mgo" "github.com/globalsign/mgo/bson" @@ -79,6 +81,7 @@ func GetNodeList(filter interface{}) ([]Node, error) { var results []Node if err := c.Find(filter).All(&results); err != nil { + log.Error("get node list error: " + err.Error()) debug.PrintStack() return results, err } @@ -153,3 +156,47 @@ func GetNodeCount(query interface{}) (int, error) { return count, nil } + +// 节点基本信息 +func GetNodeBaseInfo() (ip string, mac string, key string, error error) { + ip, err := register.GetRegister().GetIp() + if err != nil { + debug.PrintStack() + return "", "", "", err + } + + mac, err = register.GetRegister().GetMac() + if err != nil { + debug.PrintStack() + return "", "", "", err + } + + key, err = register.GetRegister().GetKey() + if err != nil { + debug.PrintStack() + return "", "", "", err + } + return ip, mac, key, nil +} + +// 根据redis的key值，重置node节点为offline +func ResetNodeStatusToOffline(list []string) { + nodes, _ := GetNodeList(nil) + for _, node := range nodes { + hasNode := false + for _, key := range list { + if key == node.Key { + hasNode = true + break + } + } + if !hasNode || node.Status == "" { + node.Status = constants.StatusOffline + if err := node.Save(); err != nil { + log.Errorf(err.Error()) + return + } + continue + } + } +} diff --git a/backend/model/node_test.go b/backend/model/node_test.go new file mode 100644 index 00000000..ba3f4aaa --- /dev/null +++ b/backend/model/node_test.go @@ -0,0 +1,50 @@ +package model + +import ( + "crawlab/config" + "crawlab/constants" + "crawlab/database" + "github.com/apex/log" + . "github.com/smartystreets/goconvey/convey" + "runtime/debug" + "testing" +) + +func TestAddNode(t *testing.T) { + Convey("Test AddNode", t, func() { + if err := config.InitConfig("../conf/config.yml"); err != nil { + log.Error("init config error:" + err.Error()) + panic(err) + } + log.Info("初始化配置成功") + + // 初始化Mongodb数据库 + if err := database.InitMongo(); err != nil { + log.Error("init mongodb error:" + err.Error()) + debug.PrintStack() + panic(err) + } + log.Info("初始化Mongodb数据库成功") + + // 初始化Redis数据库 + if err := database.InitRedis(); err != nil { + log.Error("init redis error:" + err.Error()) + debug.PrintStack() + panic(err) + } + + var node = Node{ + Key: "c4:b3:01:bd:b5:e7", + Name: "10.27.238.101", + Ip: "10.27.238.101", + Port: "8000", + Mac: "c4:b3:01:bd:b5:e7", + Status: constants.StatusOnline, + IsMaster: true, + } + if err := node.Add(); err != nil { + log.Error("add node error:" + err.Error()) + panic(err) + } + }) +} diff --git a/backend/services/node.go b/backend/services/node.go index 083fdc3d..3ed84149 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -73,23 +73,11 @@ func GetCurrentNode() (model.Node, error) { if err != nil { // 如果为主节点，表示为第一次注册，插入节点信息 if IsMaster() { - // 获取本机IP地址 - ip, err := register.GetRegister().GetIp() + // 获取本机信息 + ip, mac, key, err := model.GetNodeBaseInfo() if err != nil { debug.PrintStack() - return model.Node{}, err - } - - mac, err := register.GetRegister().GetMac() - if err != nil { - debug.PrintStack() - return model.Node{}, err - } - - key, err := register.GetRegister().GetKey() - if err != nil { - debug.PrintStack() - return model.Node{}, err + return node, err } // 生成节点 @@ -179,70 +167,56 @@ func UpdateNodeStatus() { log.Errorf(err.Error()) return } - // 在MongoDB中该节点设置状态为离线 - s, c := database.GetCol("nodes") - defer s.Close() - var node model.Node - if err := c.Find(bson.M{"key": key}).One(&node); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return - } - node.Status = constants.StatusOffline - if err := node.Save(); err != nil { - log.Errorf(err.Error()) - return - } + keys, _ := database.RedisClient.HKeys("nodes") + model.ResetNodeStatusToOffline(keys) continue } - // 更新节点信息到数据库 - s, c := database.GetCol("nodes") - defer s.Close() - var node model.Node - if err := c.Find(bson.M{"key": key}).One(&node); err != nil { - // 数据库不存在该节点 - node = model.Node{ - Key: key, - Name: data.Ip, - Ip: data.Ip, - Port: "8000", - Mac: data.Mac, - Status: constants.StatusOnline, - IsMaster: data.Master, - } - if err := node.Add(); err != nil { - log.Errorf(err.Error()) - return - } - } else { - // 数据库存在该节点 - node.Status = constants.StatusOnline - if err := node.Save(); err != nil { - log.Errorf(err.Error()) - return - } + // 处理node信息 + handleNodeInfo(key, data) + } + + // 重置不在redis的key为offline + model.ResetNodeStatusToOffline(list) +} + +func handleNodeInfo(key string, data Data) { + // 更新节点信息到数据库 + s, c := database.GetCol("nodes") + defer s.Close() + + // 同个key可能因为并发，被注册多次 + var nodes []model.Node + _ = c.Find(bson.M{"key": key}).All(&nodes) + if nodes != nil && len(nodes) > 1 { + for _, node := range nodes { + _ = c.RemoveId(node.Id) } } - // 遍历数据库中的节点列表 - nodes, err := model.GetNodeList(nil) - for _, node := range nodes { - hasNode := false - for _, key := range list { - if key == node.Key { - hasNode = true - break - } + var node model.Node + if err := c.Find(bson.M{"key": key}).One(&node); err != nil { + // 数据库不存在该节点 + node = model.Node{ + Key: key, + Name: data.Ip, + Ip: data.Ip, + Port: "8000", + Mac: data.Mac, + Status: constants.StatusOnline, + IsMaster: data.Master, } - if !hasNode { - node.Status = constants.StatusOffline - if err := node.Save(); err != nil { - log.Errorf(err.Error()) - return - } - continue + if err := node.Add(); err != nil { + log.Errorf(err.Error()) + return + } + } else { + // 数据库存在该节点 + node.Status = constants.StatusOnline + if err := node.Save(); err != nil { + log.Errorf(err.Error()) + return } } } diff --git a/backend/services/spider.go b/backend/services/spider.go index ad0c0ae5..a3242849 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -40,7 +40,7 @@ func GetSpidersFromDir() ([]model.Spider, error) { // 如果爬虫项目目录不存在，则创建一个 if !utils.Exists(srcPath) { - mask := syscall.Umask(0) // 改为 0000 八进制 + mask := syscall.Umask(0) // 改为 0000 八进制 defer syscall.Umask(mask) // 改为原来的 umask if err := os.MkdirAll(srcPath, 0666); err != nil { debug.PrintStack() @@ -301,7 +301,6 @@ func PublishSpider(spider model.Spider) (err error) { return } channel := "files:upload" - log.Info("publish files.upload event, file id:" + msg.FileId) if err = database.Publish(channel, string(msgStr)); err != nil { log.Errorf(err.Error()) debug.PrintStack() @@ -313,7 +312,6 @@ func PublishSpider(spider model.Spider) (err error) { // 上传爬虫回调 func OnFileUpload(channel string, msgStr string) { - log.Info("received files.upload event, msgStr:" + msgStr) s, gf := database.GetGridFs("files") defer s.Close() From f750fe5f8a42a9b6a294658133ac98ecb6cb8fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 13:50:01 +0800 Subject: [PATCH 09/19] =?UTF-8?q?fix=20=E5=89=8D=E7=AB=AF=E6=8A=A5?= =?UTF-8?q?=E9=94=99=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/store/modules/node.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frontend/src/store/modules/node.js b/frontend/src/store/modules/node.js index 266beb3e..5e21a222 100644 --- a/frontend/src/store/modules/node.js +++ b/frontend/src/store/modules/node.js @@ -25,15 +25,7 @@ const mutations = { const { id, systemInfo } = payload for (let i = 0; i < state.nodeList.length; i++) { if (state.nodeList[i]._id === id) { - // Vue.set(state.nodeList[i], 'systemInfo', {}) state.nodeList[i].systemInfo = systemInfo - // for (const key in systemInfo) { - // if (systemInfo.hasOwnProperty(key)) { - // console.log(key) - // state.nodeList[i].systemInfo[key] = systemInfo[key] - // // Vue.set(state.nodeList[i].systemInfo, key, systemInfo[key]) - // } - // } break } } @@ -76,10 +68,12 @@ const actions = { getTaskList ({ state, commit }, id) { return request.get(`/nodes/${id}/tasks`) .then(response => { - commit('task/SET_TASK_LIST', - response.data.data.map(d => d) - .sort((a, b) => a.create_ts < b.create_ts ? 1 : -1), - { root: true }) + if (response.data.data) { + commit('task/SET_TASK_LIST', + response.data.data.map(d => d) + .sort((a, b) => a.create_ts < b.create_ts ? 1 : -1), + { root: true }) + } }) }, getNodeSystemInfo ({ state, commit }, id) { From ecaf31f0f07644742dcf8c4e5051abd5207510b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 13:57:14 +0800 Subject: [PATCH 10/19] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/services/node.go b/backend/services/node.go index 3ed84149..977ff0ef 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -165,11 +165,7 @@ func UpdateNodeStatus() { // 在Redis中删除该节点 if err := database.RedisClient.HDel("nodes", data.Key); err != nil { log.Errorf(err.Error()) - return } - // 在MongoDB中该节点设置状态为离线 - keys, _ := database.RedisClient.HKeys("nodes") - model.ResetNodeStatusToOffline(keys) continue } From d8acab77539dc8a8084771323234b99d4a8ec0ad Mon Sep 17 00:00:00 2001 From: yaziming Date: Sat, 31 Aug 2019 14:57:09 +0800 Subject: [PATCH 11/19] =?UTF-8?q?Backend:=20=20=20=20=201.[improve]=20=20?= =?UTF-8?q?=20=20=20=20=20=20=E4=BD=BF=E7=94=A8gin=E6=8F=90=E4=BE=9B?= =?UTF-8?q?=E7=9A=84RouteGroup=E5=8A=9F=E8=83=BD=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E4=B8=AD=E9=97=B4=E4=BB=B6=E9=80=BB=E8=BE=91=20=20=20=20=202.[?= =?UTF-8?q?break=20change]=20=20=20=20=20=20=20=20=E7=A7=BB=E9=99=A4Author?= =?UTF-8?q?ization=20Middleware=E4=B8=AD=E5=AF=B9=E7=99=BB=E5=BD=95?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E4=B8=8B=E8=BD=BD=E7=89=B9=E6=AE=8A=E5=A4=84?= =?UTF-8?q?=E7=90=86=E9=80=BB=E8=BE=91=20=20=20=20=203.[unsafe=20problem]?= =?UTF-8?q?=20=20=20=20=20=20=20=20=E4=B8=8B=E8=BD=BD=E4=BB=BB=E5=8A=A1csv?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=99=BB=E5=BD=95=E9=AA=8C=E8=AF=81=20Fronte?= =?UTF-8?q?nd:=20=20=20=20=201.=20=E6=9B=B4=E6=94=B9csv=E4=B8=8B=E8=BD=BD?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=20=20=20=20=202.=20task=20list=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2table=20section=20=E7=A7=BB=E9=99=A4=E5=9B=BA=E5=AE=9A?= =?UTF-8?q?width=E8=AE=BE=E7=BD=AE,=E9=87=87=E7=94=A8=E8=87=AA=E9=80=82?= =?UTF-8?q?=E5=BA=94,=E9=98=B2=E6=AD=A2=E5=A4=A7=E5=B1=8F=E7=A9=BA?= =?UTF-8?q?=E7=99=BD=E6=96=AD=E8=A3=82=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 99 ++++++++++++++------------ backend/middlewares/auth.go | 12 ++-- frontend/src/api/request.js | 5 +- frontend/src/store/modules/task.js | 16 +++++ frontend/src/views/task/TaskDetail.vue | 2 +- frontend/src/views/task/TaskList.vue | 2 +- 6 files changed, 80 insertions(+), 56 deletions(-) diff --git a/backend/main.go b/backend/main.go index 489a17ce..31b5d64b 100644 --- a/backend/main.go +++ b/backend/main.go @@ -90,53 +90,60 @@ func main() { if services.IsMaster() { // 中间件 app.Use(middlewares.CORSMiddleware()) - app.Use(middlewares.AuthorizationMiddleware()) + //app.Use(middlewares.AuthorizationMiddleware()) + anonymousGroup := app.Group("/") + { + anonymousGroup.POST("/login", routes.Login) // 用户登录 + anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 + + } + authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) + { + // 路由 + // 节点 + authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + // 爬虫 + authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + authGroup.POST("/spiders", routes.PutSpider) // 上传爬虫 + authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫目录写入 + authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + // 任务 + authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 + authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 + authGroup.PUT("/tasks", routes.PutTask) // 派发任务 + authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 + authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 + authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 + authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 + authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 + // 定时任务 + authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 + authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 + authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 + authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 + authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 + // 统计数据 + authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 + // 用户 + authGroup.GET("/users", routes.GetUserList) // 用户列表 + authGroup.GET("/users/:id", routes.GetUser) // 用户详情 + authGroup.POST("/users/:id", routes.PostUser) // 更改用户 + authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + authGroup.GET("/me", routes.GetMe) // 获取自己账户 + } - // 路由 - // 节点 - app.GET("/nodes", routes.GetNodeList) // 节点列表 - app.GET("/nodes/:id", routes.GetNode) // 节点详情 - app.POST("/nodes/:id", routes.PostNode) // 修改节点 - app.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - app.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - app.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 - // 爬虫 - app.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - app.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - app.POST("/spiders", routes.PutSpider) // 上传爬虫 - app.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - app.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - app.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - app.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - app.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - app.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫目录写入 - app.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - app.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - // 任务 - app.GET("/tasks", routes.GetTaskList) // 任务列表 - app.GET("/tasks/:id", routes.GetTask) // 任务详情 - app.PUT("/tasks", routes.PutTask) // 派发任务 - app.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 - app.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 - app.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 - app.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 - app.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 - // 定时任务 - app.GET("/schedules", routes.GetScheduleList) // 定时任务列表 - app.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 - app.PUT("/schedules", routes.PutSchedule) // 创建定时任务 - app.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 - app.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 - // 统计数据 - app.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 - // 用户 - app.GET("/users", routes.GetUserList) // 用户列表 - app.GET("/users/:id", routes.GetUser) // 用户详情 - app.PUT("/users", routes.PutUser) // 添加用户 - app.POST("/users/:id", routes.PostUser) // 更改用户 - app.DELETE("/users/:id", routes.DeleteUser) // 删除用户 - app.POST("/login", routes.Login) // 用户登录 - app.GET("/me", routes.GetMe) // 获取自己账户 } // 路由ping diff --git a/backend/middlewares/auth.go b/backend/middlewares/auth.go index 977fea78..5298beea 100644 --- a/backend/middlewares/auth.go +++ b/backend/middlewares/auth.go @@ -12,12 +12,12 @@ import ( func AuthorizationMiddleware() gin.HandlerFunc { return func(c *gin.Context) { // 如果为登录或注册，不用校验 - if c.Request.URL.Path == "/login" || - (c.Request.URL.Path == "/users" && c.Request.Method == "PUT") || - strings.HasSuffix(c.Request.URL.Path, "download") { - c.Next() - return - } + //if c.Request.URL.Path == "/login" || + // (c.Request.URL.Path == "/users" && c.Request.Method == "PUT") || + // strings.HasSuffix(c.Request.URL.Path, "download") { + // c.Next() + // return + //} // 获取token string tokenStr := c.GetHeader("Authorization") diff --git a/frontend/src/api/request.js b/frontend/src/api/request.js index 38734c46..5b612719 100644 --- a/frontend/src/api/request.js +++ b/frontend/src/api/request.js @@ -3,7 +3,7 @@ import router from '../router' let baseUrl = process.env.VUE_APP_BASE_URL ? process.env.VUE_APP_BASE_URL : 'http://localhost:8000' -const request = (method, path, params, data) => { +const request = (method, path, params, data, others = {}) => { return new Promise((resolve, reject) => { const url = baseUrl + path const headers = { @@ -14,7 +14,8 @@ const request = (method, path, params, data) => { url, params, data, - headers + headers, + ...others }) .then(resolve) .catch(error => { diff --git a/frontend/src/store/modules/task.js b/frontend/src/store/modules/task.js index 545a169b..1d7e6c09 100644 --- a/frontend/src/store/modules/task.js +++ b/frontend/src/store/modules/task.js @@ -120,6 +120,22 @@ const actions = { commit('SET_TASK_RESULTS_TOTAL_COUNT', response.data.total) }) }, + async getTaskResultExcel ({ state, commit }, id) { + const { data } = await request.request('GET', '/tasks/' + id + '/results/download', {}, { + responseType: 'blob' // important + }) + const downloadUrl = window.URL.createObjectURL(new Blob([data])) + + const link = document.createElement('a') + + link.href = downloadUrl + + link.setAttribute('download', 'data.csv') // any other extension + + document.body.appendChild(link) + link.click() + link.remove() + }, cancelTask ({ state, dispatch }, id) { return request.post(`/tasks/${id}/cancel`) .then(() => { diff --git a/frontend/src/views/task/TaskDetail.vue b/frontend/src/views/task/TaskDetail.vue index a1edb497..2361c0c2 100644 --- a/frontend/src/views/task/TaskDetail.vue +++ b/frontend/src/views/task/TaskDetail.vue @@ -95,7 +95,7 @@ export default { this.$store.dispatch('task/getTaskResults', this.$route.params.id) }, downloadCSV () { - window.location.href = this.$request.baseUrl + '/tasks/' + this.$route.params.id + '/results/download' + this.$store.dispatch('task/getTaskResultExcel', this.$route.params.id) this.$st.sendEv('任务详情-结果', '下载CSV') } }, diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index e15aa661..9efcaa92 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -45,7 +45,7 @@ :label="$t(col.label)" :sortable="col.sortable" :align="col.align" - :width="col.width"> + > From 1877677844fa2e165fddb7215dd5b0c55d4b0d38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 15:03:57 +0800 Subject: [PATCH 12/19] =?UTF-8?q?fix=20=E6=97=A0=E6=B3=95=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=97=A5=E5=BF=97=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/log.go | 2 +- backend/services/node.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/services/log.go b/backend/services/log.go index a248c176..a0ba0311 100644 --- a/backend/services/log.go +++ b/backend/services/log.go @@ -41,7 +41,7 @@ func GetLocalLog(logPath string) (fileBytes []byte, err error) { } n, err := f.ReadAt(logBuf, off) // 到文件结尾会有EOF的报错 - if err.Error() != "EOF" && err != nil { + if err != nil && err.Error() != "EOF" { log.Error(err.Error()) debug.PrintStack() return nil, err diff --git a/backend/services/node.go b/backend/services/node.go index 977ff0ef..124f5bba 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -308,8 +308,10 @@ func WorkerNodeCallback(channel string, msgStr string) { log.Errorf(err.Error()) debug.PrintStack() msgSd.Error = err.Error() + msgSd.Log = err.Error() + } else { + msgSd.Log = string(logStr) } - msgSd.Log = string(logStr) // 序列化 msgSdBytes, err := json.Marshal(&msgSd) From c0201f566ad9c14f644cab49e144340afc1ed211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 15:59:43 +0800 Subject: [PATCH 13/19] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/model/spider.go | 6 ++++-- backend/services/node.go | 3 ++- frontend/src/views/task/TaskList.vue | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/model/spider.go b/backend/model/spider.go index 65782fe8..d44f651b 100644 --- a/backend/model/spider.go +++ b/backend/model/spider.go @@ -23,13 +23,14 @@ type Spider struct { Col string `json:"col"` // 结果储存位置 Site string `json:"site"` // 爬虫网站 Envs []Env `json:"envs" bson:"envs"` // 环境变量 - + Remark string `json:"remark"` // 备注 // 自定义爬虫 Src string `json:"src" bson:"src"` // 源码位置 Cmd string `json:"cmd" bson:"cmd"` // 执行命令 // 前端展示 - LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间 + LastRunTs time.Time `json:"last_run_ts"` // 最后一次执行时间 + LastStatus string `json:"last_status"` // 最后执行状态 // TODO: 可配置爬虫 //Fields []interface{} `json:"fields"` @@ -115,6 +116,7 @@ func GetSpiderList(filter interface{}, skip int, limit int) ([]Spider, error) { // 赋值 spiders[i].LastRunTs = task.CreateTs + spiders[i].LastStatus = task.Status } return spiders, nil diff --git a/backend/services/node.go b/backend/services/node.go index 124f5bba..9685a1bb 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -304,6 +304,7 @@ func WorkerNodeCallback(channel string, msgStr string) { // 获取本地日志 logStr, err := GetLocalLog(msg.LogPath) + log.Info(string(logStr)) if err != nil { log.Errorf(err.Error()) debug.PrintStack() @@ -322,7 +323,7 @@ func WorkerNodeCallback(channel string, msgStr string) { } // 发布消息给主节点 - fmt.Println(msgSd) + log.Info("publish get log msg to master") if err := database.Publish("nodes:master", string(msgSdBytes)); err != nil { log.Errorf(err.Error()) return diff --git a/frontend/src/views/task/TaskList.vue b/frontend/src/views/task/TaskList.vue index 9cbceb20..a3ffbeea 100644 --- a/frontend/src/views/task/TaskList.vue +++ b/frontend/src/views/task/TaskList.vue @@ -119,7 +119,7 @@ :width="col.width"> - + + + + Date: Sat, 31 Aug 2019 16:27:01 +0800 Subject: [PATCH 16/19] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E7=8E=AF=E5=A2=83=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/package.json b/frontend/package.json index 139297d3..e3bc84f8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,6 +6,7 @@ "serve": "vue-cli-service serve --ip=0.0.0.0", "serve:prod": "vue-cli-service serve --mode=production --ip=0.0.0.0", "config": "vue ui", + "build:dev": "vue-cli-service build --mode development", "build:prod": "vue-cli-service build --mode production", "lint": "vue-cli-service lint", "test:unit": "vue-cli-service test:unit" From 1a383dac1730ab6d761f974f5705275e45af82cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 17:04:49 +0800 Subject: [PATCH 17/19] =?UTF-8?q?fix=20=E7=88=AC=E8=99=AB=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E6=97=A0=E6=B3=95=E6=89=93=E5=BC=80=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 2 +- backend/services/task.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index a3242849..f526e11d 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -42,7 +42,7 @@ func GetSpidersFromDir() ([]model.Spider, error) { if !utils.Exists(srcPath) { mask := syscall.Umask(0) // 改为 0000 八进制 defer syscall.Umask(mask) // 改为原来的 umask - if err := os.MkdirAll(srcPath, 0666); err != nil { + if err := os.MkdirAll(srcPath, 0766); err != nil { debug.PrintStack() return []model.Spider{}, err } diff --git a/backend/services/task.go b/backend/services/task.go index 8c3c3407..1b0a5676 100644 --- a/backend/services/task.go +++ b/backend/services/task.go @@ -408,9 +408,12 @@ func GetTaskLog(id string) (logStr string, err error) { logStr = string(logBytes) if err != nil { log.Errorf(err.Error()) - return "", err + logStr = string(err.Error()) + // return "", err + } else { + logStr = string(logBytes) } - logStr = string(logBytes) + } else { // 若不为主节点，获取远端日志 logStr, err = GetRemoteLog(task) From e027aeb71fe7efd8aa94bf0228df067a904ec93e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=99=AF=E9=98=B3?= <1656488874@qq.com> Date: Sat, 31 Aug 2019 17:56:42 +0800 Subject: [PATCH 18/19] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/spider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/services/spider.go b/backend/services/spider.go index f526e11d..c3f63139 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -31,6 +31,7 @@ type SpiderFileData struct { type SpiderUploadMessage struct { FileId string FileName string + SpiderId string } // 从项目目录中获取爬虫列表 @@ -295,6 +296,7 @@ func PublishSpider(spider model.Spider) (err error) { msg := SpiderUploadMessage{ FileId: fid.Hex(), FileName: fileName, + SpiderId: spider.Id.Hex(), } msgStr, err := json.Marshal(msg) if err != nil { @@ -326,7 +328,7 @@ func OnFileUpload(channel string, msgStr string) { // 从GridFS获取该文件 f, err := gf.OpenId(bson.ObjectIdHex(msg.FileId)) if err != nil { - log.Errorf("open file id: " + msg.FileId + ", error: " + err.Error()) + log.Errorf("open file id: " + msg.FileId + ", spider id:" + msg.SpiderId + ", error: " + err.Error()) debug.PrintStack() return } From 9f43e08ff95bda2ca0e329bd5964509fdab114e1 Mon Sep 17 00:00:00 2001 From: yaziming Date: Sat, 31 Aug 2019 21:22:47 +0800 Subject: [PATCH 19/19] =?UTF-8?q?Backend:=20=20=20=20improve=20=20=20=20?= =?UTF-8?q?=20=20-=20AuthMiddleware=20=E6=B3=A8=E5=85=A5=E5=BD=93=E5=89=8D?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=9A=84=E4=BF=A1=E6=81=AF=20=20=20=20=20=20?= =?UTF-8?q?-=20=E5=A2=9E=E5=8A=A0Context=E6=9C=8D=E5=8A=A1=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=BF=AB=E6=8D=B7=E8=8E=B7=E5=8F=96=E5=BD=93=E5=89=8D?= =?UTF-8?q?=E7=99=BB=E5=BD=95=E8=80=85=E4=BF=A1=E6=81=AF=20=20=20=20=20=20?= =?UTF-8?q?-=20=E9=87=8D=E6=9E=84Login/GetMe=E6=8E=A5=E5=8F=A3=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E9=81=BF=E5=85=8D=E9=87=8D=E5=A4=8D=E7=9A=84=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E6=9F=A5=E8=AF=A2=20=20=20=20=20=20-=20?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=8C=96error=E4=BF=A1=E6=81=AF=E5=A3=B0?= =?UTF-8?q?=E6=98=8E(=E5=90=91=E4=B8=8B=E5=85=BC=E5=AE=B9,=E6=97=A7?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=8F=AF=E9=80=90=E6=B8=90=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=8C=96)=20=20=20=20=20=20-=20=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E9=83=A8=E5=88=86=E4=B8=8D=E7=AC=A6=E5=90=88=E8=A7=84?= =?UTF-8?q?=E8=8C=83=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/constants/context.go | 5 ++ backend/constants/errors.go | 8 ++++ backend/errors/errors.go | 43 +++++++++++++++++ backend/middlewares/auth.go | 1 + backend/mock/schedule.go | 2 +- backend/routes/user.go | 27 +++++------ backend/services/context/context.go | 73 +++++++++++++++++++++++++++++ backend/services/spider.go | 3 +- backend/services/user.go | 40 +++++++++------- 9 files changed, 168 insertions(+), 34 deletions(-) create mode 100644 backend/constants/context.go create mode 100644 backend/constants/errors.go create mode 100644 backend/errors/errors.go create mode 100644 backend/services/context/context.go diff --git a/backend/constants/context.go b/backend/constants/context.go new file mode 100644 index 00000000..0759b54b --- /dev/null +++ b/backend/constants/context.go @@ -0,0 +1,5 @@ +package constants + +const ( + ContextUser = "currentUser" +) diff --git a/backend/constants/errors.go b/backend/constants/errors.go new file mode 100644 index 00000000..a6175319 --- /dev/null +++ b/backend/constants/errors.go @@ -0,0 +1,8 @@ +package constants + +import "crawlab/errors" + +var ( + //users + ErrorUserNotFound = errors.NewBusinessError(10001, "user not found.") +) diff --git a/backend/errors/errors.go b/backend/errors/errors.go new file mode 100644 index 00000000..0110808b --- /dev/null +++ b/backend/errors/errors.go @@ -0,0 +1,43 @@ +package errors + +import "fmt" + +type Scope int + +const ( + ScopeSystem Scope = 1 + ScopeBusiness Scope = 2 +) + +type OPError struct { + Message string + Code int + Scope Scope +} + +func (O OPError) Error() string { + var scope string + switch O.Scope { + case ScopeSystem: + scope = "system" + break + case ScopeBusiness: + scope = "business" + } + return fmt.Sprintf("%s : %d -> %s.", scope, O.Code, O.Message) +} + +func NewSystemOPError(code int, message string) *OPError { + return &OPError{ + Message: message, + Code: code, + Scope: ScopeSystem, + } +} +func NewBusinessError(code int, message string) *OPError { + return &OPError{ + Message: message, + Code: code, + Scope: ScopeBusiness, + } +} diff --git a/backend/middlewares/auth.go b/backend/middlewares/auth.go index 5298beea..07249e82 100644 --- a/backend/middlewares/auth.go +++ b/backend/middlewares/auth.go @@ -46,6 +46,7 @@ func AuthorizationMiddleware() gin.HandlerFunc { return } } + c.Set(constants.ContextUser, &user) // 校验成功 c.Next() diff --git a/backend/mock/schedule.go b/backend/mock/schedule.go index ae982ca6..702e8754 100644 --- a/backend/mock/schedule.go +++ b/backend/mock/schedule.go @@ -113,7 +113,7 @@ func PutSchedule(c *gin.Context) { func DeleteSchedule(c *gin.Context) { id := bson.ObjectIdHex("5d429e6c19f7abede924fee2") for _, sch := range scheduleList { - if sch.Id == bson.ObjectId(id) { + if sch.Id == id { fmt.Println("delete a schedule") } } diff --git a/backend/routes/user.go b/backend/routes/user.go index a3d5a431..a6d44cae 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -4,6 +4,7 @@ import ( "crawlab/constants" "crawlab/model" "crawlab/services" + "crawlab/services/context" "crawlab/utils" "github.com/gin-gonic/gin" "github.com/globalsign/mgo/bson" @@ -171,7 +172,7 @@ func Login(c *gin.Context) { } // 获取token - tokenStr, err := services.GetToken(user.Username) + tokenStr, err := services.MakeToken(&user) if err != nil { HandleError(http.StatusUnauthorized, c, errors.New("not authorized")) return @@ -185,20 +186,16 @@ func Login(c *gin.Context) { } func GetMe(c *gin.Context) { - // 获取token string - tokenStr := c.GetHeader("Authorization") - - // 校验token - user, err := services.CheckToken(tokenStr) - if err != nil { - HandleError(http.StatusUnauthorized, c, errors.New("not authorized")) + ctx := context.WithGinContext(c) + user := ctx.User() + if user == nil { + ctx.FailedWithError(constants.ErrorUserNotFound, http.StatusUnauthorized) return } - user.Password = "" - - c.JSON(http.StatusOK, Response{ - Status: "ok", - Message: "success", - Data: user, - }) + ctx.Success(struct { + *model.User + Password string `json:"password,omitempty"` + }{ + User: user, + }, nil) } diff --git a/backend/services/context/context.go b/backend/services/context/context.go new file mode 100644 index 00000000..d5d2b6ad --- /dev/null +++ b/backend/services/context/context.go @@ -0,0 +1,73 @@ +package context + +import ( + "crawlab/constants" + "crawlab/errors" + "crawlab/model" + "fmt" + "github.com/apex/log" + "github.com/gin-gonic/gin" + errors2 "github.com/pkg/errors" + "net/http" + "runtime/debug" +) + +type Context struct { + *gin.Context +} + +func (c *Context) User() *model.User { + userIfe, exists := c.Get(constants.ContextUser) + if !exists { + return nil + } + user, ok := userIfe.(*model.User) + if !ok { + return nil + } + return user +} +func (c *Context) Success(data interface{}, meta interface{}) { + if meta == nil { + meta = gin.H{} + } + if data == nil { + data = gin.H{} + } + c.JSON(http.StatusOK, gin.H{ + "status": "ok", + "message": "success", + "data": data, + "error": "", + }) +} +func (c *Context) FailedWithError(err error, httpCode ...int) { + + var code = 200 + if len(httpCode) > 0 { + code = httpCode[0] + } + log.Errorf("handle error:" + err.Error()) + debug.PrintStack() + switch errors2.Cause(err).(type) { + case errors.OPError: + c.AbortWithStatusJSON(code, gin.H{ + "status": "ok", + "message": "error", + "error": err.Error(), + }) + break + default: + fmt.Println("deprecated....") + c.AbortWithStatusJSON(code, gin.H{ + "status": "ok", + "message": "error", + "error": err.Error(), + }) + } + +} + +func WithGinContext(context *gin.Context) *Context { + return &Context{Context: context} +} diff --git a/backend/services/spider.go b/backend/services/spider.go index c3f63139..47c1fa33 100644 --- a/backend/services/spider.go +++ b/backend/services/spider.go @@ -230,7 +230,7 @@ func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCre for { switch nr, err := f.Read(s[:]); true { case nr < 0: - fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error()) + _, _ = fmt.Fprintf(os.Stderr, "cat: error reading: %s\n", err.Error()) debug.PrintStack() case nr == 0: // EOF return nil @@ -238,7 +238,6 @@ func ReadFileByStep(filePath string, handle func([]byte, *mgo.GridFile), fileCre handle(s[0:nr], fileCreate) } } - return nil } // 发布所有爬虫 diff --git a/backend/services/user.go b/backend/services/user.go index fb688fd1..4811f767 100644 --- a/backend/services/user.go +++ b/backend/services/user.go @@ -5,11 +5,9 @@ import ( "crawlab/model" "crawlab/utils" "errors" - "github.com/apex/log" "github.com/dgrijalva/jwt-go" "github.com/globalsign/mgo/bson" "github.com/spf13/viper" - "runtime/debug" "time" ) @@ -24,28 +22,38 @@ func InitUserService() error { } return nil } - -func GetToken(username string) (tokenStr string, err error) { - user, err := model.GetUserByUsername(username) - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return - } - +func MakeToken(user *model.User) (tokenStr string, err error) { token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ "id": user.Id, "username": user.Username, "nbf": time.Now().Unix(), }) - tokenStr, err = token.SignedString([]byte(viper.GetString("server.secret"))) - if err != nil { - return - } - return + return token.SignedString([]byte(viper.GetString("server.secret"))) + } +//func GetToken(username string) (tokenStr string, err error) { +// user, err := model.GetUserByUsername(username) +// if err != nil { +// log.Errorf(err.Error()) +// debug.PrintStack() +// return +// } +// +// token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ +// "id": user.Id, +// "username": user.Username, +// "nbf": time.Now().Unix(), +// }) +// +// tokenStr, err = token.SignedString([]byte(viper.GetString("server.secret"))) +// if err != nil { +// return +// } +// return +//} + func SecretFunc() jwt.Keyfunc { return func(token *jwt.Token) (interface{}, error) { return []byte(viper.GetString("server.secret")), nil