From c02c58fd2d9521589dea82fe25c5e37481e58f80 Mon Sep 17 00:00:00 2001 From: yaziming Date: Tue, 26 May 2020 16:15:05 +0800 Subject: [PATCH 1/6] local node --- backend/go.mod | 3 +- backend/go.sum | 24 +-- backend/main.go | 8 +- backend/model/node.go | 39 ---- backend/services/local_node/local_node.go | 79 ++----- backend/services/local_node/mongo_info.go | 62 ++++++ backend/services/local_node/node_info.go | 74 +++++++ backend/services/node.go | 52 +---- backend/services/register/register.go | 243 ---------------------- backend/services/rpc/base.go | 6 +- 10 files changed, 176 insertions(+), 414 deletions(-) create mode 100644 backend/services/local_node/mongo_info.go create mode 100644 backend/services/local_node/node_info.go delete mode 100644 backend/services/register/register.go diff --git a/backend/go.mod b/backend/go.mod index de0c468e..3f1fda3a 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -9,7 +9,6 @@ require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/aokoli/goutils v1.0.1 // indirect github.com/apex/log v1.1.4 - github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect github.com/cenkalti/backoff/v4 v4.0.2 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/fsnotify/fsnotify v1.4.9 @@ -17,6 +16,7 @@ require ( github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-playground/validator/v10 v10.3.0 github.com/gomodule/redigo v2.0.0+incompatible + github.com/hashicorp/go-sockaddr v1.0.0 github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/imroc/req v0.3.0 @@ -32,7 +32,6 @@ require ( github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/swaggo/gin-swagger v1.2.0 github.com/swaggo/swag v1.6.6 - github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect go.uber.org/atomic v1.6.0 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/backend/go.sum b/backend/go.sum index c2ce0d11..c53a2aaf 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -40,15 +40,12 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= -github.com/apex/log v1.1.1 h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA= -github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA= github.com/apex/log v1.1.4 h1:3Zk+boorIQAAGBrHn0JUtAau4ihMamT4WdnfdnXM1zQ= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -66,9 +63,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -178,7 +173,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -191,6 +185,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -209,8 +204,6 @@ github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0= -github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= @@ -246,8 +239,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -294,8 +285,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olivere/elastic/v7 v7.0.14 h1:89dYPg6kD3WJx42ZtO4U6WDIzRy69FvQqz/yRiwekuM= -github.com/olivere/elastic/v7 v7.0.14/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -339,8 +328,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= @@ -354,8 +341,6 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= @@ -401,7 +386,6 @@ github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -436,6 +420,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -455,7 +440,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -556,8 +540,6 @@ google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBr google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= @@ -573,8 +555,6 @@ gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXa gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 h1:NvePS/smRcFQ4bMtTddFtknbGCtoBkJxGmpSpVRafCc= -gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= diff --git a/backend/main.go b/backend/main.go index 64d640d5..4a801b44 100644 --- a/backend/main.go +++ b/backend/main.go @@ -11,6 +11,7 @@ import ( "crawlab/routes" "crawlab/services" "crawlab/services/challenge" + "crawlab/services/local_node" "crawlab/services/rpc" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -72,7 +73,12 @@ func main() { log.Error("init log error:" + err.Error()) panic(err) } - log.Info("initialized log successfully") + log.Info("initialized log successfully") // 初始化日志设置 + if err := local_node.InitLocalNodeInfo(); err != nil { + log.Error("init local node error:" + err.Error()) + panic(err) + } + log.Info("initialized local node successfully") if model.IsMaster() { // 初始化Master节点信息 diff --git a/backend/model/node.go b/backend/model/node.go index 3bdbb5d1..2b7f193a 100644 --- a/backend/model/node.go +++ b/backend/model/node.go @@ -3,7 +3,6 @@ package model import ( "crawlab/constants" "crawlab/database" - "crawlab/services/register" "errors" "github.com/apex/log" "github.com/globalsign/mgo" @@ -42,16 +41,6 @@ func IsMaster() bool { return viper.GetString("server.master") == Yes } -// 获取本机节点 -func GetCurrentNode() (Node, error) { - // 获得注册的key值 - key, err := register.GetRegister().GetKey() - if err != nil { - return Node{}, err - } - return GetNodeByKey(key) -} - func (n *Node) Save() error { s, c := database.GetCol("nodes") defer s.Close() @@ -190,34 +179,6 @@ func GetNodeCount(query interface{}) (int, error) { return count, nil } -// 节点基本信息 -func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) { - ip, err := register.GetRegister().GetIp() - if err != nil { - debug.PrintStack() - return "", "", "", "", err - } - - mac, err = register.GetRegister().GetMac() - if err != nil { - debug.PrintStack() - return "", "", "", "", err - } - - hostname, err = register.GetRegister().GetHostname() - if err != nil { - debug.PrintStack() - return "", "", "", "", err - } - - key, err = register.GetRegister().GetKey() - if err != nil { - debug.PrintStack() - return "", "", "", "", err - } - return ip, mac, hostname, key, nil -} - // 根据redis的key值，重置node节点为offline func ResetNodeStatusToOffline(list []string) { nodes, _ := GetNodeList(nil) diff --git a/backend/services/local_node/local_node.go b/backend/services/local_node/local_node.go index 849853de..109f6534 100644 --- a/backend/services/local_node/local_node.go +++ b/backend/services/local_node/local_node.go @@ -2,71 +2,24 @@ package local_node import ( "crawlab/model" - "github.com/apex/log" - "github.com/cenkalti/backoff/v4" - "go.uber.org/atomic" - "sync" - "time" + "github.com/spf13/viper" ) -var localNode *LocalNode -var locker atomic.Int32 -var once sync.Once - -type LocalNode struct { - node *model.Node - sync.RWMutex -} - -func (n *LocalNode) load(retry bool) (err error) { - n.Lock() - defer n.Unlock() - var node model.Node - if retry { - b := backoff.NewConstantBackOff(1 * time.Second) - err = backoff.Retry(func() error { - node, err = model.GetCurrentNode() - if err != nil { - log.WithError(err).Warnf("Get current node info from database failed. Will after %f seconds, try again.", b.NextBackOff().Seconds()) - } - return err - }, b) - } else { - node, err = model.GetCurrentNode() - } - - if err != nil { - return - } - n.node = &node - return nil -} -func (n *LocalNode) watch() { - timer := time.NewTicker(time.Second * 5) - for range timer.C { - if locker.CAS(0, 1) { - - err := n.load(false) - - if err != nil { - log.WithError(err).Errorf("load current node from database failed,") - } - locker.Store(0) - } - continue - } -} - -func (n *LocalNode) Current() *model.Node { - n.RLock() - defer n.RUnlock() - return n.node +func GetLocalNode() *LocalNode { + return localNode } func CurrentNode() *model.Node { - once.Do(func() { - localNode = &LocalNode{} - _ = localNode.load(true) - go localNode.watch() - }) - return localNode.Current() + return GetLocalNode().Current() +} + +func InitLocalNodeInfo() (err error) { + registerType := viper.GetString("server.register.type") + ip := viper.GetString("server.register.ip") + customNodeName := viper.GetString("server.register.customNodeName") + + localNode, err = NewLocalNode(ip, customNodeName, registerType) + if err != nil { + return err + } + return localNode.Ready() } diff --git a/backend/services/local_node/mongo_info.go b/backend/services/local_node/mongo_info.go new file mode 100644 index 00000000..d8d4df06 --- /dev/null +++ b/backend/services/local_node/mongo_info.go @@ -0,0 +1,62 @@ +package local_node + +import ( + "crawlab/model" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "go.uber.org/atomic" + "sync" + "time" +) + +var locker atomic.Int32 + +type mongo struct { + node *model.Node + sync.RWMutex +} + +func (n *mongo) load(retry bool) (err error) { + n.Lock() + defer n.Unlock() + var node model.Node + if retry { + b := backoff.NewConstantBackOff(1 * time.Second) + err = backoff.Retry(func() error { + node, err = model.GetNodeByKey(GetLocalNode().Identify) + if err != nil { + log.WithError(err).Warnf("Get current node info from database failed. Will after %f seconds, try again.", b.NextBackOff().Seconds()) + } + return err + }, b) + } else { + node, err = model.GetNodeByKey(localNode.Identify) + } + + if err != nil { + return + } + n.node = &node + return nil +} +func (n *mongo) watch() { + timer := time.NewTicker(time.Second * 5) + for range timer.C { + if locker.CAS(0, 1) { + + err := n.load(false) + + if err != nil { + log.WithError(err).Errorf("load current node from database failed") + } + locker.Store(0) + } + continue + } +} + +func (n *mongo) Current() *model.Node { + n.RLock() + defer n.RUnlock() + return n.node +} diff --git a/backend/services/local_node/node_info.go b/backend/services/local_node/node_info.go new file mode 100644 index 00000000..d037da6d --- /dev/null +++ b/backend/services/local_node/node_info.go @@ -0,0 +1,74 @@ +package local_node + +import ( + "errors" + "github.com/hashicorp/go-sockaddr" + "os" +) + +var localNode *LocalNode + +type IdentifyType string + +const ( + Ip = IdentifyType("ip") + Mac = IdentifyType("mac") + Hostname = IdentifyType("hostname") +) + +type local struct { + Ip string + Mac string + Hostname string + Identify string + IdentifyType IdentifyType +} +type LocalNode struct { + local + mongo +} + +func (l *LocalNode) Ready() error { + err := localNode.load(true) + if err != nil { + return err + } + go localNode.watch() + return nil +} + +func NewLocalNode(ip string, identify string, identifyTypeString string) (node *LocalNode, err error) { + addrs, err := sockaddr.GetPrivateInterfaces() + if ip == "" { + if err != nil { + return node, err + } + if len(addrs) == 0 { + return node, errors.New("address not found") + } + ipaddr := *sockaddr.ToIPAddr(addrs[0].SockAddr) + ip = ipaddr.NetIP().String() + } + + mac := addrs[0].HardwareAddr.String() + hostname, err := os.Hostname() + if err != nil { + return node, err + } + local := local{Ip: ip, Mac: mac, Hostname: hostname} + switch IdentifyType(identifyTypeString) { + case Ip: + local.Identify = local.Ip + local.IdentifyType = Ip + case Mac: + local.Identify = local.Mac + local.IdentifyType = Mac + case Hostname: + local.Identify = local.Hostname + local.IdentifyType = Hostname + default: + local.Identify = identify + local.IdentifyType = IdentifyType(identifyTypeString) + } + return &LocalNode{local: local, mongo: mongo{}}, nil +} diff --git a/backend/services/node.go b/backend/services/node.go index 9257f007..2b094e98 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -8,7 +8,6 @@ import ( "crawlab/model" "crawlab/services/local_node" "crawlab/services/msg_handler" - "crawlab/services/register" "crawlab/utils" "encoding/json" "fmt" @@ -41,9 +40,10 @@ func IsMasterNode(id string) bool { // 获取节点数据 func GetNodeData() (Data, error) { - key, err := register.GetRegister().GetKey() + localNode := local_node.GetLocalNode() + key := localNode.Identify if key == "" { - return Data{}, err + return Data{}, nil } value, err := database.RedisClient.HGet("nodes", key) @@ -162,41 +162,14 @@ func UpdateNodeInfo(data *Data) (err error) { // 更新节点数据 func UpdateNodeData() { - // 获取MAC地址 - mac, err := register.GetRegister().GetMac() - if err != nil { - log.Errorf(err.Error()) - return - } - - // 获取IP地址 - ip, err := register.GetRegister().GetIp() - if err != nil { - log.Errorf(err.Error()) - return - } - - // 获取Hostname - hostname, err := register.GetRegister().GetHostname() - if err != nil { - log.Errorf(err.Error()) - return - } - - // 获取redis的key - key, err := register.GetRegister().GetKey() - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return - } - + localNode := local_node.GetLocalNode() + key := localNode.Identify // 构造节点数据 data := Data{ Key: key, - Mac: mac, - Ip: ip, - Hostname: hostname, + Mac: localNode.Mac, + Ip: localNode.Ip, + Hostname: localNode.Hostname, Master: model.IsMaster(), UpdateTs: time.Now(), UpdateTsUnix: time.Now().Unix(), @@ -312,11 +285,6 @@ func InitNodeService() error { return nil } func InitMasterNodeInfo() (err error) { - // 获取本机信息 - ip, mac, hostname, key, err := model.GetNodeBaseInfo() - if err != nil { - debug.PrintStack() - return err - } - return model.UpdateMasterNodeInfo(key, ip, mac, hostname) + localNode := local_node.GetLocalNode() + return model.UpdateMasterNodeInfo(localNode.Identify, localNode.Ip, localNode.Mac, localNode.Hostname) } diff --git a/backend/services/register/register.go b/backend/services/register/register.go deleted file mode 100644 index 9eedf0e7..00000000 --- a/backend/services/register/register.go +++ /dev/null @@ -1,243 +0,0 @@ -package register - -import ( - "bytes" - "crawlab/constants" - "fmt" - "github.com/apex/log" - "github.com/spf13/viper" - "net" - "os/exec" - "reflect" - "runtime/debug" - "strings" - "sync" -) - -type Register interface { - // 注册的key类型 - GetType() string - // 注册的key的值，唯一标识节点 - GetKey() (string, error) - // 注册的节点IP - GetIp() (string, error) - // 注册节点的mac地址 - GetMac() (string, error) - // 注册节点的Hostname - GetHostname() (string, error) - GetCustomName() (string, error) -} - -// ===================== mac 地址注册 ===================== -type MacRegister struct{} - -func (mac *MacRegister) GetType() string { - return "mac" -} - -func (mac *MacRegister) GetKey() (string, error) { - return mac.GetMac() -} - -func (mac *MacRegister) GetMac() (string, error) { - return getMac() -} - -func (mac *MacRegister) GetIp() (string, error) { - return getIp() -} - -func (mac *MacRegister) GetHostname() (string, error) { - return getHostname() -} - -func (mac *MacRegister) GetCustomName() (string, error) { - return getMac() -} - -// ===================== ip 地址注册 ===================== -type IpRegister struct { - Ip string -} - -func (ip *IpRegister) GetCustomName() (string, error) { - return ip.Ip, nil -} - -// ============= 自定义节点名称注册 ============== -type CustomNameRegister struct { - CustomName string -} - -func (c *CustomNameRegister) GetType() string { - return "customName" -} - -func (c *CustomNameRegister) GetIp() (string, error) { - return getIp() -} - -func (c *CustomNameRegister) GetMac() (string, error) { - return getMac() -} - -func (c *CustomNameRegister) GetKey() (string, error) { - return c.CustomName, nil -} - -func (c *CustomNameRegister) GetHostname() (string, error) { - - return getHostname() -} - -func (c *CustomNameRegister) GetCustomName() (string, error) { - return c.CustomName, nil -} - -// ============================================================ -func (ip *IpRegister) GetType() string { - return "ip" -} - -func (ip *IpRegister) GetKey() (string, error) { - return ip.Ip, nil -} - -func (ip *IpRegister) GetIp() (string, error) { - return ip.Ip, nil -} - -func (ip *IpRegister) GetMac() (string, error) { - return getMac() -} - -func (ip *IpRegister) GetHostname() (string, error) { - return getHostname() -} - -// ===================== mac 地址注册 ===================== -type HostnameRegister struct{} - -func (h *HostnameRegister) GetType() string { - return "mac" -} - -func (h *HostnameRegister) GetKey() (string, error) { - return h.GetHostname() -} - -func (h *HostnameRegister) GetMac() (string, error) { - return getMac() -} - -func (h *HostnameRegister) GetIp() (string, error) { - return getIp() -} - -func (h *HostnameRegister) GetHostname() (string, error) { - return getHostname() -} - -func (h *HostnameRegister) GetCustomName() (string, error) { - return getHostname() -} - -// ===================== 公共方法 ===================== -// 获取本机的IP地址 -// TODO: 考虑多个IP地址的情况 -func getIp() (string, error) { - addrList, err := net.InterfaceAddrs() - if err != nil { - return "", err - } - for _, value := range addrList { - if ipNet, ok := value.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { - if ipNet.IP.To4() != nil { - return ipNet.IP.String(), nil - } - } - } - return "", nil -} - -func getMac() (string, error) { - interfaces, err := net.Interfaces() - if err != nil { - log.Errorf("get interfaces error:" + err.Error()) - debug.PrintStack() - return "", err - } - for _, inter := range interfaces { - if inter.HardwareAddr != nil { - mac := inter.HardwareAddr.String() - return mac, nil - } - } - return "", nil -} - -func getHostname() (string, error) { - var stdout bytes.Buffer - var stderr bytes.Buffer - - cmd := exec.Command("hostname") - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - log.Errorf(err.Error()) - log.Errorf(fmt.Sprintf("error: %s", stderr.String())) - debug.PrintStack() - return "", err - } - - return strings.Replace(stdout.String(), "\n", "", -1), nil -} - -// ===================== 获得注册简单工厂 ===================== -var register Register - -// 获得注册器 -var once sync.Once - -func GetRegister() Register { - once.Do(func() { - registerType := viper.GetString("server.register.type") - - switch registerType { - case constants.RegisterTypeMac: - - register = &MacRegister{} - - case constants.RegisterTypeIp: - - ip := viper.GetString("server.register.ip") - if ip == "" { - log.Error("server.register.ip is empty") - debug.PrintStack() - register = nil - } - register = &IpRegister{ - Ip: ip, - } - - case constants.RegisterTypeHostname: - - register = &HostnameRegister{} - - case constants.RegisterTypeCustomName: - - customNodeName := viper.GetString("server.register.customNodeName") - if customNodeName == "" { - log.Error("server.register.customNodeName is empty") - debug.PrintStack() - register = nil - } - register = &CustomNameRegister{ - CustomName: customNodeName, - } - } - log.Info("register type is :" + reflect.TypeOf(register).String()) - - }) - return register -} diff --git a/backend/services/rpc/base.go b/backend/services/rpc/base.go index 60d2151a..369c62f4 100644 --- a/backend/services/rpc/base.go +++ b/backend/services/rpc/base.go @@ -85,6 +85,7 @@ func handleMsg(msgStr string, node *model.Node) { if err := json.Unmarshal([]byte(msgStr), &msg); err != nil { log.Errorf(err.Error()) debug.PrintStack() + return } // 获取service @@ -95,6 +96,7 @@ func handleMsg(msgStr string, node *model.Node) { if err != nil { log.Errorf(err.Error()) debug.PrintStack() + return } // 发送返回消息 @@ -107,9 +109,9 @@ func handleMsg(msgStr string, node *model.Node) { // 初始化服务端RPC服务 func InitRpcService() error { go func() { + node := local_node.CurrentNode() for { // 获取当前节点 - node := local_node.CurrentNode() //node, err := model.GetCurrentNode() //if err != nil { // log.Errorf(err.Error()) @@ -129,7 +131,7 @@ func InitRpcService() error { } return err }, bp) - if err != nil && err != redis.ErrNil { + if err != nil { continue } // 处理消息 From 6687695438d3d68dd28a9eb04a1e2a68b7e10ff9 Mon Sep 17 00:00:00 2001 From: yaziming Date: Wed, 27 May 2020 19:04:54 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E4=B8=8D=E5=AD=98=E5=9C=A8=E6=97=B6=EF=BC=8C=E6=97=A0=E6=B3=95?= =?UTF-8?q?=E4=B8=BB=E5=8A=A8=E5=88=9B=E5=BB=BAmaster=E8=8A=82=E7=82=B9?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/main.go | 7 +------ backend/services/local_node/local_node.go | 7 +++++++ backend/services/node.go | 4 ---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/backend/main.go b/backend/main.go index 4a801b44..19846a03 100644 --- a/backend/main.go +++ b/backend/main.go @@ -81,12 +81,7 @@ func main() { log.Info("initialized local node successfully") if model.IsMaster() { - // 初始化Master节点信息 - if err := services.InitMasterNodeInfo(); err != nil { - log.Error("init master node info error:" + err.Error()) - debug.PrintStack() - panic(err) - } + // 初始化定时任务 if err := services.InitScheduler(); err != nil { log.Error("init scheduler error:" + err.Error()) diff --git a/backend/services/local_node/local_node.go b/backend/services/local_node/local_node.go index 109f6534..9b1e1229 100644 --- a/backend/services/local_node/local_node.go +++ b/backend/services/local_node/local_node.go @@ -21,5 +21,12 @@ func InitLocalNodeInfo() (err error) { if err != nil { return err } + if model.IsMaster() { + err = model.UpdateMasterNodeInfo(localNode.Identify, localNode.Ip, localNode.Mac, localNode.Hostname) + + if err != nil { + return err + } + } return localNode.Ready() } diff --git a/backend/services/node.go b/backend/services/node.go index 2b094e98..0cfcebb2 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -284,7 +284,3 @@ func InitNodeService() error { c.Start() return nil } -func InitMasterNodeInfo() (err error) { - localNode := local_node.GetLocalNode() - return model.UpdateMasterNodeInfo(localNode.Identify, localNode.Ip, localNode.Mac, localNode.Hostname) -} From 4575ecd673fb100361dae4d27682132fc74acd5f Mon Sep 17 00:00:00 2001 From: yaziming Date: Fri, 29 May 2020 22:51:23 +0800 Subject: [PATCH 3/6] validate use input mongo id --- backend/mock/node.go | 13 ++++++- backend/mock/schedule.go | 9 ++++- backend/mock/spider.go | 11 ++++-- backend/mock/task.go | 20 ++++++++--- backend/routes/action.go | 5 ++- backend/routes/config_spider.go | 15 ++++++-- backend/routes/node.go | 25 ++++++++++--- backend/routes/schedule.go | 23 ++++++++++-- backend/routes/spider.go | 62 ++++++++++++++++++++++++++++----- backend/routes/system.go | 30 +++++++++++++--- backend/routes/task.go | 40 ++++++++++++++++----- backend/routes/token.go | 5 ++- backend/routes/user.go | 5 ++- backend/routes/variable.go | 9 +++++ 14 files changed, 229 insertions(+), 43 deletions(-) diff --git a/backend/mock/node.go b/backend/mock/node.go index 6c77c32e..2d3a978d 100644 --- a/backend/mock/node.go +++ b/backend/mock/node.go @@ -125,6 +125,10 @@ func GetNodeList(c *gin.Context) { func GetNode(c *gin.Context) { var result model.Node id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } for _, node := range NodeList { if node.Id == bson.ObjectId(id) { result = node @@ -150,6 +154,10 @@ func Ping(c *gin.Context) { func PostNode(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var oldItem model.Node for _, node := range NodeList { if node.Id == bson.ObjectId(id) { @@ -200,7 +208,10 @@ func DeleteNode(c *gin.Context) { func GetSystemInfo(c *gin.Context) { id := c.Param("id") - log.Info(id) + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } sysInfo := systemInfo c.JSON(http.StatusOK, Response{ diff --git a/backend/mock/schedule.go b/backend/mock/schedule.go index 015236f8..e5c45546 100644 --- a/backend/mock/schedule.go +++ b/backend/mock/schedule.go @@ -54,7 +54,10 @@ func GetScheduleList(c *gin.Context) { func GetSchedule(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var result model.Schedule for _, sch := range scheduleList { if sch.Id == bson.ObjectId(id) { @@ -70,6 +73,10 @@ func GetSchedule(c *gin.Context) { func PostSchedule(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var oldItem model.Schedule for _, sch := range scheduleList { if sch.Id == bson.ObjectId(id) { diff --git a/backend/mock/spider.go b/backend/mock/spider.go index e6c20a6b..1994196e 100644 --- a/backend/mock/spider.go +++ b/backend/mock/spider.go @@ -49,6 +49,7 @@ func GetSpider(c *gin.Context) { if !bson.IsObjectIdHex(id) { HandleErrorF(http.StatusBadRequest, c, "invalid id") + return } for _, spider := range SpiderList { @@ -87,7 +88,10 @@ func PostSpider(c *gin.Context) { func GetSpiderDir(c *gin.Context) { // 爬虫ID id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 目录相对路径 path := c.Query("path") var spi model.Spider @@ -127,7 +131,10 @@ func GetSpiderDir(c *gin.Context) { func GetSpiderTasks(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var spider model.Spider for _, spi := range SpiderList { if spi.Id == bson.ObjectId(id) { diff --git a/backend/mock/task.go b/backend/mock/task.go index 7b77d07e..3a2c8fcd 100644 --- a/backend/mock/task.go +++ b/backend/mock/task.go @@ -65,7 +65,10 @@ func GetTaskList(c *gin.Context) { func GetTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var result model.Task for _, task := range TaskList { if task.Id == id { @@ -111,7 +114,10 @@ func PutTask(c *gin.Context) { func DeleteTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } for _, task := range TaskList { if task.Id == id { fmt.Println("delete the task") @@ -126,7 +132,10 @@ func DeleteTask(c *gin.Context) { func GetTaskResults(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 绑定数据 data := TaskResultsRequestData{} if err := c.ShouldBindQuery(&data); err != nil { @@ -157,7 +166,10 @@ func GetTaskResults(c *gin.Context) { func DownloadTaskResultsCsv(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取任务 var task model.Task for _, ta := range TaskList { diff --git a/backend/routes/action.go b/backend/routes/action.go index ba832cf6..19144a4e 100644 --- a/backend/routes/action.go +++ b/backend/routes/action.go @@ -10,7 +10,10 @@ import ( func GetAction(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } user, err := model.GetAction(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/config_spider.go b/backend/routes/config_spider.go index 46594c59..93b5c7f9 100644 --- a/backend/routes/config_spider.go +++ b/backend/routes/config_spider.go @@ -129,7 +129,10 @@ func PostConfigSpider(c *gin.Context) { // @Router /config_spiders/{id}/upload [post] func UploadConfigSpider(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取爬虫 var spider model.Spider spider, err := model.GetSpider(bson.ObjectIdHex(id)) @@ -231,7 +234,10 @@ func PostConfigSpiderSpiderfile(c *gin.Context) { } id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 文件内容 var reqBody Body if err := c.ShouldBindJSON(&reqBody); err != nil { @@ -297,7 +303,10 @@ func PostConfigSpiderSpiderfile(c *gin.Context) { // @Router /config_spiders/{id}/config [post] func PostConfigSpiderConfig(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取爬虫 var spider model.Spider spider, err := model.GetSpider(bson.ObjectIdHex(id)) diff --git a/backend/routes/node.go b/backend/routes/node.go index dfa593eb..434e1b33 100644 --- a/backend/routes/node.go +++ b/backend/routes/node.go @@ -45,7 +45,10 @@ func GetNodeList(c *gin.Context) { // @Router /nodes/{id} [get] func GetNode(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } result, err := model.GetNode(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -71,7 +74,6 @@ func Ping(c *gin.Context) { }) } - // @Summary Post node // @Description Post node // @Tags node @@ -84,7 +86,10 @@ func Ping(c *gin.Context) { // @Router /nodes/{id} [post] func PostNode(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } item, err := model.GetNode(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -120,7 +125,10 @@ func PostNode(c *gin.Context) { // @Router /nodes/{id}/tasks [get] func GetNodeTaskList(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } tasks, err := model.GetNodeTaskList(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -145,7 +153,10 @@ func GetNodeTaskList(c *gin.Context) { // @Router /nodes/{id}/system [get] func GetSystemInfo(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } sysInfo, _ := services.GetSystemInfo(id) c.JSON(http.StatusOK, Response{ @@ -166,6 +177,10 @@ func GetSystemInfo(c *gin.Context) { // @Router /nodes/{id} [delete] func DeleteNode(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } node, err := model.GetNode(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/schedule.go b/backend/routes/schedule.go index e1e2fef3..6edfc705 100644 --- a/backend/routes/schedule.go +++ b/backend/routes/schedule.go @@ -41,7 +41,10 @@ func GetScheduleList(c *gin.Context) { // @Router /schedules/{id} [get] func GetSchedule(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } result, err := model.GetSchedule(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -64,7 +67,10 @@ func GetSchedule(c *gin.Context) { // @Router /schedules/{id} [post] func PostSchedule(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 绑定数据模型 var newItem model.Schedule if err := c.ShouldBindJSON(&newItem); err != nil { @@ -148,7 +154,10 @@ func PutSchedule(c *gin.Context) { // @Router /schedules/{id} [delete] func DeleteSchedule(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 删除定时任务 if err := model.RemoveSchedule(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -177,6 +186,10 @@ func DeleteSchedule(c *gin.Context) { // @Router /schedules/{id}/disable [post] func DisableSchedule(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } if err := services.Sched.Disable(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -197,6 +210,10 @@ func DisableSchedule(c *gin.Context) { // @Router /schedules/{id}/enable [post] func EnableSchedule(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } if err := services.Sched.Enable(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 0a25927d..3363ec9f 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -567,7 +567,10 @@ func UploadSpiderFromId(c *gin.Context) { // TODO: 与 UploadSpider 部分逻辑重复，需要优化代码 // 爬虫ID spiderId := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取爬虫 spider, err := model.GetSpider(bson.ObjectIdHex(spiderId)) if err != nil { @@ -877,7 +880,10 @@ func RunSelectedSpider(c *gin.Context) { // @Router /spiders/{id}/tasks [get] func GetSpiderTasks(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -924,7 +930,10 @@ func GetSpiderStats(c *gin.Context) { } id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { log.Errorf(err.Error()) @@ -1074,7 +1083,10 @@ func GetSpiderSchedules(c *gin.Context) { func GetSpiderDir(c *gin.Context) { // 爬虫ID id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 目录相对路径 path := c.Query("path") @@ -1131,7 +1143,10 @@ type SpiderFileReqBody struct { func GetSpiderFile(c *gin.Context) { // 爬虫ID id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 文件相对路径 path := c.Query("path") @@ -1168,7 +1183,10 @@ func GetSpiderFile(c *gin.Context) { func GetSpiderFileTree(c *gin.Context) { // 爬虫ID id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取爬虫 spider, err := model.GetSpider(bson.ObjectIdHex(id)) if err != nil { @@ -1208,7 +1226,10 @@ func GetSpiderFileTree(c *gin.Context) { func PostSpiderFile(c *gin.Context) { // 爬虫ID id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 文件相对路径 var reqBody SpiderFileReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { @@ -1254,6 +1275,10 @@ func PostSpiderFile(c *gin.Context) { // @Router /spiders/{id}/file [post] func PutSpiderFile(c *gin.Context) { spiderId := c.Param("id") + if !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody SpiderFileReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -1304,6 +1329,10 @@ func PutSpiderFile(c *gin.Context) { // @Router /spiders/{id}/file [put] func PutSpiderDir(c *gin.Context) { spiderId := c.Param("id") + if !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody SpiderFileReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -1354,6 +1383,10 @@ func PutSpiderDir(c *gin.Context) { // @Router /spiders/{id}/file [delete] func DeleteSpiderFile(c *gin.Context) { spiderId := c.Param("id") + if !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody SpiderFileReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -1394,6 +1427,11 @@ func DeleteSpiderFile(c *gin.Context) { // @Router /spiders/{id}/file/rename [post] func RenameSpiderFile(c *gin.Context) { spiderId := c.Param("id") + + if !bson.IsObjectIdHex(spiderId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody SpiderFileReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -1494,7 +1532,10 @@ func PutSpiderScrapySpiders(c *gin.Context) { } id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody ReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleErrorF(http.StatusBadRequest, c, "invalid request") @@ -1724,7 +1765,10 @@ func GetSpiderScrapyPipelines(c *gin.Context) { // @Router /spiders/{id}/scrapy/spider/filepath [get] func GetSpiderScrapySpiderFilepath(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } spiderName := c.Query("spider_name") if spiderName == "" { HandleErrorF(http.StatusBadRequest, c, "spider_name is empty") diff --git a/backend/routes/system.go b/backend/routes/system.go index 5f3ab7c5..a94be822 100644 --- a/backend/routes/system.go +++ b/backend/routes/system.go @@ -7,6 +7,7 @@ import ( "crawlab/services/rpc" "fmt" "github.com/gin-gonic/gin" + "github.com/globalsign/mgo/bson" "net/http" "strings" ) @@ -22,6 +23,10 @@ import ( // @Router /nodes/{id}/langs [get] func GetLangList(c *gin.Context) { nodeId := c.Param("id") + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } c.JSON(http.StatusOK, Response{ Status: "ok", Message: "success", @@ -44,7 +49,10 @@ func GetDepList(c *gin.Context) { nodeId := c.Param("id") lang := c.Query("lang") depName := c.Query("dep_name") - + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var depList []entity.Dependency if lang == constants.Python { list, err := services.GetPythonDepList(nodeId, depName) @@ -85,6 +93,11 @@ func GetDepList(c *gin.Context) { func GetInstalledDepList(c *gin.Context) { nodeId := c.Param("id") lang := c.Query("lang") + + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var depList []entity.Dependency if services.IsMasterNode(nodeId) { list, err := rpc.GetInstalledDepsLocal(lang) @@ -177,7 +190,10 @@ func InstallDep(c *gin.Context) { } nodeId := c.Param("id") - + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody ReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -218,7 +234,10 @@ func UninstallDep(c *gin.Context) { } nodeId := c.Param("id") - + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody ReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -292,7 +311,10 @@ func InstallLang(c *gin.Context) { } nodeId := c.Param("id") - + if !bson.IsObjectIdHex(nodeId) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqBody ReqBody if err := c.ShouldBindJSON(&reqBody); err != nil { HandleError(http.StatusBadRequest, c, err) diff --git a/backend/routes/task.go b/backend/routes/task.go index 6f3f2182..a11f197f 100644 --- a/backend/routes/task.go +++ b/backend/routes/task.go @@ -101,7 +101,10 @@ func GetTaskList(c *gin.Context) { // @Router /tasks/{id} [get] func GetTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } result, err := model.GetTask(id) if err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -273,7 +276,10 @@ func DeleteSelectedTask(c *gin.Context) { // @Router /task/{id} [delete] func DeleteTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 删除日志文件 if err := services.RemoveLogByTaskId(id); err != nil { HandleError(http.StatusInternalServerError, c, err) @@ -303,6 +309,10 @@ func GetTaskLog(c *gin.Context) { Keyword string `form:"keyword"` } id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var reqData RequestData if err := c.ShouldBindQuery(&reqData); err != nil { HandleErrorF(http.StatusBadRequest, c, "invalid request") @@ -332,6 +342,10 @@ func GetTaskLog(c *gin.Context) { // @Router /tasks/{id}/error-log [delete] func GetTaskErrorLog(c *gin.Context) { id := c.Param("id") + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } u := services.GetCurrentUser(c) errLogItems, err := services.GetTaskErrorLog(id, u.Setting.MaxErrorLog) if err != nil { @@ -357,7 +371,10 @@ func GetTaskErrorLog(c *gin.Context) { // @Router /tasks/{id}/results [get] func GetTaskResults(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 绑定数据 data := TaskResultsRequestData{} if err := c.ShouldBindQuery(&data); err != nil { @@ -387,7 +404,6 @@ func GetTaskResults(c *gin.Context) { }) } - // @Summary Get task results // @Description Get task results // @Tags task @@ -399,7 +415,10 @@ func GetTaskResults(c *gin.Context) { // @Router /tasks/{id}/results/download [get] func DownloadTaskResultsCsv(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } // 获取任务 task, err := model.GetTask(id) if err != nil { @@ -466,7 +485,6 @@ func DownloadTaskResultsCsv(c *gin.Context) { c.Data(http.StatusOK, "text/csv", bytesBuffer.Bytes()) } - // @Summary Cancel task // @Description Cancel task // @Tags task @@ -478,7 +496,10 @@ func DownloadTaskResultsCsv(c *gin.Context) { // @Router /tasks/{id}/cancel [post] func CancelTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } if err := services.CancelTask(id); err != nil { HandleError(http.StatusInternalServerError, c, err) return @@ -497,7 +518,10 @@ func CancelTask(c *gin.Context) { // @Router /tasks/{id}/restart [post] func RestartTask(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } uid := services.GetCurrentUserId(c) if err := services.RestartTask(id, uid); err != nil { diff --git a/backend/routes/token.go b/backend/routes/token.go index 8ad25aed..f62680a0 100644 --- a/backend/routes/token.go +++ b/backend/routes/token.go @@ -80,7 +80,10 @@ func PutToken(c *gin.Context) { // @Router /tokens/{id} [delete] func DeleteToken(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } if err := model.DeleteTokenById(bson.ObjectIdHex(id)); err != nil { HandleError(http.StatusInternalServerError, c, err) return diff --git a/backend/routes/user.go b/backend/routes/user.go index 7e6f1465..23391a48 100644 --- a/backend/routes/user.go +++ b/backend/routes/user.go @@ -36,7 +36,10 @@ type UserRequestData struct { // @Router /users/{id} [get] func GetUser(c *gin.Context) { id := c.Param("id") - + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } user, err := model.GetUser(bson.ObjectIdHex(id)) if err != nil { HandleError(http.StatusInternalServerError, c, err) diff --git a/backend/routes/variable.go b/backend/routes/variable.go index c55652ca..8b837538 100644 --- a/backend/routes/variable.go +++ b/backend/routes/variable.go @@ -45,6 +45,11 @@ func PutVariable(c *gin.Context) { // @Router /variable/{id} [post] func PostVariable(c *gin.Context) { var id = c.Param("id") + + if !bson.IsObjectIdHex(id) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var variable model.Variable if err := c.ShouldBindJSON(&variable); err != nil { HandleError(http.StatusBadRequest, c, err) @@ -71,6 +76,10 @@ func PostVariable(c *gin.Context) { // @Router /variable/{id} [delete] func DeleteVariable(c *gin.Context) { var idStr = c.Param("id") + if !bson.IsObjectIdHex(idStr) { + HandleErrorF(http.StatusBadRequest, c, "invalid id") + return + } var id = bson.ObjectIdHex(idStr) variable, err := model.GetVariable(id) if err != nil { From 6fac8e6b17b8999e825dcc91cc04ab792614f73d Mon Sep 17 00:00:00 2001 From: yaziming Date: Tue, 2 Jun 2020 15:32:25 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AD=90=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E4=B8=8D=E8=83=BD=E5=85=88=E6=B3=A8=E5=86=8C=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/database/pubsub.go | 2 -- backend/main.go | 14 ++++-------- backend/services/local_node/local_node.go | 13 +++-------- backend/services/node.go | 27 ++++++++++++++--------- 4 files changed, 24 insertions(+), 32 deletions(-) diff --git a/backend/database/pubsub.go b/backend/database/pubsub.go index fcb80b3e..f9eae535 100644 --- a/backend/database/pubsub.go +++ b/backend/database/pubsub.go @@ -39,8 +39,6 @@ func (r *Redis) subscribe(ctx context.Context, consume ConsumeFunc, channel ...s continue } case redis.Subscription: - fmt.Println(msg) - if msg.Count == 0 { // all channels are unsubscribed return diff --git a/backend/main.go b/backend/main.go index 19846a03..4b46b033 100644 --- a/backend/main.go +++ b/backend/main.go @@ -11,7 +11,6 @@ import ( "crawlab/routes" "crawlab/services" "crawlab/services/challenge" - "crawlab/services/local_node" "crawlab/services/rpc" "github.com/apex/log" "github.com/gin-gonic/gin" @@ -74,8 +73,10 @@ func main() { panic(err) } log.Info("initialized log successfully") // 初始化日志设置 - if err := local_node.InitLocalNodeInfo(); err != nil { - log.Error("init local node error:" + err.Error()) + + // 初始化节点服务 + if err := services.InitNodeService(); err != nil { + log.Error("init node service error:" + err.Error()) panic(err) } log.Info("initialized local node successfully") @@ -131,13 +132,6 @@ func main() { } log.Info("initialized task executor successfully") - // 初始化节点服务 - if err := services.InitNodeService(); err != nil { - log.Error("init node service error:" + err.Error()) - panic(err) - } - log.Info("initialized node service successfully") - // 初始化爬虫服务 if err := services.InitSpiderService(); err != nil { log.Error("init spider service error:" + err.Error()) diff --git a/backend/services/local_node/local_node.go b/backend/services/local_node/local_node.go index 9b1e1229..ad1321ca 100644 --- a/backend/services/local_node/local_node.go +++ b/backend/services/local_node/local_node.go @@ -12,21 +12,14 @@ func CurrentNode() *model.Node { return GetLocalNode().Current() } -func InitLocalNodeInfo() (err error) { +func InitLocalNode() (node *LocalNode, err error) { registerType := viper.GetString("server.register.type") ip := viper.GetString("server.register.ip") customNodeName := viper.GetString("server.register.customNodeName") localNode, err = NewLocalNode(ip, customNodeName, registerType) if err != nil { - return err + return nil, err } - if model.IsMaster() { - err = model.UpdateMasterNodeInfo(localNode.Identify, localNode.Ip, localNode.Mac, localNode.Hostname) - - if err != nil { - return err - } - } - return localNode.Ready() + return localNode, err } diff --git a/backend/services/node.go b/backend/services/node.go index 0cfcebb2..95c326f3 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -226,6 +226,11 @@ func WorkerNodeCallback(message redis.Message) (err error) { // 初始化节点服务 func InitNodeService() error { + node, err := local_node.InitLocalNode() + if err != nil { + return err + } + // 构造定时任务 c := cron.New(cron.WithSeconds()) @@ -239,23 +244,25 @@ func InitNodeService() error { // 首次更新节点数据（注册到Redis） UpdateNodeData() - // 获取当前节点 - //node, err := model.GetCurrentNode() - // - //if err != nil { - // log.Errorf(err.Error()) - // return err - //} - node := local_node.CurrentNode() + err = node.Ready() + + if err != nil { + return err + } if model.IsMaster() { + err = model.UpdateMasterNodeInfo(node.Identify, node.Ip, node.Mac, node.Hostname) + + if err != nil { + return err + } // 如果为主节点，订阅主节点通信频道 if err := database.Sub(constants.ChannelMasterNode, MasterNodeCallback); err != nil { return err } } else { // 若为工作节点，订阅单独指定通信频道 - channel := constants.ChannelWorkerNode + node.Id.Hex() + channel := constants.ChannelWorkerNode + node.Current().Id.Hex() if err := database.Sub(channel, WorkerNodeCallback); err != nil { return err } @@ -276,7 +283,7 @@ func InitNodeService() error { } // 更新在当前节点执行中的任务状态为：abnormal - if err := model.UpdateTaskToAbnormal(node.Id); err != nil { + if err := model.UpdateTaskToAbnormal(node.Current().Id); err != nil { debug.PrintStack() return err } From fe9e1f65cfb63fd55bb6811cefe016be458be42d Mon Sep 17 00:00:00 2001 From: yaziming Date: Wed, 3 Jun 2020 18:11:13 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AD=90=E8=8A=82?= =?UTF-8?q?=E7=82=B9=E5=90=8D=E7=A7=B0=E6=98=BE=E7=A4=BA=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/node.go | 54 +++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/backend/services/node.go b/backend/services/node.go index 95c326f3..2228b97f 100644 --- a/backend/services/node.go +++ b/backend/services/node.go @@ -14,16 +14,18 @@ import ( "github.com/apex/log" "github.com/globalsign/mgo/bson" "github.com/gomodule/redigo/redis" - "github.com/spf13/viper" "runtime/debug" "time" ) type Data struct { - Key string `json:"key"` - Mac string `json:"mac"` - Ip string `json:"ip"` - Hostname string `json:"hostname"` + Key string `json:"key"` + Mac string `json:"mac"` + Ip string `json:"ip"` + Hostname string `json:"hostname"` + Name string `json:"name"` + NameType string `json:"name_type"` + Master bool `json:"master"` UpdateTs time.Time `json:"update_ts"` UpdateTsUnix int64 `json:"update_ts_unix"` @@ -122,19 +124,6 @@ func UpdateNodeStatus() { } } -func getNodeName(data *Data) string { - registerType := viper.GetString("server.register.type") - if registerType == constants.RegisterTypeMac { - return data.Ip - } else if registerType == constants.RegisterTypeIp { - return data.Ip - } else if registerType == constants.RegisterTypeHostname { - return data.Hostname - } else { - return data.Ip - } -} - // 处理节点信息 func UpdateNodeInfo(data *Data) (err error) { // 更新节点信息到数据库 @@ -144,17 +133,18 @@ func UpdateNodeInfo(data *Data) (err error) { _, err = c.Upsert(bson.M{"key": data.Key}, bson.M{ "$set": bson.M{ "status": constants.StatusOnline, + "key": data.Key, + "name": data.Name, + "name_type": data.NameType, + "ip": data.Ip, + "port": "8000", + "mac": data.Mac, + "is_master": data.Master, "update_ts": time.Now(), "update_ts_unix": time.Now().Unix(), }, "$setOnInsert": bson.M{ - "_id": bson.NewObjectId(), - "key": data.Key, - "name": getNodeName(data), - "ip": data.Ip, - "port": "8000", - "mac": data.Mac, - "is_master": data.Master, + "_id": bson.NewObjectId(), }, }) return err @@ -170,6 +160,8 @@ func UpdateNodeData() { Mac: localNode.Mac, Ip: localNode.Ip, Hostname: localNode.Hostname, + Name: localNode.Identify, + NameType: string(localNode.IdentifyType), Master: model.IsMaster(), UpdateTs: time.Now(), UpdateTsUnix: time.Now().Unix(), @@ -243,7 +235,12 @@ func InitNodeService() error { // 首次更新节点数据（注册到Redis） UpdateNodeData() - + if model.IsMaster() { + err = model.UpdateMasterNodeInfo(node.Identify, node.Ip, node.Mac, node.Hostname) + if err != nil { + return err + } + } err = node.Ready() if err != nil { @@ -251,11 +248,6 @@ func InitNodeService() error { } if model.IsMaster() { - err = model.UpdateMasterNodeInfo(node.Identify, node.Ip, node.Mac, node.Hostname) - - if err != nil { - return err - } // 如果为主节点，订阅主节点通信频道 if err := database.Sub(constants.ChannelMasterNode, MasterNodeCallback); err != nil { return err From 1907e1536a07661bd2c3e5a67ad4858c7babaeec Mon Sep 17 00:00:00 2001 From: yaziming Date: Wed, 3 Jun 2020 18:58:33 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E7=9A=84=E5=8F=98=E9=87=8F=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/routes/spider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/routes/spider.go b/backend/routes/spider.go index 3363ec9f..e7faf051 100644 --- a/backend/routes/spider.go +++ b/backend/routes/spider.go @@ -567,7 +567,7 @@ func UploadSpiderFromId(c *gin.Context) { // TODO: 与 UploadSpider 部分逻辑重复，需要优化代码 // 爬虫ID spiderId := c.Param("id") - if !bson.IsObjectIdHex(id) { + if !bson.IsObjectIdHex(spiderId) { HandleErrorF(http.StatusBadRequest, c, "invalid id") return }