Merge pull request #748 from yaziming/reactors/cached-node-info

cache local node info
This commit is contained in:
Marvin Zhang
2020-05-26 19:26:27 +08:00
committed by GitHub
10 changed files with 176 additions and 414 deletions

View File

@@ -9,7 +9,6 @@ require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/aokoli/goutils v1.0.1 // indirect
github.com/apex/log v1.1.4
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect
github.com/cenkalti/backoff/v4 v4.0.2
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/fsnotify/fsnotify v1.4.9
@@ -17,6 +16,7 @@ require (
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8
github.com/go-playground/validator/v10 v10.3.0
github.com/gomodule/redigo v2.0.0+incompatible
github.com/hashicorp/go-sockaddr v1.0.0
github.com/huandu/xstrings v1.2.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/imroc/req v0.3.0
@@ -32,7 +32,6 @@ require (
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/swaggo/gin-swagger v1.2.0
github.com/swaggo/swag v1.6.6
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect
go.uber.org/atomic v1.6.0
golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

View File

@@ -40,15 +40,12 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
github.com/apex/log v1.1.1 h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA=
github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA=
github.com/apex/log v1.1.4 h1:3Zk+boorIQAAGBrHn0JUtAau4ihMamT4WdnfdnXM1zQ=
github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
github.com/apex/logs v0.0.4/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -66,9 +63,7 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
@@ -178,7 +173,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -191,6 +185,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -209,8 +204,6 @@ github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0
github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4=
github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imroc/req v0.2.4 h1:8XbvaQpERLAJV6as/cB186DtH5f0m5zAOtHEaTQ4ac0=
github.com/imroc/req v0.2.4/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE=
github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U=
github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw=
github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ=
@@ -246,8 +239,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -294,8 +285,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/olivere/elastic/v7 v7.0.14 h1:89dYPg6kD3WJx42ZtO4U6WDIzRy69FvQqz/yRiwekuM=
github.com/olivere/elastic/v7 v7.0.14/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE=
github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -339,8 +328,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
@@ -354,8 +341,6 @@ github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
@@ -401,7 +386,6 @@ github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -436,6 +420,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -455,7 +440,6 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -556,8 +540,6 @@ google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBr
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
@@ -573,8 +555,6 @@ gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXa
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737 h1:NvePS/smRcFQ4bMtTddFtknbGCtoBkJxGmpSpVRafCc=
gopkg.in/gomail.v2 v2.0.0-20150902115704-41f357289737/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=

View File

@@ -11,6 +11,7 @@ import (
"crawlab/routes"
"crawlab/services"
"crawlab/services/challenge"
"crawlab/services/local_node"
"crawlab/services/rpc"
"github.com/apex/log"
"github.com/gin-gonic/gin"
@@ -72,7 +73,12 @@ func main() {
log.Error("init log error:" + err.Error())
panic(err)
}
log.Info("initialized log successfully")
log.Info("initialized log successfully") // 初始化日志设置
if err := local_node.InitLocalNodeInfo(); err != nil {
log.Error("init local node error:" + err.Error())
panic(err)
}
log.Info("initialized local node successfully")
if model.IsMaster() {
// 初始化Master节点信息

View File

@@ -3,7 +3,6 @@ package model
import (
"crawlab/constants"
"crawlab/database"
"crawlab/services/register"
"errors"
"github.com/apex/log"
"github.com/globalsign/mgo"
@@ -42,16 +41,6 @@ func IsMaster() bool {
return viper.GetString("server.master") == Yes
}
// 获取本机节点
func GetCurrentNode() (Node, error) {
// 获得注册的key值
key, err := register.GetRegister().GetKey()
if err != nil {
return Node{}, err
}
return GetNodeByKey(key)
}
func (n *Node) Save() error {
s, c := database.GetCol("nodes")
defer s.Close()
@@ -190,34 +179,6 @@ func GetNodeCount(query interface{}) (int, error) {
return count, nil
}
// 节点基本信息
func GetNodeBaseInfo() (ip string, mac string, hostname string, key string, error error) {
ip, err := register.GetRegister().GetIp()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
mac, err = register.GetRegister().GetMac()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
hostname, err = register.GetRegister().GetHostname()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
key, err = register.GetRegister().GetKey()
if err != nil {
debug.PrintStack()
return "", "", "", "", err
}
return ip, mac, hostname, key, nil
}
// 根据redis的key值重置node节点为offline
func ResetNodeStatusToOffline(list []string) {
nodes, _ := GetNodeList(nil)

View File

@@ -2,71 +2,24 @@ package local_node
import (
"crawlab/model"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"go.uber.org/atomic"
"sync"
"time"
"github.com/spf13/viper"
)
var localNode *LocalNode
var locker atomic.Int32
var once sync.Once
type LocalNode struct {
node *model.Node
sync.RWMutex
}
func (n *LocalNode) load(retry bool) (err error) {
n.Lock()
defer n.Unlock()
var node model.Node
if retry {
b := backoff.NewConstantBackOff(1 * time.Second)
err = backoff.Retry(func() error {
node, err = model.GetCurrentNode()
if err != nil {
log.WithError(err).Warnf("Get current node info from database failed. Will after %f seconds, try again.", b.NextBackOff().Seconds())
}
return err
}, b)
} else {
node, err = model.GetCurrentNode()
}
if err != nil {
return
}
n.node = &node
return nil
}
func (n *LocalNode) watch() {
timer := time.NewTicker(time.Second * 5)
for range timer.C {
if locker.CAS(0, 1) {
err := n.load(false)
if err != nil {
log.WithError(err).Errorf("load current node from database failed,")
}
locker.Store(0)
}
continue
}
}
func (n *LocalNode) Current() *model.Node {
n.RLock()
defer n.RUnlock()
return n.node
func GetLocalNode() *LocalNode {
return localNode
}
func CurrentNode() *model.Node {
once.Do(func() {
localNode = &LocalNode{}
_ = localNode.load(true)
go localNode.watch()
})
return localNode.Current()
return GetLocalNode().Current()
}
func InitLocalNodeInfo() (err error) {
registerType := viper.GetString("server.register.type")
ip := viper.GetString("server.register.ip")
customNodeName := viper.GetString("server.register.customNodeName")
localNode, err = NewLocalNode(ip, customNodeName, registerType)
if err != nil {
return err
}
return localNode.Ready()
}

View File

@@ -0,0 +1,62 @@
package local_node
import (
"crawlab/model"
"github.com/apex/log"
"github.com/cenkalti/backoff/v4"
"go.uber.org/atomic"
"sync"
"time"
)
var locker atomic.Int32
type mongo struct {
node *model.Node
sync.RWMutex
}
func (n *mongo) load(retry bool) (err error) {
n.Lock()
defer n.Unlock()
var node model.Node
if retry {
b := backoff.NewConstantBackOff(1 * time.Second)
err = backoff.Retry(func() error {
node, err = model.GetNodeByKey(GetLocalNode().Identify)
if err != nil {
log.WithError(err).Warnf("Get current node info from database failed. Will after %f seconds, try again.", b.NextBackOff().Seconds())
}
return err
}, b)
} else {
node, err = model.GetNodeByKey(localNode.Identify)
}
if err != nil {
return
}
n.node = &node
return nil
}
func (n *mongo) watch() {
timer := time.NewTicker(time.Second * 5)
for range timer.C {
if locker.CAS(0, 1) {
err := n.load(false)
if err != nil {
log.WithError(err).Errorf("load current node from database failed")
}
locker.Store(0)
}
continue
}
}
func (n *mongo) Current() *model.Node {
n.RLock()
defer n.RUnlock()
return n.node
}

View File

@@ -0,0 +1,74 @@
package local_node
import (
"errors"
"github.com/hashicorp/go-sockaddr"
"os"
)
var localNode *LocalNode
type IdentifyType string
const (
Ip = IdentifyType("ip")
Mac = IdentifyType("mac")
Hostname = IdentifyType("hostname")
)
type local struct {
Ip string
Mac string
Hostname string
Identify string
IdentifyType IdentifyType
}
type LocalNode struct {
local
mongo
}
func (l *LocalNode) Ready() error {
err := localNode.load(true)
if err != nil {
return err
}
go localNode.watch()
return nil
}
func NewLocalNode(ip string, identify string, identifyTypeString string) (node *LocalNode, err error) {
addrs, err := sockaddr.GetPrivateInterfaces()
if ip == "" {
if err != nil {
return node, err
}
if len(addrs) == 0 {
return node, errors.New("address not found")
}
ipaddr := *sockaddr.ToIPAddr(addrs[0].SockAddr)
ip = ipaddr.NetIP().String()
}
mac := addrs[0].HardwareAddr.String()
hostname, err := os.Hostname()
if err != nil {
return node, err
}
local := local{Ip: ip, Mac: mac, Hostname: hostname}
switch IdentifyType(identifyTypeString) {
case Ip:
local.Identify = local.Ip
local.IdentifyType = Ip
case Mac:
local.Identify = local.Mac
local.IdentifyType = Mac
case Hostname:
local.Identify = local.Hostname
local.IdentifyType = Hostname
default:
local.Identify = identify
local.IdentifyType = IdentifyType(identifyTypeString)
}
return &LocalNode{local: local, mongo: mongo{}}, nil
}

View File

@@ -8,7 +8,6 @@ import (
"crawlab/model"
"crawlab/services/local_node"
"crawlab/services/msg_handler"
"crawlab/services/register"
"crawlab/utils"
"encoding/json"
"fmt"
@@ -41,9 +40,10 @@ func IsMasterNode(id string) bool {
// 获取节点数据
func GetNodeData() (Data, error) {
key, err := register.GetRegister().GetKey()
localNode := local_node.GetLocalNode()
key := localNode.Identify
if key == "" {
return Data{}, err
return Data{}, nil
}
value, err := database.RedisClient.HGet("nodes", key)
@@ -162,41 +162,14 @@ func UpdateNodeInfo(data *Data) (err error) {
// 更新节点数据
func UpdateNodeData() {
// 获取MAC地址
mac, err := register.GetRegister().GetMac()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取IP地址
ip, err := register.GetRegister().GetIp()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取Hostname
hostname, err := register.GetRegister().GetHostname()
if err != nil {
log.Errorf(err.Error())
return
}
// 获取redis的key
key, err := register.GetRegister().GetKey()
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
localNode := local_node.GetLocalNode()
key := localNode.Identify
// 构造节点数据
data := Data{
Key: key,
Mac: mac,
Ip: ip,
Hostname: hostname,
Mac: localNode.Mac,
Ip: localNode.Ip,
Hostname: localNode.Hostname,
Master: model.IsMaster(),
UpdateTs: time.Now(),
UpdateTsUnix: time.Now().Unix(),
@@ -312,11 +285,6 @@ func InitNodeService() error {
return nil
}
func InitMasterNodeInfo() (err error) {
// 获取本机信息
ip, mac, hostname, key, err := model.GetNodeBaseInfo()
if err != nil {
debug.PrintStack()
return err
}
return model.UpdateMasterNodeInfo(key, ip, mac, hostname)
localNode := local_node.GetLocalNode()
return model.UpdateMasterNodeInfo(localNode.Identify, localNode.Ip, localNode.Mac, localNode.Hostname)
}

View File

@@ -1,243 +0,0 @@
package register
import (
"bytes"
"crawlab/constants"
"fmt"
"github.com/apex/log"
"github.com/spf13/viper"
"net"
"os/exec"
"reflect"
"runtime/debug"
"strings"
"sync"
)
type Register interface {
// 注册的key类型
GetType() string
// 注册的key的值唯一标识节点
GetKey() (string, error)
// 注册的节点IP
GetIp() (string, error)
// 注册节点的mac地址
GetMac() (string, error)
// 注册节点的Hostname
GetHostname() (string, error)
GetCustomName() (string, error)
}
// ===================== mac 地址注册 =====================
type MacRegister struct{}
func (mac *MacRegister) GetType() string {
return "mac"
}
func (mac *MacRegister) GetKey() (string, error) {
return mac.GetMac()
}
func (mac *MacRegister) GetMac() (string, error) {
return getMac()
}
func (mac *MacRegister) GetIp() (string, error) {
return getIp()
}
func (mac *MacRegister) GetHostname() (string, error) {
return getHostname()
}
func (mac *MacRegister) GetCustomName() (string, error) {
return getMac()
}
// ===================== ip 地址注册 =====================
type IpRegister struct {
Ip string
}
func (ip *IpRegister) GetCustomName() (string, error) {
return ip.Ip, nil
}
// ============= 自定义节点名称注册 ==============
type CustomNameRegister struct {
CustomName string
}
func (c *CustomNameRegister) GetType() string {
return "customName"
}
func (c *CustomNameRegister) GetIp() (string, error) {
return getIp()
}
func (c *CustomNameRegister) GetMac() (string, error) {
return getMac()
}
func (c *CustomNameRegister) GetKey() (string, error) {
return c.CustomName, nil
}
func (c *CustomNameRegister) GetHostname() (string, error) {
return getHostname()
}
func (c *CustomNameRegister) GetCustomName() (string, error) {
return c.CustomName, nil
}
// ============================================================
func (ip *IpRegister) GetType() string {
return "ip"
}
func (ip *IpRegister) GetKey() (string, error) {
return ip.Ip, nil
}
func (ip *IpRegister) GetIp() (string, error) {
return ip.Ip, nil
}
func (ip *IpRegister) GetMac() (string, error) {
return getMac()
}
func (ip *IpRegister) GetHostname() (string, error) {
return getHostname()
}
// ===================== mac 地址注册 =====================
type HostnameRegister struct{}
func (h *HostnameRegister) GetType() string {
return "mac"
}
func (h *HostnameRegister) GetKey() (string, error) {
return h.GetHostname()
}
func (h *HostnameRegister) GetMac() (string, error) {
return getMac()
}
func (h *HostnameRegister) GetIp() (string, error) {
return getIp()
}
func (h *HostnameRegister) GetHostname() (string, error) {
return getHostname()
}
func (h *HostnameRegister) GetCustomName() (string, error) {
return getHostname()
}
// ===================== 公共方法 =====================
// 获取本机的IP地址
// TODO: 考虑多个IP地址的情况
func getIp() (string, error) {
addrList, err := net.InterfaceAddrs()
if err != nil {
return "", err
}
for _, value := range addrList {
if ipNet, ok := value.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
if ipNet.IP.To4() != nil {
return ipNet.IP.String(), nil
}
}
}
return "", nil
}
func getMac() (string, error) {
interfaces, err := net.Interfaces()
if err != nil {
log.Errorf("get interfaces error:" + err.Error())
debug.PrintStack()
return "", err
}
for _, inter := range interfaces {
if inter.HardwareAddr != nil {
mac := inter.HardwareAddr.String()
return mac, nil
}
}
return "", nil
}
func getHostname() (string, error) {
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd := exec.Command("hostname")
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
log.Errorf(err.Error())
log.Errorf(fmt.Sprintf("error: %s", stderr.String()))
debug.PrintStack()
return "", err
}
return strings.Replace(stdout.String(), "\n", "", -1), nil
}
// ===================== 获得注册简单工厂 =====================
var register Register
// 获得注册器
var once sync.Once
func GetRegister() Register {
once.Do(func() {
registerType := viper.GetString("server.register.type")
switch registerType {
case constants.RegisterTypeMac:
register = &MacRegister{}
case constants.RegisterTypeIp:
ip := viper.GetString("server.register.ip")
if ip == "" {
log.Error("server.register.ip is empty")
debug.PrintStack()
register = nil
}
register = &IpRegister{
Ip: ip,
}
case constants.RegisterTypeHostname:
register = &HostnameRegister{}
case constants.RegisterTypeCustomName:
customNodeName := viper.GetString("server.register.customNodeName")
if customNodeName == "" {
log.Error("server.register.customNodeName is empty")
debug.PrintStack()
register = nil
}
register = &CustomNameRegister{
CustomName: customNodeName,
}
}
log.Info("register type is :" + reflect.TypeOf(register).String())
})
return register
}

View File

@@ -85,6 +85,7 @@ func handleMsg(msgStr string, node *model.Node) {
if err := json.Unmarshal([]byte(msgStr), &msg); err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
// 获取service
@@ -95,6 +96,7 @@ func handleMsg(msgStr string, node *model.Node) {
if err != nil {
log.Errorf(err.Error())
debug.PrintStack()
return
}
// 发送返回消息
@@ -107,9 +109,9 @@ func handleMsg(msgStr string, node *model.Node) {
// 初始化服务端RPC服务
func InitRpcService() error {
go func() {
node := local_node.CurrentNode()
for {
// 获取当前节点
node := local_node.CurrentNode()
//node, err := model.GetCurrentNode()
//if err != nil {
// log.Errorf(err.Error())
@@ -129,7 +131,7 @@ func InitRpcService() error {
}
return err
}, bp)
if err != nil && err != redis.ErrNil {
if err != nil {
continue
}
// 处理消息