diff --git a/backend/go.mod b/backend/go.mod index d9ef355a..f70efcd7 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -5,10 +5,11 @@ go 1.15 replace ( github.com/crawlab-team/crawlab-core => /Users/marvzhang/projects/crawlab-team/crawlab-core github.com/crawlab-team/crawlab-db => /Users/marvzhang/projects/crawlab-team/crawlab-db + github.com/crawlab-team/crawlab-fs => /Users/marvzhang/projects/crawlab-team/crawlab-fs github.com/crawlab-team/crawlab-grpc => /Users/marvzhang/projects/crawlab-team/crawlab-grpc/dist/go github.com/crawlab-team/crawlab-log => /Users/marvzhang/projects/crawlab-team/crawlab-log github.com/crawlab-team/crawlab-vcs => /Users/marvzhang/projects/crawlab-team/crawlab-vcs - github.com/crawlab-team/crawlab-fs => /Users/marvzhang/projects/crawlab-team/crawlab-fs + github.com/crawlab-team/go-trace => /Users/marvzhang/projects/crawlab-team/go-trace github.com/linxGnu/goseaweedfs => /Users/marvzhang/projects/tikazyq/goseaweedfs ) @@ -16,6 +17,7 @@ require ( github.com/apex/log v1.9.0 github.com/crawlab-team/crawlab-core v0.0.1 github.com/crawlab-team/crawlab-db v0.0.2 + github.com/crawlab-team/go-trace v0.0.0 github.com/gin-gonic/gin v1.6.3 github.com/go-playground/validator/v10 v10.3.0 github.com/olivere/elastic/v7 v7.0.15 diff --git a/backend/go.sum b/backend/go.sum index 98dfcdc3..b60adc4f 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -43,6 +43,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -119,7 +121,32 @@ github.com/go-playground/validator/v10 v10.3.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -143,6 +170,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -154,6 +183,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -202,6 +232,7 @@ github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= @@ -209,8 +240,12 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= @@ -221,11 +256,16 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -245,10 +285,14 @@ github.com/linxGnu/goseaweedfs v0.1.5 h1:7dChPdq8+fsPH0yqxKEofPhiosaar4LWePm4M+1 github.com/linxGnu/goseaweedfs v0.1.5/go.mod h1:Zwe/7H7FJaPQyMTNKXgv6fhVDw6qi34MMJQp1K0VLNc= github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k= +github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matcornic/hermes v1.2.0 h1:AuqZpYcTOtTB7cahdevLfnhIpfzmpqw5Czv8vpdnFDU= github.com/matcornic/hermes v1.2.0/go.mod h1:lujJomb016Xjv8wBnWlNvUdtmvowjjfkqri5J/+1hYc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -284,6 +328,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -298,6 +343,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -316,8 +363,12 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= @@ -333,6 +384,9 @@ github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -346,6 +400,7 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= @@ -369,6 +424,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -384,8 +440,16 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y= +github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.4.5 h1:TLtO+iD8krabXxvY1F1qpBOHgOxhLWR7XsT7kQeRmMY= +go.mongodb.org/mongo-driver v1.4.5/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -399,9 +463,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -456,7 +522,10 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -468,10 +537,14 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= @@ -504,9 +577,13 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= diff --git a/backend/main.go b/backend/main.go index 8832bad7..33c1d00d 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1,338 +1,252 @@ package main -import ( - "context" - "github.com/apex/log" - "github.com/crawlab-team/crawlab-core/config" - validate2 "github.com/crawlab-team/crawlab-core/lib/validate" - "github.com/crawlab-team/crawlab-core/middlewares" - "github.com/crawlab-team/crawlab-core/model" - "github.com/crawlab-team/crawlab-core/routes" - "github.com/crawlab-team/crawlab-core/services" - "github.com/crawlab-team/crawlab-core/services/rpc" - "github.com/crawlab-team/crawlab-db" - "github.com/gin-gonic/gin" - "github.com/gin-gonic/gin/binding" - "github.com/go-playground/validator/v10" - "github.com/olivere/elastic/v7" - "github.com/spf13/viper" - "net" - "net/http" - "os" - "os/signal" - "runtime/debug" - "syscall" - "time" -) +import "crawlab/services" func main() { - app := gin.New() - app.Use(gin.Logger(), gin.Recovery()) - if v, ok := binding.Validator.Engine().(*validator.Validate); ok { - _ = v.RegisterValidation("bid", validate2.MongoID) - } + api := services.NewApiService() + api.Init() + api.Run() - // 初始化配置 - if err := config.InitConfig(""); err != nil { - log.Error("init config error:" + err.Error()) - panic(err) - } - log.Info("initialized config successfully") - // 初始化Mongodb数据库 - if err := db.InitMongo(); err != nil { - log.Error("init mongodb error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized mongodb successfully") - - // 初始化Redis数据库 - if err := db.InitRedis(); err != nil { - log.Error("init redis error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized redis successfully") - - // 初始化日志设置 - if err := services.InitLogService(); err != nil { - log.Error("init log error:" + err.Error()) - panic(err) - } - log.Info("initialized log successfully") // 初始化日志设置 - - // 初始化节点服务 - if err := services.InitNodeService(); err != nil { - log.Error("init node service error:" + err.Error()) - panic(err) - } - log.Info("initialized local node successfully") - - if model.IsMaster() { - - // 初始化定时任务 - if err := services.InitScheduler(); err != nil { - log.Error("init scheduler error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized schedule successfully") - - // 初始化用户服务 - if err := services.InitUserService(); err != nil { - log.Error("init user service error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized user service successfully") - - // 初始化依赖服务 - if err := services.InitDepsFetcher(); err != nil { - log.Error("init dependency fetcher error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized dependency fetcher successfully") - - // 初始化清理服务 - if err := services.InitCleanService(); err != nil { - log.Error("init clean service error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized clean service successfully") - } - - // 初始化任务执行器 - if err := services.InitTaskExecutor(); err != nil { - log.Error("init task executor error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized task executor successfully") - - // 初始化爬虫服务 - if err := services.InitSpiderService(); err != nil { - log.Error("init spider service error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized spider service successfully") - - // 初始化RPC服务 - if err := rpc.InitRpcService(); err != nil { - log.Error("init rpc service error:" + err.Error()) - debug.PrintStack() - panic(err) - } - log.Info("initialized rpc service successfully") + //if model.IsMaster() { + // // 初始化定时任务 + // if err := services.InitScheduler(); err != nil { + // log.Error("init scheduler error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized schedule successfully") + // + // // 初始化用户服务 + // if err := services.InitUserService(); err != nil { + // log.Error("init user service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized user service successfully") + // + // // 初始化依赖服务 + // if err := services.InitDepsFetcher(); err != nil { + // log.Error("init dependency fetcher error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized dependency fetcher successfully") + // + // // 初始化清理服务 + // if err := services.InitCleanService(); err != nil { + // log.Error("init clean service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + // } + // log.Info("initialized clean service successfully") + //} + // + //// 初始化任务执行器 + //if err := services.InitTaskExecutor(); err != nil { + // log.Error("init task executor error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized task executor successfully") + // + //// 初始化爬虫服务 + //if err := services.InitSpiderService(); err != nil { + // log.Error("init spider service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized spider service successfully") + // + //// 初始化RPC服务 + //if err := rpc.InitRpcService(); err != nil { + // log.Error("init rpc service error:" + err.Error()) + // debug.PrintStack() + // panic(err) + //} + //log.Info("initialized rpc service successfully") // 以下为主节点服务 - if model.IsMaster() { - // 中间件 - esClientStr := viper.GetString("setting.esClient") - if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" { - ctx := context.Background() - esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false)) - if err != nil { - log.Error("Init es client Error:" + err.Error()) - } - app.Use(middlewares.EsLog(ctx, esClient)) - } - app.Use(middlewares.CORSMiddleware()) - anonymousGroup := app.Group("/") - { - anonymousGroup.POST("/login", routes.Login) // 用户登录 - anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 - anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 - // release版本 - anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 - anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 - // 文档 - anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据 - } - authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) - { - // 节点 - { - authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 - authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 - authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 - authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 - authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 - authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 - authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 - authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 - authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 - authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 - authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 - } - // 爬虫 - { - authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 - authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 - authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 - authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 - authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 - authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 - authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） - authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫 - authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 - authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫 - authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 - authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 - authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 - authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 - authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 - authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 - authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 - authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 - authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 - authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 - authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 - authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 - authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 - authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 - authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 - authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items - authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items - authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines - authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines - authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 - authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 - authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 - authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫 - authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目 - } - // 可配置爬虫 - { - authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 - authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 - authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 - authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 - authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 - authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 - authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 - } - // 任务 - { - authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 - authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 - authGroup.PUT("/tasks", routes.PutTask) // 派发任务 - authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务 - authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 - authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务 - //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务 - authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 - authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 - authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志 - authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 - authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 - authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务 - authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务 - authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务 - } - // 系统任务/脚本 - { - authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务 - authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表 - } - // 定时任务 - { - authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 - authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 - authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 - authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务 - authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 - authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 - authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务 - authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 - authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 - authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态 - } - // 用户 - { - authGroup.GET("/users", routes.GetUserList) // 用户列表 - authGroup.GET("/users/:id", routes.GetUser) // 用户详情 - authGroup.POST("/users/:id", routes.PostUser) // 更改用户 - authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 - authGroup.PUT("/users-add", routes.PutUser) // 添加用户 - authGroup.GET("/me", routes.GetMe) // 获取自己账户 - authGroup.POST("/me", routes.PostMe) // 修改自己账户 - authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码 - } - // 系统 - { - authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 - authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON - } - // 全局变量 - { - authGroup.GET("/variables", routes.GetVariableList) // 列表 - authGroup.PUT("/variable", routes.PutVariable) // 新增 - authGroup.POST("/variable/:id", routes.PostVariable) // 修改 - authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除 - } - // 项目 - { - authGroup.GET("/projects", routes.GetProjectList) // 列表 - authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 - authGroup.PUT("/projects", routes.PutProject) // 修改 - authGroup.POST("/projects/:id", routes.PostProject) // 新增 - authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 - } - // 操作 - { - //authGroup.GET("/actions", routes.GetActionList) // 操作列表 - //authGroup.GET("/actions/:id", routes.GetAction) // 操作 - authGroup.PUT("/actions", routes.PutAction) // 新增操作 - //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 - } - // API Token - { - authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens - authGroup.PUT("/tokens", routes.PutToken) // 添加 Token - authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token - } - // 统计数据 - authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 - // 文件 - authGroup.GET("/file", routes.GetFile) // 获取文件 - // Git - authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 - authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 - authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits - authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits - } - } + //if model.IsMaster() { + // // 中间件 + // esClientStr := viper.GetString("setting.esClient") + // if viper.GetString("setting.crawlabLogToES") == "Y" && esClientStr != "" { + // ctx := context.Background() + // esClient, err := elastic.NewClient(elastic.SetURL(esClientStr), elastic.SetSniff(false)) + // if err != nil { + // log.Error("Init es client Error:" + err.Error()) + // } + // app.Use(middlewares.EsLog(ctx, esClient)) + // } + // app.Use(middlewares.CORSMiddleware()) + // anonymousGroup := app.Group("/") + // { + // anonymousGroup.POST("/login", routes.Login) // 用户登录 + // anonymousGroup.PUT("/users", routes.PutUser) // 添加用户 + // anonymousGroup.GET("/setting", routes.GetSetting) // 获取配置信息 + // // release版本 + // anonymousGroup.GET("/version", routes.GetVersion) // 获取发布的版本 + // anonymousGroup.GET("/releases/latest", routes.GetLatestRelease) // 获取最近发布的版本 + // // 文档 + // anonymousGroup.GET("/docs", routes.GetDocs) // 获取文档数据 + // } + // authGroup := app.Group("/", middlewares.AuthorizationMiddleware()) + // { + // // 节点 + // { + // authGroup.GET("/nodes", routes.GetNodeList) // 节点列表 + // authGroup.GET("/nodes/:id", routes.GetNode) // 节点详情 + // authGroup.POST("/nodes/:id", routes.PostNode) // 修改节点 + // authGroup.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 + // authGroup.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 + // authGroup.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + // authGroup.GET("/nodes/:id/langs", routes.GetLangList) // 节点语言环境列表 + // authGroup.GET("/nodes/:id/deps", routes.GetDepList) // 节点第三方依赖列表 + // authGroup.GET("/nodes/:id/deps/installed", routes.GetInstalledDepList) // 节点已安装第三方依赖列表 + // authGroup.POST("/nodes/:id/deps/install", routes.InstallDep) // 节点安装依赖 + // authGroup.POST("/nodes/:id/deps/uninstall", routes.UninstallDep) // 节点卸载依赖 + // authGroup.POST("/nodes/:id/langs/install", routes.InstallLang) // 节点安装语言 + // } + // // 爬虫 + // { + // authGroup.GET("/spiders", routes.GetSpiderList) // 爬虫列表 + // authGroup.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 + // authGroup.PUT("/spiders", routes.PutSpider) // 添加爬虫 + // authGroup.POST("/spiders", routes.UploadSpider) // 上传爬虫 + // authGroup.POST("/spiders/:id", routes.PostSpider) // 修改爬虫 + // authGroup.POST("/spiders/:id/publish", routes.PublishSpider) // 发布爬虫 + // authGroup.POST("/spiders/:id/upload", routes.UploadSpiderFromId) // 上传爬虫（ID） + // authGroup.DELETE("/spiders", routes.DeleteSelectedSpider) // 删除选择的爬虫 + // authGroup.DELETE("/spiders/:id", routes.DeleteSpider) // 删除爬虫 + // authGroup.POST("/spiders/:id/copy", routes.CopySpider) // 拷贝爬虫 + // authGroup.GET("/spiders/:id/tasks", routes.GetSpiderTasks) // 爬虫任务列表 + // authGroup.GET("/spiders/:id/file/tree", routes.GetSpiderFileTree) // 爬虫文件目录树读取 + // authGroup.GET("/spiders/:id/file", routes.GetSpiderFile) // 爬虫文件读取 + // authGroup.POST("/spiders/:id/file", routes.PostSpiderFile) // 爬虫文件更改 + // authGroup.PUT("/spiders/:id/file", routes.PutSpiderFile) // 爬虫文件创建 + // authGroup.PUT("/spiders/:id/dir", routes.PutSpiderDir) // 爬虫目录创建 + // authGroup.DELETE("/spiders/:id/file", routes.DeleteSpiderFile) // 爬虫文件删除 + // authGroup.POST("/spiders/:id/file/rename", routes.RenameSpiderFile) // 爬虫文件重命名 + // authGroup.GET("/spiders/:id/dir", routes.GetSpiderDir) // 爬虫目录 + // authGroup.GET("/spiders/:id/stats", routes.GetSpiderStats) // 爬虫统计数据 + // authGroup.GET("/spiders/:id/schedules", routes.GetSpiderSchedules) // 爬虫定时任务 + // authGroup.GET("/spiders/:id/scrapy/spiders", routes.GetSpiderScrapySpiders) // Scrapy 爬虫名称列表 + // authGroup.PUT("/spiders/:id/scrapy/spiders", routes.PutSpiderScrapySpiders) // Scrapy 爬虫创建爬虫 + // authGroup.GET("/spiders/:id/scrapy/settings", routes.GetSpiderScrapySettings) // Scrapy 爬虫设置 + // authGroup.POST("/spiders/:id/scrapy/settings", routes.PostSpiderScrapySettings) // Scrapy 爬虫修改设置 + // authGroup.GET("/spiders/:id/scrapy/items", routes.GetSpiderScrapyItems) // Scrapy 爬虫 items + // authGroup.POST("/spiders/:id/scrapy/items", routes.PostSpiderScrapyItems) // Scrapy 爬虫修改 items + // authGroup.GET("/spiders/:id/scrapy/pipelines", routes.GetSpiderScrapyPipelines) // Scrapy 爬虫 pipelines + // authGroup.GET("/spiders/:id/scrapy/spider/filepath", routes.GetSpiderScrapySpiderFilepath) // Scrapy 爬虫 pipelines + // authGroup.POST("/spiders/:id/git/sync", routes.PostSpiderSyncGit) // 爬虫 Git 同步 + // authGroup.POST("/spiders/:id/git/reset", routes.PostSpiderResetGit) // 爬虫 Git 重置 + // authGroup.POST("/spiders-cancel", routes.CancelSelectedSpider) // 停止所选爬虫任务 + // authGroup.POST("/spiders-run", routes.RunSelectedSpider) // 运行所选爬虫 + // authGroup.POST("/spiders-set-projects", routes.SetProjectsSelectedSpider) // 批量设置爬虫项目 + // } + // // 可配置爬虫 + // { + // authGroup.GET("/config_spiders/:id/config", routes.GetConfigSpiderConfig) // 获取可配置爬虫配置 + // authGroup.POST("/config_spiders/:id/config", routes.PostConfigSpiderConfig) // 更改可配置爬虫配置 + // authGroup.PUT("/config_spiders", routes.PutConfigSpider) // 添加可配置爬虫 + // authGroup.POST("/config_spiders/:id", routes.PostConfigSpider) // 修改可配置爬虫 + // authGroup.POST("/config_spiders/:id/upload", routes.UploadConfigSpider) // 上传可配置爬虫 + // authGroup.POST("/config_spiders/:id/spiderfile", routes.PostConfigSpiderSpiderfile) // 上传可配置爬虫 + // authGroup.GET("/config_spiders_templates", routes.GetConfigSpiderTemplateList) // 获取可配置爬虫模版列表 + // } + // // 任务 + // { + // authGroup.GET("/tasks", routes.GetTaskList) // 任务列表 + // authGroup.GET("/tasks/:id", routes.GetTask) // 任务详情 + // authGroup.PUT("/tasks", routes.PutTask) // 派发任务 + // authGroup.PUT("/tasks/batch", routes.PutBatchTasks) // 批量派发任务 + // authGroup.DELETE("/tasks/:id", routes.DeleteTask) // 删除任务 + // authGroup.DELETE("/tasks", routes.DeleteSelectedTask) // 删除多个任务 + // //authGroup.DELETE("/tasks_by_status", routes.DeleteTaskByStatus) // 删除指定状态的任务 + // authGroup.POST("/tasks/:id/cancel", routes.CancelTask) // 取消任务 + // authGroup.GET("/tasks/:id/log", routes.GetTaskLog) // 任务日志 + // authGroup.GET("/tasks/:id/error-log", routes.GetTaskErrorLog) // 任务错误日志 + // authGroup.GET("/tasks/:id/results", routes.GetTaskResults) // 任务结果 + // authGroup.GET("/tasks/:id/results/download", routes.DownloadTaskResultsCsv) // 下载任务结果 + // authGroup.POST("/tasks/:id/restart", routes.RestartTask) // 重新开始任务 + // authGroup.POST("/tasks-cancel", routes.CancelSelectedTask) // 批量取消任务 + // authGroup.POST("/tasks-restart", routes.RestartSelectedTask) // 批量重试任务 + // } + // // 系统任务/脚本 + // { + // authGroup.PUT("/system-tasks", routes.PutSystemTask) // 运行系统任务 + // authGroup.GET("/system-scripts", routes.GetSystemScripts) // 获取系统脚本列表 + // } + // // 定时任务 + // { + // authGroup.GET("/schedules", routes.GetScheduleList) // 定时任务列表 + // authGroup.GET("/schedules/:id", routes.GetSchedule) // 定时任务详情 + // authGroup.PUT("/schedules", routes.PutSchedule) // 创建定时任务 + // authGroup.PUT("/schedules/batch", routes.PutBatchSchedules) // 批量创建定时任务 + // authGroup.POST("/schedules/:id", routes.PostSchedule) // 修改定时任务 + // authGroup.DELETE("/schedules/:id", routes.DeleteSchedule) // 删除定时任务 + // authGroup.DELETE("/schedules", routes.DeleteBatchSchedules) // 批量删除定时任务 + // authGroup.POST("/schedules/:id/disable", routes.DisableSchedule) // 禁用定时任务 + // authGroup.POST("/schedules/:id/enable", routes.EnableSchedule) // 启用定时任务 + // authGroup.POST("/schedules-set-enabled", routes.SetEnabledSchedules) // 批量设置定时任务状态 + // } + // // 用户 + // { + // authGroup.GET("/users", routes.GetUserList) // 用户列表 + // authGroup.GET("/users/:id", routes.GetUser) // 用户详情 + // authGroup.POST("/users/:id", routes.PostUser) // 更改用户 + // authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户 + // authGroup.PUT("/users-add", routes.PutUser) // 添加用户 + // authGroup.GET("/me", routes.GetMe) // 获取自己账户 + // authGroup.POST("/me", routes.PostMe) // 修改自己账户 + // authGroup.POST("/me/change-password", routes.PostMeChangePassword) // 修改自己密码 + // } + // // 系统 + // { + // authGroup.GET("/system/deps/:lang", routes.GetAllDepList) // 节点所有第三方依赖列表 + // authGroup.GET("/system/deps/:lang/:dep_name/json", routes.GetDepJson) // 节点第三方依赖JSON + // } + // // 全局变量 + // { + // authGroup.GET("/variables", routes.GetVariableList) // 列表 + // authGroup.PUT("/variable", routes.PutVariable) // 新增 + // authGroup.POST("/variable/:id", routes.PostVariable) // 修改 + // authGroup.DELETE("/variable/:id", routes.DeleteVariable) // 删除 + // } + // // 项目 + // { + // authGroup.GET("/projects", routes.GetProjectList) // 列表 + // authGroup.GET("/projects/tags", routes.GetProjectTags) // 项目标签 + // authGroup.PUT("/projects", routes.PutProject) // 修改 + // authGroup.POST("/projects/:id", routes.PostProject) // 新增 + // authGroup.DELETE("/projects/:id", routes.DeleteProject) // 删除 + // } + // // 操作 + // { + // //authGroup.GET("/actions", routes.GetActionList) // 操作列表 + // //authGroup.GET("/actions/:id", routes.GetAction) // 操作 + // authGroup.PUT("/actions", routes.PutAction) // 新增操作 + // //authGroup.POST("/actions/:id", routes.PostAction) // 修改操作 + // } + // // API Token + // { + // authGroup.GET("/tokens", routes.GetTokens) // 获取 Tokens + // authGroup.PUT("/tokens", routes.PutToken) // 添加 Token + // authGroup.DELETE("/tokens/:id", routes.DeleteToken) // 删除 Token + // } + // // 统计数据 + // authGroup.GET("/stats/home", routes.GetHomeStats) // 首页统计数据 + // // 文件 + // authGroup.GET("/file", routes.GetFile) // 获取文件 + // // Git + // authGroup.GET("/git/branches", routes.GetGitRemoteBranches) // 获取 Git 分支 + // authGroup.GET("/git/public-key", routes.GetGitSshPublicKey) // 获取 SSH 公钥 + // authGroup.GET("/git/commits", routes.GetGitCommits) // 获取 Git Commits + // authGroup.POST("/git/checkout", routes.PostGitCheckout) // 获取 Git Commits + // } + //} // 路由ping - app.GET("/ping", routes.Ping) + //app.GET("/ping", routes.Ping) // 运行服务器 - host := viper.GetString("server.host") - port := viper.GetString("server.port") - address := net.JoinHostPort(host, port) - srv := &http.Server{ - Handler: app, - Addr: address, - } - go func() { - if err := srv.ListenAndServe(); err != nil { - if err != http.ErrServerClosed { - log.Error("run server error:" + err.Error()) - } else { - log.Info("server graceful down") - } - } - }() - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) - <-quit - ctx2, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - if err := srv.Shutdown(ctx2); err != nil { - log.Error("run server error:" + err.Error()) - } } diff --git a/backend/services/api.go b/backend/services/api.go new file mode 100644 index 00000000..c9846321 --- /dev/null +++ b/backend/services/api.go @@ -0,0 +1,97 @@ +package services + +import ( + "context" + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab-core/config" + "github.com/crawlab-team/crawlab-core/middlewares" + "github.com/crawlab-team/crawlab-core/routes" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab-db/redis" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +type ApiInterface interface { + Init() + Run() +} + +type ApiService struct { + app *gin.Engine +} + +func (svc *ApiService) Init() { + // initialize config + _ = svc.initService("config", config.InitConfig) + + // initialize mongo + _ = svc.initService("mongo", mongo.InitMongo) + + // initialize redis + _ = svc.initService("redis", redis.InitRedis) + + // initialize middlewares + _ = svc.initServiceWithApp("middlewares", middlewares.InitMiddlewares) + + // initialize routes + _ = svc.initServiceWithApp("routes", routes.InitRoutes) +} + +func (svc *ApiService) Run() { + host := viper.GetString("server.host") + port := viper.GetString("server.port") + address := net.JoinHostPort(host, port) + srv := &http.Server{ + Handler: svc.app, + Addr: address, + } + go func() { + if err := srv.ListenAndServe(); err != nil { + if err != http.ErrServerClosed { + log.Error("run server error:" + err.Error()) + } else { + log.Info("server graceful down") + } + } + }() + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + ctx2, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := srv.Shutdown(ctx2); err != nil { + log.Error("run server error:" + err.Error()) + } +} + +func (svc *ApiService) initServiceWithApp(name string, fn func(app *gin.Engine) error) (err error) { + return svc.initService(name, func() error { + return fn(svc.app) + }) +} + +func (svc *ApiService) initService(name string, fn func() error) (err error) { + if err := fn(); err != nil { + log.Error(fmt.Sprintf("init %s error: %s", name, err.Error())) + _ = trace.TraceError(err) + panic(err) + } + log.Info(fmt.Sprintf("initialized %s successfully", name)) + return nil +} + +func NewApiService() *ApiService { + app := gin.New() + return &ApiService{ + app: app, + } +} diff --git a/frontend/src/views/spider/SpiderList.vue b/frontend/src/views/spider/SpiderList.vue index ce945609..10230199 100644 --- a/frontend/src/views/spider/SpiderList.vue +++ b/frontend/src/views/spider/SpiderList.vue @@ -24,7 +24,6 @@ > -