diff --git a/core/go.mod b/core/go.mod index 06e62076..8e703070 100644 --- a/core/go.mod +++ b/core/go.mod @@ -6,7 +6,6 @@ replace ( github.com/crawlab-team/crawlab/db => ../db github.com/crawlab-team/crawlab/fs => ../fs github.com/crawlab-team/crawlab/grpc => ../grpc - github.com/crawlab-team/crawlab/template-parser => ../template-parser github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs ) @@ -18,13 +17,11 @@ require ( github.com/crawlab-team/crawlab/db v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/fs v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/grpc v0.0.0-20240701075920-023ba275664a - github.com/crawlab-team/crawlab/template-parser v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/trace v0.0.0-20240701075920-023ba275664a github.com/crawlab-team/crawlab/vcs v0.0.0-20240701075920-023ba275664a github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.7.0 - github.com/gavv/httpexpect/v2 v2.16.0 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 @@ -32,13 +29,11 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 github.com/hashicorp/go-uuid v1.0.3 github.com/imroc/req v0.3.2 - github.com/matcornic/hermes/v2 v2.1.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 github.com/segmentio/kafka-go v0.4.39 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.3.0 github.com/spf13/viper v1.19.0 @@ -57,16 +52,10 @@ require ( ) require ( - github.com/Masterminds/semver v1.4.2 // indirect - github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/PuerkitoBio/goquery v1.9.2 // indirect - github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect - github.com/ajg/form v1.5.1 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/aokoli/goutils v1.0.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -76,8 +65,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/fatih/structs v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect @@ -89,28 +76,22 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect + github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect - github.com/gorilla/css v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.2.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.10.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect @@ -120,26 +101,19 @@ require ( github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.4 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/olekukonko/tablewriter v0.0.1 // indirect github.com/onsi/gomega v1.30.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/sanity-io/litter v1.5.5 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect @@ -148,28 +122,17 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.34.0 // indirect - github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect - github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/yudai/gojsondiff v1.0.0 // indirect - github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect @@ -192,8 +155,6 @@ require ( google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - moul.io/http2curl/v2 v2.3.0 // indirect ) diff --git a/core/go.sum b/core/go.sum index 9bfd5e46..5fd1eb82 100644 --- a/core/go.sum +++ b/core/go.sum @@ -69,6 +69,7 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0k github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= @@ -83,6 +84,7 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -112,6 +114,7 @@ github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -289,6 +292,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 h1:saBP362Qm7zDdDXqv61kI4rzhmLFq3Z1gx34xpl6cWE= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -393,6 +398,7 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.2 h1:M/JkeU6RPmX+WYvT2vaaOL0K+q8ufL5LxwvJc4xeB4o= +github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -409,6 +415,7 @@ github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -426,20 +433,24 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -467,6 +478,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -556,6 +568,7 @@ github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -763,6 +776,7 @@ go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQc go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= +go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -842,6 +856,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -924,6 +939,7 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1043,6 +1059,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1051,6 +1068,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1066,6 +1084,7 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/core/models/models/v2/node_v2.go b/core/models/models/v2/node_v2.go index 6d5a8442..968b9bc9 100644 --- a/core/models/models/v2/node_v2.go +++ b/core/models/models/v2/node_v2.go @@ -10,7 +10,6 @@ type NodeV2 struct { Key string `json:"key" bson:"key"` Name string `json:"name" bson:"name"` Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` Hostname string `json:"hostname" bson:"hostname"` Description string `json:"description" bson:"description"` diff --git a/core/models/models/v2/spider_v2.go b/core/models/models/v2/spider_v2.go index ac3bf300..24b4cb05 100644 --- a/core/models/models/v2/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -8,7 +8,6 @@ type SpiderV2 struct { any `collection:"spiders"` BaseModelV2[SpiderV2] `bson:",inline"` Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id ColName string `json:"col_name,omitempty" bson:"-"` // data collection name DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id diff --git a/core/models/models/v2/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go index 5456946d..65459244 100644 --- a/core/models/models/v2/task_stat_v2.go +++ b/core/models/models/v2/task_stat_v2.go @@ -7,12 +7,10 @@ import ( type TaskStatV2 struct { any `collection:"task_stats"` BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` } diff --git a/core/notification/im.go b/core/notification/im.go index 74856f81..12f0162b 100644 --- a/core/notification/im.go +++ b/core/notification/im.go @@ -14,11 +14,11 @@ type ResBody struct { ErrMsg string `json:"errmsg"` } -func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) error { +func SendIMNotification(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) error { // TODO: compatibility with different IM providers switch ch.Provider { case ChannelIMProviderLark: - return sendImLark(ch, content) + return sendImLark(ch, title, content) } // request header @@ -30,7 +30,7 @@ func SendIMNotification(s *models.NotificationSettingV2, ch *models.Notification data := req.Param{ "msgtype": "markdown", "markdown": req.Param{ - "title": s.Title, + "title": title, "text": content, "content": content, }, @@ -99,11 +99,17 @@ func performIMRequest(webhookUrl string, data req.Param) error { return nil } -func sendImLark(ch *models.NotificationChannelV2, content string) error { +func sendImLark(ch *models.NotificationChannelV2, title, content string) error { // request header data := req.Param{ "msg_type": "interactive", "card": req.Param{ + "header": req.Param{ + "title": req.Param{ + "tag": "plain_text", + "content": title, + }, + }, "elements": []req.Param{ { "tag": "markdown", diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 2f2118ac..343d3023 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -7,6 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" + "github.com/gomarkdown/markdown" "go.mongodb.org/mongo-driver/bson/primitive" "regexp" "strings" @@ -18,6 +19,7 @@ type ServiceV2 struct { } func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { + title := s.Title content := svc.getContent(s, args...) wg := sync.WaitGroup{} @@ -32,30 +34,30 @@ func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { } switch ch.Type { case TypeMail: - svc.SendMail(s, ch, content) + svc.SendMail(s, ch, title, content) case TypeIM: - svc.SendIM(s, ch, content) + svc.SendIM(s, ch, title, content) } }(chId) } wg.Wait() } -func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { // TODO: parse to/cc/bcc mailTo := s.MailTo mailCc := s.MailCc mailBcc := s.MailBcc // send mail - err := SendMail(s, ch, mailTo, mailCc, mailBcc, s.Title, content) + err := SendMail(s, ch, mailTo, mailCc, mailBcc, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mail error: %v", err) } } -func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, content string) { - err := SendIMNotification(s, ch, content) +func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { + err := SendIMNotification(s, ch, title, content) if err != nil { log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } @@ -68,7 +70,9 @@ func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, args ...any) ( switch s.TemplateMode { case constants.NotificationTemplateModeMarkdown: variables := svc.parseTemplateVariables(s.TemplateMarkdown) - return svc.getTaskContent(s.TemplateMarkdown, variables, vd) + content = svc.getTaskContent(s.TemplateMarkdown, variables, vd) + content = svc.convertMarkdownToHtml(content) + return content case constants.NotificationTemplateModeRichText: variables := svc.parseTemplateVariables(s.TemplateRichText) return svc.getTaskContent(s.TemplateRichText, variables, vd) @@ -87,6 +91,10 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific for _, v := range variables { switch v.Category { case "task": + if vd.Task == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Id.Hex()) @@ -106,20 +114,46 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Mode) case "priority": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Priority)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Task.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.UpdatedBy)) + } + + case "task_stat": + if vd.TaskStat == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "start_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.StartTs)) + case "end_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.EndTs)) + case "wait_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.WaitDuration/1000)) + case "runtime_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.RuntimeDuration/1000)) + case "total_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.TotalDuration/1000)) + case "result_count": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.TaskStat.ResultCount)) } case "spider": + if vd.Spider == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Id.Hex()) case "name": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Name) - case "type": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Type) case "description": content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Description) case "mode": @@ -130,13 +164,21 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Param) case "priority": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Spider.Priority)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.UpdatedBy)) } case "node": + if vd.Node == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Id.Hex()) @@ -146,8 +188,8 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Name) case "ip": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Ip) - case "port": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Port) + case "mac": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Mac) case "hostname": content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Hostname) case "description": @@ -159,18 +201,26 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific case "active": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Active)) case "active_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.ActiveAt.Format("2006-01-02 15:04:05")) + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.ActiveAt)) case "available_runners": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.AvailableRunners)) case "max_runners": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.MaxRunners)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Node.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.UpdatedBy)) } case "schedule": + if vd.Schedule == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } switch v.Name { case "id": content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Id.Hex()) @@ -190,10 +240,14 @@ func (svc *ServiceV2) getTaskContent(template string, variables []entity.Notific content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Schedule.Priority)) case "enabled": content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Schedule.Enabled)) - case "created_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.CreatedAt.Format(time.DateTime)) - case "updated_at": - content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.UpdatedAt.Format(time.DateTime)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.UpdatedBy)) } } } @@ -248,8 +302,27 @@ func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entit return variables } -func (svc *ServiceV2) getUserById(id primitive.ObjectID) { +func (svc *ServiceV2) getUsernameById(id primitive.ObjectID) (username string) { + if id.IsZero() { + return "" + } + u, err := service.NewModelServiceV2[models.UserV2]().GetById(id) + if err != nil { + log.Errorf("[NotificationServiceV2] get user error: %v", err) + return "" + } + return u.Username +} +func (svc *ServiceV2) getFormattedTime(t time.Time) (res string) { + if t.IsZero() { + return "N/A" + } + return t.Local().Format(time.DateTime) +} + +func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { + return string(markdown.ToHTML([]byte(content), nil, nil)) } func newNotificationServiceV2() *ServiceV2 { diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index b3c7136a..8fcba6c9 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -41,7 +41,7 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task - mainTask := &models2.TaskV2{ + t := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, NodeIds: opts.NodeIds, @@ -49,66 +49,38 @@ func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.Spider Param: opts.Param, ScheduleId: opts.ScheduleId, Priority: opts.Priority, - UserId: opts.UserId, } - mainTask.SetId(primitive.NewObjectID()) + t.SetId(primitive.NewObjectID()) // normalize - if mainTask.Mode == "" { - mainTask.Mode = s.Mode + if t.Mode == "" { + t.Mode = s.Mode } - if mainTask.NodeIds == nil { - mainTask.NodeIds = s.NodeIds + if t.NodeIds == nil { + t.NodeIds = s.NodeIds } - if mainTask.Cmd == "" { - mainTask.Cmd = s.Cmd + if t.Cmd == "" { + t.Cmd = s.Cmd } - if mainTask.Param == "" { - mainTask.Param = s.Param + if t.Param == "" { + t.Param = s.Param } - if mainTask.Priority == 0 { - mainTask.Priority = s.Priority + if t.Priority == 0 { + t.Priority = s.Priority } - if svc.isMultiTask(opts) { - // multi tasks - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - for _, nodeId := range nodeIds { - t := &models2.TaskV2{ - SpiderId: s.Id, - Mode: opts.Mode, - Cmd: opts.Cmd, - Param: opts.Param, - NodeId: nodeId, - ScheduleId: opts.ScheduleId, - Priority: opts.Priority, - UserId: opts.UserId, - } - t.SetId(primitive.NewObjectID()) - t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) - } - } else { - // single task - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - if len(nodeIds) > 0 { - mainTask.NodeId = nodeIds[0] - } - t2, err := svc.schedulerSvc.Enqueue(mainTask, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err } + if len(nodeIds) > 0 { + t.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) return taskIds, nil } diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index b4baac24..3788661f 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -522,14 +522,12 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { } } + // update stats + r._updateTaskStat(status) + r._updateSpiderStat(status) + // send notification go r.sendNotification() - - // update stats - go func() { - r._updateTaskStat(status) - r._updateSpiderStat(status) - }() } // get task @@ -579,15 +577,15 @@ func (r *RunnerV2) _updateTaskStat(status string) { // do nothing case constants.TaskStatusRunning: ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: if ts.StartTs.IsZero() { ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } ts.EndTs = time.Now() ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() - ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index 81d5907e..ef4d7ac7 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -39,12 +39,12 @@ func (svc *ServiceV2) Start() { func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { // set task status t.Status = constants.TaskStatusPending - t.SetCreatedBy(by) + t.SetCreated(by) t.SetUpdated(by) // add task taskModelSvc := service.NewModelServiceV2[models2.TaskV2]() - _, err = taskModelSvc.InsertOne(*t) + id, err := taskModelSvc.InsertOne(*t) if err != nil { return nil, err } @@ -54,13 +54,15 @@ func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *mod Priority: t.Priority, NodeId: t.NodeId, } - tq.SetId(t.Id) + tq.SetId(id) + tq.SetCreated(by) + tq.SetUpdated(by) // task stat - ts := models2.TaskStatV2{ - CreateTs: time.Now(), - } - ts.SetId(t.Id) + ts := models2.TaskStatV2{} + ts.SetId(id) + ts.SetCreated(by) + ts.SetUpdated(by) // enqueue task _, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq) diff --git a/go.work.sum b/go.work.sum index b4321010..99ae07af 100644 --- a/go.work.sum +++ b/go.work.sum @@ -582,6 +582,7 @@ github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/fasthttp/websocket v1.4.3-rc.6 h1:omHqsl8j+KXpmzRjF8bmzOSYJ8GnS0E3efi1wYT+niY= github.com/fasthttp/websocket v1.4.3-rc.6/go.mod h1:43W9OM2T8FeXpCWMsBd9Cb7nE2CACNqNvCqQCoty/Lc= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -607,7 +608,6 @@ github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= @@ -723,7 +723,6 @@ github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LF github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= @@ -765,6 +764,7 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=