diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 24cc31c5..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,56 +0,0 @@ -pipeline { - agent { - node { - label 'crawlab' - } - } - - stages { - stage('Setup') { - steps { - echo "Running Setup..." - script { - if (env.GIT_BRANCH == 'develop') { - env.TAG = 'develop' - env.DOCKERFILE = 'Dockerfile.local' - } else if (env.GIT_BRANCH == 'master') { - env.TAG = 'master' - env.DOCKERFILE = 'Dockerfile.local' - } - } - } - } - stage('Build') { - steps { - echo "Building..." - sh """ - docker build -t tikazyq/crawlab:${ENV:TAG} -f ${ENV:DOCKERFILE} . - """ - } - } - stage('Test') { - steps { - echo 'Testing..' - } - } - stage('Deploy') { - steps { - echo 'Deploying....' - sh """ - # 重启docker compose - cd ./jenkins/${ENV:GIT_BRANCH} - docker-compose down | true - docker-compose up -d | true - """ - } - } - stage('Cleanup') { - steps { - echo 'Cleanup...' - sh """ - docker rmi -f `docker images | grep '' | grep -v IMAGE | awk '{ print \$3 }' | xargs` - """ - } - } - } -} \ No newline at end of file diff --git a/backend/bin/update-deps.sh b/backend/bin/update-deps.sh index a8ed28fd..38d6eab9 100755 --- a/backend/bin/update-deps.sh +++ b/backend/bin/update-deps.sh @@ -1,4 +1,4 @@ #!/bin/bash -go get -u github.com/crawlab-team/crawlab-core@main +go get -u github.com/crawlab-team/crawlab/core@main go mod tidy diff --git a/backend/conf/config.yml b/backend/conf/config.yml index 322ade50..dbcfed87 100755 --- a/backend/conf/config.yml +++ b/backend/conf/config.yml @@ -1,3 +1,27 @@ -info: - edition: global.edition.community - version: v0.6.3 +# Crawlab Configuration File +edition: global.edition.community +version: v0.6.3 + +mongo: + host: localhost + port: 27017 + db: crawlab_test + username: "" + password: "" + authSource: "admin" + +server: + host: 0.0.0.0 + port: 8000 + +grpc: + address: localhost:9666 + server: + address: 0.0.0.0:9666 + authKey: Crawlab2021! + +api: + endpoint: http://localhost:8000 + +log: + path: /var/log/crawlab diff --git a/backend/go.mod b/backend/go.mod index 861f2919..7484cf7c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,59 +1,75 @@ module crawlab -go 1.18 +go 1.22 -require github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203 +replace ( + github.com/crawlab-team/crawlab/core => ../core + github.com/crawlab-team/crawlab/db => ../db + github.com/crawlab-team/crawlab/fs => ../fs + github.com/crawlab-team/crawlab/grpc => ../grpc + github.com/crawlab-team/crawlab/template-parser => ../template-parser + github.com/crawlab-team/crawlab/trace => ../trace + github.com/crawlab-team/crawlab/vcs => ../vcs +) + +require github.com/crawlab-team/crawlab/core v0.0.0-20240614095218-7b4ee8399ab0 require ( + dario.cat/mergo v1.0.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/sirupsen/logrus v1.9.0 // indirect +) + +require ( + cloud.google.com/go/compute/metadata v0.3.0 // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec // indirect + github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/PuerkitoBio/goquery v1.8.0 // indirect github.com/ReneKroon/ttlcache v1.7.0 // indirect - github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aokoli/goutils v1.0.1 // indirect github.com/apex/log v1.9.0 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.1.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/cloudflare/circl v1.3.3 // indirect - github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 // indirect - github.com/crawlab-team/crawlab/fs v0.6.3 // indirect - github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c // indirect - github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 // indirect - github.com/crawlab-team/crawlab/vcs v0.1.1 // indirect - github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 // indirect - github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/crawlab-team/crawlab/db v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/fs v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/grpc v0.0.0-20240614111723-e5b20af9a40b // indirect + github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/crawlab/vcs v0.0.0-20240614095218-7b4ee8399ab0 // indirect + github.com/crawlab-team/goseaweedfs v0.6.3 // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect - github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect - github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect - github.com/elastic/go-elasticsearch/v8 v8.7.0 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.4.1 // indirect - github.com/go-git/go-git/v5 v5.7.0 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.3 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect - github.com/hashicorp/go-uuid v1.0.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -63,55 +79,50 @@ require ( github.com/jackc/pgconn v1.11.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.2.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect github.com/jackc/pgtype v1.10.0 // indirect - github.com/jackc/pgx/v4 v4.15.0 // indirect + github.com/jackc/pgx/v4 v4.18.2 // indirect github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/linxGnu/gumble v1.0.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/matcornic/hermes/v2 v2.1.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/olekukonko/tablewriter v0.0.1 // indirect - github.com/olivere/elastic/v7 v7.0.15 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect github.com/robfig/cron/v3 v3.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/satori/go.uuid v1.2.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/segmentio/kafka-go v0.4.39 // indirect - github.com/sergi/go-diff v1.3.1 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/skeema/knownhosts v1.1.1 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/skeema/knownhosts v1.2.2 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/cobra v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.10.0 // indirect + github.com/spf13/viper v1.19.0 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/thoas/go-funk v0.9.1 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect @@ -122,29 +133,34 @@ require ( github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.0.2 // indirect - github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect - go.mongodb.org/mongo-driver v1.8.0 // indirect + go.mongodb.org/mongo-driver v1.15.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.10.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect - golang.org/x/tools v0.10.0 // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect - google.golang.org/grpc v1.42.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/go.mod.dev b/backend/go.mod.dev index f45ffe25..89afabb1 100644 --- a/backend/go.mod.dev +++ b/backend/go.mod.dev @@ -3,7 +3,7 @@ module crawlab go 1.16 replace ( - github.com/crawlab-team/crawlab-core => ../../crawlab-core + github.com/crawlab-team/crawlab/core => ../../crawlab-core github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs github.com/crawlab-team/crawlab/fs => ../../crawlab-fs github.com/crawlab-team/crawlab/db => ../../crawlab-db @@ -11,7 +11,7 @@ replace ( require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 + github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200 github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.mod.local b/backend/go.mod.local index ff0ab04c..fd8c268e 100644 --- a/backend/go.mod.local +++ b/backend/go.mod.local @@ -3,7 +3,7 @@ module crawlab go 1.16 replace ( - github.com/crawlab-team/crawlab-core => /libs/crawlab-team/crawlab-core + github.com/crawlab-team/crawlab/core => /libs/crawlab-team/crawlab-core github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs github.com/crawlab-team/crawlab/fs => /libs/crawlab-team/crawlab-fs github.com/crawlab-team/crawlab/db => /libs/crawlab-team/crawlab-db @@ -11,7 +11,7 @@ replace ( require ( github.com/apex/log v1.9.0 - github.com/crawlab-team/crawlab-core v0.6.0-beta.20211230.1200 + github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200 github.com/crawlab-team/crawlab/vcs v0.1.1 github.com/gin-gonic/gin v1.7.1 github.com/spf13/cobra v1.1.3 diff --git a/backend/go.sum b/backend/go.sum index 4e7bd4b3..4407623e 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -26,7 +26,6 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -34,9 +33,10 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -47,6 +47,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -61,24 +63,28 @@ github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec h1:vV3RryLxt42+ZIVOFbYJCH1jsZNTNmj2NYru5zfx+4E= -github.com/ProtonMail/go-crypto v0.0.0-20230626094100-7e9e0395ebec/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= -github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= -github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= +github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= @@ -93,16 +99,13 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= @@ -123,8 +126,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -137,58 +141,26 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230708092022-c40c0fdc2c28 h1:x2mzygXuU6mjlFk8zr8C45VvYEJn/wF1AgrGj5wfW8k= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230708092022-c40c0fdc2c28/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230726071808-25ed70356f13 h1:fP4OGh4q4hRLj9xOxdxHxJ3kMx8cxxcHMp/DxtJukJ4= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230726071808-25ed70356f13/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230804030437-664646cf1665 h1:LAX9MN0NVIZHkyFDNH+3LvsD5AgoV7/FsbdDjVQr3OQ= -github.com/crawlab-team/crawlab-core v0.6.3-0.20230804030437-664646cf1665/go.mod h1:rrcJUzh/PHR2WKBaahsR8baRstCdnEEFLx+pOA8IenU= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653 h1:uOCkVHVqr2KGk/lv68LbBM3uQoJT6p4ywXInPitM3wo= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231021045242-07956209f653/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203 h1:nyANfzoPgTSYJxuTye1uj44An8Cjou9QmcKRES7Gdwg= -github.com/crawlab-team/crawlab-core v0.6.3-0.20231031044528-37e6d73eb203/go.mod h1:HNuAjSVVZpHhpyUP4k1F2YKxEiarZESFolSyx7YjgZ8= -github.com/crawlab-team/crawlab/db v0.6.0-1/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484 h1:1CXWC3lYcVWcgPRc3PNKzZ3fcfX5WZ/V8xwzHEMUFHQ= -github.com/crawlab-team/crawlab/db v0.6.0-beta.20220417.1300.0.20221226064900-5a357ee73484/go.mod h1:gfeF0nAnFuup6iYvgHkY0in/HpO/+JktXqVNMdhoxhU= -github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73 h1:xIgfVPa3ZJWC72Y57oHS41n4jRtGZPn1YDEYBgMj2EU= -github.com/crawlab-team/crawlab/fs v0.6.0-beta.20211101.1940.0.20221218100256-a28d12756f73/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= -github.com/crawlab-team/crawlab/fs v0.6.3 h1:mS91sYu+tOPavjYvt4CZ8YwY5okEiwCAuyx/5RIbXJY= -github.com/crawlab-team/crawlab/fs v0.6.3/go.mod h1:y9YhLLR3GuPrDuPKe7ZuiHCITK9K2IcI8nlznF8YIEc= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c h1:jX0iax3WHwomWGQVWrCTy8a4zYDsKKyuspP3+04XCcU= -github.com/crawlab-team/crawlab-grpc v0.6.0-beta.20211219.1930.0.20221020032435-afa1c691f73c/go.mod h1:Bq2Pm967EYWbjhP5Ghc4DV2LZgbOLMzLftJXDJYz/gs= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185 h1:A/XSUuGgGMn+z+lFd2ye2ClgIKhDZYUerhOL5jePQhU= -github.com/crawlab-team/crawlab-vcs v0.6.2-0.20230629045457-afe0be0e2185/go.mod h1:YHMYUEoSqfXUZHsWW/M/DaLh/zOpRtiElaRWcrGyv/I= -github.com/crawlab-team/crawlab/vcs v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= -github.com/crawlab-team/crawlab/vcs v0.1.1 h1:AecgAOld+ZrSVvujyhK3zoaOmViGKHSCT8/weJ7adB8= -github.com/crawlab-team/crawlab/vcs v0.1.1/go.mod h1:4U+pWgLhRuD3pbXHonwcaHcW+y8AUqyOfKoZnvKwCug= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0= -github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA= -github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e h1:Gwg9kKNZUAI4bSssomlzXCN01Q3MapgwQOCeOxGX/NU= -github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e/go.mod h1:FImmp7V0VcIdTRM68F3PQUqewzuShvUjYBhAHRjD1Aw= +github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= -github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= -github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= -github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= -github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -205,28 +177,35 @@ github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gavv/httpexpect/v2 v2.2.0 h1:0VwaEBmQaNFHX9x591A8Up+8shCwdF/nF0qlRd/nI48= +github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= +github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= -github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE= -github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -237,33 +216,35 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -294,12 +275,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -314,8 +295,10 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -339,8 +322,8 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -351,15 +334,10 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -373,27 +351,23 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -405,7 +379,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/imkira/go-interpol v1.0.0 h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E= +github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -464,13 +439,6 @@ github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= -github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -484,23 +452,22 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -510,25 +477,16 @@ github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M= -github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= -github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -536,6 +494,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -548,7 +508,6 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -556,19 +515,18 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -576,27 +534,23 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olivere/elastic/v7 v7.0.15 h1:v7kX5S+oMFfYKS4ZyzD37GH6lfZSpBo9atynRwBUywE= -github.com/olivere/elastic/v7 v7.0.15/go.mod h1:+FgncZ8ho1QF3NlBo77XbuoTKYHhvEOfFZKIAfHnnDE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -604,37 +558,34 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -643,17 +594,21 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= +github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/segmentio/kafka-go v0.4.39 h1:75smaomhvkYRwtuOwqLsdhgCG30B82NsbdkdDfFbvrw= github.com/segmentio/kafka-go v0.4.39/go.mod h1:T0MLgygYvmqmBvC+s8aCcbVNfJN4znVne5j0Pzowp/Q= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -666,35 +621,33 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= -github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.2.2 h1:Iug2P4fLmDw9f41PB6thxUkNUkJzB5i+1/exaj40L3A= +github.com/skeema/knownhosts v1.2.2/go.mod h1:xYbVRSPxqBZFrdmDyMmsOs+uX1UZC3nTN3ThzgDxUwo= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -702,6 +655,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -712,14 +667,14 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -731,7 +686,6 @@ github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -740,8 +694,9 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/fasthttp v1.9.0 h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= +github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe h1:9YnI5plmy+ad6BM+JCLJb2ZV7/TNiE5l7SNKfumYKgc= @@ -750,23 +705,28 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonschema v1.1.0 h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -776,15 +736,13 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.mongodb.org/mongo-driver v1.8.0 h1:R/P/JJzu8LJvJ1lDfph9GLNIKQxEtIHFfnUUUve35zY= -go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -792,20 +750,29 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -828,7 +795,6 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -837,8 +803,8 @@ golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -850,6 +816,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -879,16 +847,14 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -934,8 +900,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -952,8 +918,9 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -967,14 +934,13 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1044,22 +1010,22 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1069,27 +1035,25 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1101,7 +1065,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1143,8 +1106,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1189,7 +1152,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1253,8 +1215,9 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1281,8 +1244,9 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1297,9 +1261,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1313,25 +1276,22 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1363,7 +1323,8 @@ modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= -moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo= +moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= +moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/backend/main.go b/backend/main.go index 013590cc..fe304832 100644 --- a/backend/main.go +++ b/backend/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/crawlab-team/crawlab-core/cmd" + "github.com/crawlab-team/crawlab/core/cmd" ) func main() { diff --git a/bin/gen-ver.sh b/bin/gen-ver.sh new file mode 100755 index 00000000..ab810167 --- /dev/null +++ b/bin/gen-ver.sh @@ -0,0 +1,4 @@ +#!/bin/bash +COMMIT_HASH=$(git rev-parse HEAD) +TIMESTAMP=$(date +%Y%m%d%H%M%S) +echo "v0.0.0-$TIMESTAMP-$COMMIT_HASH" \ No newline at end of file diff --git a/core/apps/api.go b/core/apps/api.go deleted file mode 100644 index 94e887c3..00000000 --- a/core/apps/api.go +++ /dev/null @@ -1,126 +0,0 @@ -package apps - -import ( - "context" - "errors" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net" - "net/http" - "time" -) - -func init() { - // set gin mode - if viper.GetString("gin.mode") == "" { - gin.SetMode(gin.ReleaseMode) - } else { - gin.SetMode(viper.GetString("gin.mode")) - } -} - -type Api struct { - // dependencies - interfaces.WithConfigPath - - // internals - app *gin.Engine - ln net.Listener - srv *http.Server - ready bool -} - -func (app *Api) Init() { - // initialize controllers - _ = initModule("controllers", controllers.InitControllers) - - // initialize middlewares - _ = app.initModuleWithApp("middlewares", middlewares.InitMiddlewares) - - // initialize routes - _ = app.initModuleWithApp("routes", routes.InitRoutes) -} - -func (app *Api) Start() { - // address - host := viper.GetString("server.host") - port := viper.GetString("server.port") - address := net.JoinHostPort(host, port) - - // http server - app.srv = &http.Server{ - Handler: app.app, - Addr: address, - } - - // listen - var err error - app.ln, err = net.Listen("tcp", address) - if err != nil { - panic(err) - } - app.ready = true - - // serve - if err := http.Serve(app.ln, app.app); err != nil { - if !errors.Is(err, http.ErrServerClosed) { - log.Error("run server error:" + err.Error()) - } else { - log.Info("server graceful down") - } - } -} - -func (app *Api) Wait() { - DefaultWait() -} - -func (app *Api) Stop() { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := app.srv.Shutdown(ctx); err != nil { - log.Error("run server error:" + err.Error()) - } -} - -func (app *Api) GetGinEngine() *gin.Engine { - return app.app -} - -func (app *Api) GetHttpServer() *http.Server { - return app.srv -} - -func (app *Api) Ready() (ok bool) { - return app.ready -} - -func (app *Api) initModuleWithApp(name string, fn func(app *gin.Engine) error) (err error) { - return initModule(name, func() error { - return fn(app.app) - }) -} - -func NewApi() *Api { - api := &Api{ - app: gin.New(), - } - api.Init() - return api -} - -var api *Api - -func GetApi() *Api { - if api != nil { - return api - } - api = NewApi() - return api -} diff --git a/core/apps/api_v2.go b/core/apps/api_v2.go index 966b1a08..16fe102c 100644 --- a/core/apps/api_v2.go +++ b/core/apps/api_v2.go @@ -7,6 +7,7 @@ import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" "github.com/spf13/viper" "net" @@ -73,7 +74,7 @@ func (app *ApiV2) Start() { } func (app *ApiV2) Wait() { - DefaultWait() + utils.DefaultWait() } func (app *ApiV2) Stop() { diff --git a/core/apps/server.go b/core/apps/server.go deleted file mode 100644 index bf13256c..00000000 --- a/core/apps/server.go +++ /dev/null @@ -1,149 +0,0 @@ -package apps - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/node/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/spf13/viper" - "net/http" - _ "net/http/pprof" -) - -func init() { - injectModules() -} - -type Server struct { - // settings - grpcAddress interfaces.Address - - // dependencies - interfaces.WithConfigPath - nodeSvc interfaces.NodeService - - // modules - api *Api - dck *Docker - - // internals - quit chan int -} - -func (app *Server) SetGrpcAddress(address interfaces.Address) { - app.grpcAddress = address -} - -func (app *Server) GetApi() (api ApiApp) { - return app.api -} - -func (app *Server) GetNodeService() (svc interfaces.NodeService) { - return app.nodeSvc -} - -func (app *Server) Init() { - // log node info - app.logNodeInfo() - - if utils.IsMaster() { - - // initialize controllers - if err := controllers.InitControllers(); err != nil { - panic(err) - } - } - - // pprof - app.initPprof() -} - -func (app *Server) Start() { - if utils.IsMaster() { - // start docker app - if utils.IsDocker() { - go app.dck.Start() - } - - // start api - go app.api.Start() - } - - // start node service - go app.nodeSvc.Start() -} - -func (app *Server) Wait() { - <-app.quit -} - -func (app *Server) Stop() { - app.api.Stop() - app.quit <- 1 -} - -func (app *Server) logNodeInfo() { - log.Infof("current node type: %s", utils.GetNodeType()) - if utils.IsDocker() { - log.Infof("running in docker container") - } -} - -func (app *Server) initPprof() { - if viper.GetBool("pprof") { - go func() { - fmt.Println(http.ListenAndServe("0.0.0.0:6060", nil)) - }() - } -} - -func NewServer() (app NodeApp) { - // server - svr := &Server{ - WithConfigPath: config.NewConfigPathService(), - quit: make(chan int, 1), - } - - // service options - var svcOpts []service.Option - if svr.grpcAddress != nil { - svcOpts = append(svcOpts, service.WithAddress(svr.grpcAddress)) - } - - // master modules - if utils.IsMaster() { - // api - svr.api = GetApi() - - // docker - if utils.IsDocker() { - svr.dck = GetDocker(svr) - } - } - - // node service - var err error - if utils.IsMaster() { - svr.nodeSvc, err = service.NewMasterService(svcOpts...) - } else { - svr.nodeSvc, err = service.NewWorkerService(svcOpts...) - } - if err != nil { - panic(err) - } - - return svr -} - -var server NodeApp - -func GetServer() NodeApp { - if server != nil { - return server - } - server = NewServer() - return server -} diff --git a/core/apps/server_test.go b/core/apps/server_test.go deleted file mode 100644 index a2db6e7f..00000000 --- a/core/apps/server_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package apps - -import ( - "fmt" - "github.com/imroc/req" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "os" - "testing" - "time" -) - -func init() { - _ = os.Setenv("CRAWLAB_DEMO", "false") -} - -func TestServer_Start(t *testing.T) { - svr := GetServer() - - // start - go Start(svr) - time.Sleep(1 * time.Second) - - res, err := req.Get(fmt.Sprintf("http://localhost:%s/system-info", viper.GetString("server.port"))) - require.Nil(t, err) - resStr, err := res.ToString() - require.Nil(t, err) - require.Contains(t, resStr, "success") -} diff --git a/core/apps/server_v2.go b/core/apps/server_v2.go index fdc3ee0a..80a67dfa 100644 --- a/core/apps/server_v2.go +++ b/core/apps/server_v2.go @@ -104,9 +104,9 @@ func NewServerV2() (app NodeApp) { // node service var err error if utils.IsMaster() { - svr.nodeSvc, err = service.NewMasterServiceV2() + svr.nodeSvc, err = service.GetMasterServiceV2() } else { - svr.nodeSvc, err = service.NewWorkerServiceV2() + svr.nodeSvc, err = service.GetWorkerServiceV2() } if err != nil { panic(err) diff --git a/core/apps/utils.go b/core/apps/utils.go index cd10edab..a524b467 100644 --- a/core/apps/utils.go +++ b/core/apps/utils.go @@ -3,21 +3,6 @@ package apps import ( "fmt" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/color" - "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" - grpcserver "github.com/crawlab-team/crawlab/core/grpc/server" - modelsclient "github.com/crawlab-team/crawlab/core/models/client" - modelsservice "github.com/crawlab-team/crawlab/core/models/service" - nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/schedule" - "github.com/crawlab-team/crawlab/core/spider/admin" - "github.com/crawlab-team/crawlab/core/stats" - "github.com/crawlab-team/crawlab/core/task/handler" - "github.com/crawlab-team/crawlab/core/task/scheduler" - taskstats "github.com/crawlab-team/crawlab/core/task/stats" - "github.com/crawlab-team/crawlab/core/user" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" ) @@ -46,47 +31,3 @@ func initModule(name string, fn func() error) (err error) { log.Info(fmt.Sprintf("initialized %s successfully", name)) return nil } - -func initApp(name string, app App) { - _ = initModule(name, func() error { - app.Init() - return nil - }) -} - -var injectors = []interface{}{ - modelsservice.GetService, - modelsclient.NewServiceDelegate, - modelsclient.NewNodeServiceDelegate, - modelsclient.NewSpiderServiceDelegate, - modelsclient.NewTaskServiceDelegate, - modelsclient.NewTaskStatServiceDelegate, - modelsclient.NewEnvironmentServiceDelegate, - grpcclient.NewClient, - grpcclient.NewPool, - grpcserver.GetServer, - grpcserver.NewModelDelegateServer, - grpcserver.NewModelBaseServiceServer, - grpcserver.NewNodeServer, - grpcserver.NewTaskServer, - grpcserver.NewMessageServer, - config.NewConfigPathService, - user.GetUserService, - schedule.GetScheduleService, - admin.GetSpiderAdminService, - stats.GetStatsService, - nodeconfig.NewNodeConfigService, - taskstats.GetTaskStatsService, - color.NewService, - scheduler.GetTaskSchedulerService, - handler.GetTaskHandlerService, -} - -func injectModules() { - c := container.GetContainer() - for _, injector := range injectors { - if err := c.Provide(injector); err != nil { - panic(err) - } - } -} diff --git a/core/cmd/root.go b/core/cmd/root.go index d9a11d2f..108e016b 100644 --- a/core/cmd/root.go +++ b/core/cmd/root.go @@ -23,11 +23,6 @@ func Execute() error { return rootCmd.Execute() } -// GetRootCmd get rootCmd instance -func GetRootCmd() *cobra.Command { - return rootCmd -} - func init() { rootCmd.PersistentFlags().StringVar(&cfgFile, "c", "", "Use Custom Config File") } diff --git a/core/cmd/server.go b/core/cmd/server.go index 40f976ae..afa7cd42 100644 --- a/core/cmd/server.go +++ b/core/cmd/server.go @@ -16,7 +16,6 @@ var serverCmd = &cobra.Command{ Long: `Start Crawlab node server that can serve as API, task scheduler, task runner, etc.`, Run: func(cmd *cobra.Command, args []string) { // app - //svr := apps.GetServer(opts...) svr := apps.GetServerV2() // start diff --git a/core/config/config.go b/core/config/config.go index 1caa14d7..1eace82a 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -1,7 +1,6 @@ package config import ( - "bytes" "github.com/apex/log" "github.com/crawlab-team/crawlab/trace" "github.com/fsnotify/fsnotify" @@ -63,14 +62,10 @@ func (c *Config) Init() (err error) { replacer := strings.NewReplacer(".", "_") viper.SetEnvKeyReplacer(replacer) - // read default config - defaultConfBuf := bytes.NewBufferString(DefaultConfigYaml) - if err := viper.ReadConfig(defaultConfBuf); err != nil { - return trace.TraceError(err) - } - - // merge config - if err := viper.MergeInConfig(); err != nil { // viper parsing config file + // read in config + if err := viper.ReadInConfig(); err != nil { + log.Errorf("Error reading config file, %s", err) + trace.PrintError(err) return err } diff --git a/core/config/default_config.go b/core/config/default_config.go deleted file mode 100644 index 8f0e7ea7..00000000 --- a/core/config/default_config.go +++ /dev/null @@ -1,40 +0,0 @@ -package config - -var DefaultConfigYaml = ` -info: - version: v0.6.3 - edition: global.edition.community -mongo: - host: localhost - port: 27017 - db: crawlab_test - username: "" - password: "" - authSource: "admin" -server: - host: 0.0.0.0 - port: 8000 -spider: - fs: "/spiders" - workspace: "/workspace" - repo: "/repo" -task: - workers: 16 - cancelWaitSeconds: 30 -grpc: - address: localhost:9666 - server: - address: 0.0.0.0:9666 - authKey: Crawlab2021! -fs: - filer: - proxy: http://localhost:8888 - url: http://localhost:8000/filer - authKey: Crawlab2021! -node: - master: Y -api: - endpoint: http://localhost:8000 -log: - path: /var/log/crawlab -` diff --git a/core/config/version.go b/core/config/version.go deleted file mode 100644 index 4bcbf066..00000000 --- a/core/config/version.go +++ /dev/null @@ -1,12 +0,0 @@ -package config - -import "strings" - -const Version = "v0.6.3" - -func GetVersion() (v string) { - if strings.HasPrefix(Version, "v") { - return Version - } - return "v" + Version -} diff --git a/core/constants/ds.go b/core/constants/ds.go index 1a92a8ad..268c5761 100644 --- a/core/constants/ds.go +++ b/core/constants/ds.go @@ -16,13 +16,13 @@ const ( ) const ( - DefaultMongoPort = "27017" - DefaultMysqlPort = "3306" - DefaultPostgresqlPort = "5432" - DefaultMssqlPort = "1433" - DefaultCockroachdbPort = "26257" - DefaultElasticsearchPort = "9200" - DefaultKafkaPort = "9092" + DefaultMongoPort = 27017 + DefaultMysqlPort = 3306 + DefaultPostgresqlPort = 5432 + DefaultMssqlPort = 1433 + DefaultCockroachdbPort = 26257 + DefaultElasticsearchPort = 9200 + DefaultKafkaPort = 9092 ) const ( diff --git a/core/constants/notification.go b/core/constants/notification.go index 2ce094b8..19beaf52 100644 --- a/core/constants/notification.go +++ b/core/constants/notification.go @@ -1,8 +1,21 @@ package constants +const ( + NotificationTriggerPatternTask = "^task" + NotificationTriggerPatternNode = "^node" +) + const ( NotificationTriggerTaskFinish = "task_finish" NotificationTriggerTaskError = "task_error" NotificationTriggerTaskEmptyResults = "task_empty_results" - NotificationTriggerTaskNever = "task_never" + NotificationTriggerNodeStatusChange = "node_status_change" + NotificationTriggerNodeOnline = "node_online" + NotificationTriggerNodeOffline = "node_offline" + NotificationTriggerAlert = "alert" +) + +const ( + NotificationTemplateModeRichText = "rich-text" + NotificationTemplateModeMarkdown = "markdown" ) diff --git a/core/controllers/base.go b/core/controllers/base.go deleted file mode 100644 index 261be1bf..00000000 --- a/core/controllers/base.go +++ /dev/null @@ -1,69 +0,0 @@ -package controllers - -import "github.com/gin-gonic/gin" - -const ( - ControllerIdNode = iota << 1 - ControllerIdProject - ControllerIdSpider - ControllerIdTask - ControllerIdJob - ControllerIdSchedule - ControllerIdUser - ControllerIdSetting - ControllerIdToken - ControllerIdVariable - ControllerIdTag - ControllerIdLogin - ControllerIdColor - ControllerIdDataSource - ControllerIdDataCollection - ControllerIdResult - ControllerIdStats - ControllerIdFiler - ControllerIdGit - ControllerIdRole - ControllerIdPermission - ControllerIdExport - ControllerIdNotification - ControllerIdFilter - ControllerIdEnvironment - ControllerIdSync - - ControllerIdVersion - ControllerIdI18n - ControllerIdSystemInfo - ControllerIdDemo -) - -type ControllerId int - -type BasicController interface { - Get(c *gin.Context) - Post(c *gin.Context) - Put(c *gin.Context) - Delete(c *gin.Context) -} - -type ListController interface { - BasicController - GetList(c *gin.Context) - PutList(c *gin.Context) - PostList(c *gin.Context) - DeleteList(c *gin.Context) -} - -type Action struct { - Method string - Path string - HandlerFunc gin.HandlerFunc -} - -type ActionController interface { - Actions() (actions []Action) -} - -type ListActionController interface { - ListController - ActionController -} diff --git a/core/controllers/base_file_v2.go b/core/controllers/base_file_v2.go new file mode 100644 index 00000000..5ea63d9a --- /dev/null +++ b/core/controllers/base_file_v2.go @@ -0,0 +1,303 @@ +package controllers + +import ( + "errors" + "fmt" + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "io" + "os" + "path/filepath" + "sync" +) + +func GetBaseFileListDir(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + files, err := fsSvc.List(path) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccessWithData(c, files) +} + +func GetBaseFileFile(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + data, err := fsSvc.GetFile(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, string(data)) +} + +func GetBaseFileFileInfo(rootPath string, c *gin.Context) { + path := c.Query("path") + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + info, err := fsSvc.GetFileInfo(path) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccessWithData(c, info) +} + +func PostBaseFileSaveFile(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + if c.GetHeader("Content-Type") == "application/json" { + var payload struct { + Path string `json:"path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } else { + path, ok := c.GetPostForm("path") + if !ok { + HandleErrorBadRequest(c, errors.New("missing required field 'path'")) + return + } + file, err := c.FormFile("file") + if err != nil { + HandleErrorBadRequest(c, err) + return + } + f, err := file.Open() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + fileData, err := io.ReadAll(f) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + if err := fsSvc.Save(path, fileData); err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + HandleSuccess(c) +} + +func PostBaseFileSaveFiles(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + form, err := c.MultipartForm() + if err != nil { + HandleErrorBadRequest(c, err) + return + } + wg := sync.WaitGroup{} + wg.Add(len(form.File)) + for path := range form.File { + go func(path string) { + file, err := c.FormFile(path) + if err != nil { + log2.Warnf("invalid file header: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + f, err := file.Open() + if err != nil { + log2.Warnf("unable to open file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + fileData, err := io.ReadAll(f) + if err != nil { + log2.Warnf("unable to read file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + if err := fsSvc.Save(path, fileData); err != nil { + log2.Warnf("unable to save file: %s", path) + log2.Error(err.Error()) + wg.Done() + return + } + wg.Done() + }(path) + } + wg.Wait() + + HandleSuccess(c) +} + +func PostBaseFileSaveDir(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + Data string `json:"data"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.CreateDir(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostBaseFileRenameFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } +} + +func DeleteBaseFileFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + if payload.Path == "~" { + payload.Path = "." + } + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Delete(payload.Path); err != nil { + HandleErrorInternalServerError(c, err) + return + } + _, err = fsSvc.GetFileInfo(".") + if err != nil { + _ = fsSvc.CreateDir("/") + } + + HandleSuccess(c) +} + +func PostBaseFileCopyFile(rootPath string, c *gin.Context) { + var payload struct { + Path string `json:"path"` + NewPath string `json:"new_path"` + } + if err := c.ShouldBindJSON(&payload); err != nil { + HandleErrorBadRequest(c, err) + return + } + + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + HandleSuccess(c) +} + +func PostBaseFileExport(rootPath string, c *gin.Context) { + fsSvc, err := getBaseFileFsSvc(rootPath) + if err != nil { + HandleErrorBadRequest(c, err) + return + } + + // zip file path + zipFilePath, err := fsSvc.Export() + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // download + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) + c.File(zipFilePath) +} + +func GetBaseFileFsSvc(rootPath string) (svc interfaces.FsServiceV2, err error) { + return getBaseFileFsSvc(rootPath) +} + +func getBaseFileFsSvc(rootPath string) (svc interfaces.FsServiceV2, err error) { + workspacePath := viper.GetString("workspace") + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, rootPath)) + + return fsSvc, nil +} diff --git a/core/controllers/base_v2.go b/core/controllers/base_v2.go index ddb9107e..82130a3c 100644 --- a/core/controllers/base_v2.go +++ b/core/controllers/base_v2.go @@ -11,6 +11,12 @@ import ( mongo2 "go.mongodb.org/mongo-driver/mongo" ) +type Action struct { + Method string + Path string + HandlerFunc gin.HandlerFunc +} + type BaseControllerV2[T any] struct { modelSvc *service.ModelServiceV2[T] actions []Action @@ -86,7 +92,6 @@ func (ctr *BaseControllerV2[T]) PutById(c *gin.Context) { u := GetUserFromContextV2(c) m := any(&model).(interfaces.ModelV2) - m.SetId(primitive.NewObjectID()) m.SetUpdated(u.Id) if err := ctr.modelSvc.ReplaceById(id, model); err != nil { @@ -170,14 +175,19 @@ func (ctr *BaseControllerV2[T]) DeleteList(c *gin.Context) { } func (ctr *BaseControllerV2[T]) getAll(c *gin.Context) { - models, err := ctr.modelSvc.GetMany(nil, &mongo.FindOptions{ - Sort: bson.D{{"_id", -1}}, + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + if sort == nil { + sort = bson.D{{"_id", -1}} + } + models, err := ctr.modelSvc.GetMany(query, &mongo.FindOptions{ + Sort: sort, }) if err != nil { HandleErrorInternalServerError(c, err) return } - total, err := ctr.modelSvc.Count(nil) + total, err := ctr.modelSvc.Count(query) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/base_v2_test.go b/core/controllers/base_v2_test.go index 3e0351ef..5259ca9a 100644 --- a/core/controllers/base_v2_test.go +++ b/core/controllers/base_v2_test.go @@ -6,7 +6,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/spf13/viper" @@ -24,7 +24,7 @@ func init() { } // TestModel is a simple struct to be used as a model in tests -type TestModel models.TestModel +type TestModel models.TestModelV2 var TestToken string diff --git a/core/controllers/binder.go b/core/controllers/binder.go deleted file mode 100644 index aa27508a..00000000 --- a/core/controllers/binder.go +++ /dev/null @@ -1,14 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" -) - -type BinderInterface interface { - Bind(c *gin.Context) (res interfaces.Model, err error) - BindList(c *gin.Context) (res []interfaces.Model, err error) - BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) - BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) -} diff --git a/core/controllers/binder_json.go b/core/controllers/binder_json.go deleted file mode 100644 index e60ac30c..00000000 --- a/core/controllers/binder_json.go +++ /dev/null @@ -1,208 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" -) - -func NewJsonBinder(id ControllerId) (b *JsonBinder) { - return &JsonBinder{ - id: id, - } -} - -type JsonBinder struct { - id ControllerId -} - -func (b *JsonBinder) Bind(c *gin.Context) (res interfaces.Model, err error) { - // declare - m := models.NewModelMap() - - switch b.id { - case ControllerIdNode: - err = c.ShouldBindJSON(&m.Node) - return &m.Node, err - case ControllerIdProject: - err = c.ShouldBindJSON(&m.Project) - return &m.Project, err - case ControllerIdSpider: - err = c.ShouldBindJSON(&m.Spider) - return &m.Spider, err - case ControllerIdTask: - err = c.ShouldBindJSON(&m.Task) - return &m.Task, err - case ControllerIdJob: - err = c.ShouldBindJSON(&m.Job) - return &m.Job, err - case ControllerIdSchedule: - err = c.ShouldBindJSON(&m.Schedule) - return &m.Schedule, err - case ControllerIdUser: - err = c.ShouldBindJSON(&m.User) - return &m.User, nil - case ControllerIdSetting: - err = c.ShouldBindJSON(&m.Setting) - return &m.Setting, nil - case ControllerIdToken: - err = c.ShouldBindJSON(&m.Token) - return &m.Token, nil - case ControllerIdVariable: - err = c.ShouldBindJSON(&m.Variable) - return &m.Variable, nil - case ControllerIdTag: - err = c.ShouldBindJSON(&m.Tag) - return &m.Tag, nil - case ControllerIdDataSource: - err = c.ShouldBindJSON(&m.DataSource) - return &m.DataSource, nil - case ControllerIdDataCollection: - err = c.ShouldBindJSON(&m.DataCollection) - return &m.DataCollection, nil - case ControllerIdGit: - err = c.ShouldBindJSON(&m.Git) - return &m.Git, nil - case ControllerIdRole: - err = c.ShouldBindJSON(&m.Role) - return &m.Role, nil - case ControllerIdPermission: - err = c.ShouldBindJSON(&m.Permission) - return &m.Permission, nil - case ControllerIdEnvironment: - err = c.ShouldBindJSON(&m.Environment) - return &m.Environment, nil - default: - return nil, errors.ErrorControllerInvalidControllerId - } -} - -func (b *JsonBinder) BindList(c *gin.Context) (res interface{}, err error) { - // declare - m := models.NewModelListMap() - - // bind - switch b.id { - case ControllerIdNode: - err = c.ShouldBindJSON(&m.Nodes) - return m.Nodes, err - case ControllerIdProject: - err = c.ShouldBindJSON(&m.Projects) - return m.Projects, err - case ControllerIdSpider: - err = c.ShouldBindJSON(&m.Spiders) - return m.Spiders, err - case ControllerIdTask: - err = c.ShouldBindJSON(&m.Tasks) - return m.Tasks, err - case ControllerIdJob: - err = c.ShouldBindJSON(&m.Jobs) - return m.Jobs, err - case ControllerIdSchedule: - err = c.ShouldBindJSON(&m.Schedules) - return m.Schedules, err - case ControllerIdUser: - err = c.ShouldBindJSON(&m.Users) - return m.Users, nil - case ControllerIdSetting: - err = c.ShouldBindJSON(&m.Settings) - return m.Settings, nil - case ControllerIdToken: - err = c.ShouldBindJSON(&m.Tokens) - return m.Tokens, nil - case ControllerIdVariable: - err = c.ShouldBindJSON(&m.Variables) - return m.Variables, nil - case ControllerIdTag: - err = c.ShouldBindJSON(&m.Tags) - return m.Tags, nil - case ControllerIdDataSource: - err = c.ShouldBindJSON(&m.DataSources) - return m.DataSources, nil - case ControllerIdDataCollection: - err = c.ShouldBindJSON(&m.DataCollections) - return m.DataCollections, nil - case ControllerIdGit: - err = c.ShouldBindJSON(&m.Gits) - return m.Gits, nil - case ControllerIdRole: - err = c.ShouldBindJSON(&m.Roles) - return m.Roles, nil - case ControllerIdEnvironment: - err = c.ShouldBindJSON(&m.Environments) - return m.Environments, nil - default: - return nil, errors.ErrorControllerInvalidControllerId - } -} - -func (b *JsonBinder) BindBatchRequestPayload(c *gin.Context) (payload entity.BatchRequestPayload, err error) { - if err := c.ShouldBindJSON(&payload); err != nil { - return payload, err - } - return payload, nil -} - -func (b *JsonBinder) BindBatchRequestPayloadWithStringData(c *gin.Context) (payload entity.BatchRequestPayloadWithStringData, res interfaces.Model, err error) { - // declare - m := models.NewModelMap() - - // bind - if err := c.ShouldBindJSON(&payload); err != nil { - return payload, nil, err - } - - // validate - if len(payload.Ids) == 0 || - len(payload.Fields) == 0 { - return payload, nil, errors.ErrorControllerRequestPayloadInvalid - } - - // unmarshall - switch b.id { - case ControllerIdNode: - err = json.Unmarshal([]byte(payload.Data), &m.Node) - return payload, &m.Node, err - case ControllerIdProject: - err = json.Unmarshal([]byte(payload.Data), &m.Project) - return payload, &m.Project, err - case ControllerIdSpider: - err = json.Unmarshal([]byte(payload.Data), &m.Spider) - return payload, &m.Spider, err - case ControllerIdTask: - err = json.Unmarshal([]byte(payload.Data), &m.Task) - return payload, &m.Task, err - case ControllerIdJob: - err = json.Unmarshal([]byte(payload.Data), &m.Job) - return payload, &m.Job, err - case ControllerIdSchedule: - err = json.Unmarshal([]byte(payload.Data), &m.Schedule) - return payload, &m.Schedule, err - case ControllerIdUser: - err = json.Unmarshal([]byte(payload.Data), &m.User) - return payload, &m.User, err - case ControllerIdSetting: - err = json.Unmarshal([]byte(payload.Data), &m.Setting) - return payload, &m.Setting, err - case ControllerIdToken: - err = json.Unmarshal([]byte(payload.Data), &m.Token) - return payload, &m.Token, err - case ControllerIdVariable: - err = json.Unmarshal([]byte(payload.Data), &m.Variable) - return payload, &m.Variable, err - case ControllerIdDataSource: - err = json.Unmarshal([]byte(payload.Data), &m.DataSource) - return payload, &m.DataSource, err - case ControllerIdDataCollection: - err = json.Unmarshal([]byte(payload.Data), &m.DataCollection) - return payload, &m.DataCollection, err - case ControllerIdEnvironment: - err = json.Unmarshal([]byte(payload.Data), &m.Environment) - return payload, &m.Environment, err - default: - return payload, nil, errors.ErrorControllerInvalidControllerId - } -} diff --git a/core/controllers/data_collection.go b/core/controllers/data_collection.go deleted file mode 100644 index cf9ef9a9..00000000 --- a/core/controllers/data_collection.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataCollectionController *dataCollectionController - -func getDataCollectionActions() []Action { - ctx := newDataCollectionContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/indexes", - HandlerFunc: ctx.postIndexes, - }, - } -} - -type dataCollectionController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataCollectionContext -} - -type dataCollectionContext struct { - modelSvc service.ModelService - resultSvc interfaces.ResultService -} - -func (ctx *dataCollectionContext) postIndexes(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - dc, err := ctx.modelSvc.GetDataCollectionById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - for _, f := range dc.Fields { - if err := mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{ - Keys: f.Key, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) -} - -var _dataCollectionCtx *dataCollectionContext - -func newDataCollectionContext() *dataCollectionContext { - if _dataCollectionCtx != nil { - return _dataCollectionCtx - } - - // context - ctx := &dataCollectionContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - ) { - ctx.modelSvc = modelSvc - }); err != nil { - panic(err) - } - - _dataCollectionCtx = ctx - - return ctx -} - -func newDataCollectionController() *dataCollectionController { - actions := getDataCollectionActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataCollection, modelSvc.GetBaseService(interfaces.ModelIdDataCollection), actions) - ctx := newDataCollectionContext() - - return &dataCollectionController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/data_source.go b/core/controllers/data_source.go deleted file mode 100644 index a4b90565..00000000 --- a/core/controllers/data_source.go +++ /dev/null @@ -1,148 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/ds" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - interfaces2 "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var DataSourceController *dataSourceController - -func getDataSourceActions() []Action { - ctx := newDataSourceContext() - return []Action{ - { - Path: "/:id/change-password", - Method: http.MethodPost, - HandlerFunc: ctx.changePassword, - }, - } -} - -type dataSourceController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *dataSourceContext -} - -func (ctr *dataSourceController) Post(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // add data source to db - if err := mongo.RunTransaction(func(ctx mongo2.SessionContext) error { - if err := delegate.NewModelDelegate(&_ds).Add(); err != nil { - return trace.TraceError(err) - } - pwd, err := utils.EncryptAES(_ds.Password) - if err != nil { - return trace.TraceError(err) - } - p := models.Password{Id: _ds.Id, Password: pwd} - if err := delegate.NewModelDelegate(&p).Add(); err != nil { - return trace.TraceError(err) - } - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() - - HandleSuccess(c) -} - -func (ctr *dataSourceController) Put(c *gin.Context) { - // data source - var _ds models.DataSource - if err := c.ShouldBindJSON(&_ds); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := delegate.NewModelDelegate(&_ds).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // check data source status - go func() { _ = ctr.ctx.dsSvc.CheckStatus(_ds.Id) }() -} - -type dataSourceContext struct { - dsSvc interfaces.DataSourceService -} - -var _dataSourceCtx *dataSourceContext - -func newDataSourceContext() *dataSourceContext { - if _dataSourceCtx != nil { - return _dataSourceCtx - } - dsSvc, err := ds.GetDataSourceService() - if err != nil { - panic(err) - } - _dataSourceCtx = &dataSourceContext{ - dsSvc: dsSvc, - } - return _dataSourceCtx -} - -func (ctx *dataSourceContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorDataSourceMissingRequiredFields) - return - } - if err := ctx.dsSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newDataSourceController() *dataSourceController { - actions := getDataSourceActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - d := NewListPostActionControllerDelegate(ControllerIdDataSource, modelSvc.GetBaseService(interfaces2.ModelIdDataSource), actions) - ctx := newDataSourceContext() - - return &dataSourceController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/delegate_action.go b/core/controllers/delegate_action.go deleted file mode 100644 index dccb0787..00000000 --- a/core/controllers/delegate_action.go +++ /dev/null @@ -1,17 +0,0 @@ -package controllers - -func NewActionControllerDelegate(id ControllerId, actions []Action) (d *ActionControllerDelegate) { - return &ActionControllerDelegate{ - id: id, - actions: actions, - } -} - -type ActionControllerDelegate struct { - id ControllerId - actions []Action -} - -func (ctr *ActionControllerDelegate) Actions() (actions []Action) { - return ctr.actions -} diff --git a/core/controllers/delegate_basic.go b/core/controllers/delegate_basic.go deleted file mode 100644 index 86a26f3e..00000000 --- a/core/controllers/delegate_basic.go +++ /dev/null @@ -1,99 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" -) - -func NewBasicControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *BasicControllerDelegate) { - return &BasicControllerDelegate{ - id: id, - svc: svc, - } -} - -type BasicControllerDelegate struct { - id ControllerId - svc interfaces.ModelBaseService -} - -func (d *BasicControllerDelegate) Get(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := d.svc.GetById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Post(c *gin.Context) { - doc, err := NewJsonBinder(d.id).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Put(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := NewJsonBinder(d.id).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if doc.GetId() != id { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - _, err = d.svc.GetById(id) - if err != nil { - HandleErrorNotFound(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, doc) -} - -func (d *BasicControllerDelegate) Delete(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - doc, err := d.svc.GetById(oid) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Delete(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} diff --git a/core/controllers/delegate_list.go b/core/controllers/delegate_list.go deleted file mode 100644 index 403a8930..00000000 --- a/core/controllers/delegate_list.go +++ /dev/null @@ -1,222 +0,0 @@ -package controllers - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "reflect" - "time" -) - -func NewListControllerDelegate(id ControllerId, svc interfaces.ModelBaseService) (d *ListControllerDelegate) { - if svc == nil { - panic(errors.ErrorControllerNoModelService) - } - - return &ListControllerDelegate{ - id: id, - svc: svc, - bc: NewBasicControllerDelegate(id, svc), - } -} - -type ListControllerDelegate struct { - id ControllerId - svc interfaces.ModelBaseService - bc BasicController -} - -func (d *ListControllerDelegate) Get(c *gin.Context) { - d.bc.Get(c) -} - -func (d *ListControllerDelegate) Post(c *gin.Context) { - d.bc.Post(c) -} - -func (d *ListControllerDelegate) Put(c *gin.Context) { - d.bc.Put(c) -} - -func (d *ListControllerDelegate) Delete(c *gin.Context) { - d.bc.Delete(c) -} - -func (d *ListControllerDelegate) GetList(c *gin.Context) { - // get all if query field "all" is set true - all := MustGetFilterAll(c) - if all { - d.getAll(c) - return - } - - // get list and total - l, total, err := d.getList(c) - if err != nil { - return - } - - // response - HandleSuccessWithListData(c, l, total) -} - -func (d *ListControllerDelegate) PostList(c *gin.Context) { - // bind - docs, err := NewJsonBinder(d.id).BindList(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // success ids - var ids []primitive.ObjectID - - // reflect - switch reflect.TypeOf(docs).Kind() { - case reflect.Slice, reflect.Array: - s := reflect.ValueOf(docs) - for i := 0; i < s.Len(); i++ { - item := s.Index(i) - if !item.CanAddr() { - HandleErrorInternalServerError(c, errors.ErrorModelInvalidType) - return - } - ptr := item.Addr() - doc, ok := ptr.Interface().(interfaces.Model) - if !ok { - HandleErrorInternalServerError(c, errors.ErrorModelInvalidType) - return - } - if err := delegate.NewModelDelegate(doc, GetUserFromContext(c)).Add(); err != nil { - _ = trace.TraceError(err) - continue - } - ids = append(ids, doc.GetId()) - } - } - - // check - items, err := utils.GetArrayItems(docs) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if len(ids) < len(items) { - HandleErrorInternalServerError(c, errors.ErrorControllerAddError) - return - } - - // success - HandleSuccessWithData(c, docs) -} - -func (d *ListControllerDelegate) PutList(c *gin.Context) { - payload, doc, err := NewJsonBinder(d.id).BindBatchRequestPayloadWithStringData(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // query - query := bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - } - - // update - if err := d.svc.UpdateDoc(query, doc, payload.Fields); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (d *ListControllerDelegate) DeleteList(c *gin.Context) { - payload, err := NewJsonBinder(d.id).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := d.svc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (d *ListControllerDelegate) getAll(c *gin.Context) { - // get list - tic := time.Now() - log.Debugf("getAll -> d.svc.GetMany:start") - list, err := d.svc.GetList(nil, &mongo.FindOptions{ - Sort: bson.D{{"_id", -1}}, - }) - if err != nil { - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - log.Debugf("getAll -> d.svc.GetMany:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) - tic = time.Now() - - // total count - tic = time.Now() - log.Debugf("getAll -> d.svc.Count:start") - total, err := d.svc.Count(nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - log.Debugf("getAll -> d.svc.Count:end. elapsed: %d ms", time.Now().Sub(tic).Milliseconds()) - - // response - HandleSuccessWithListData(c, list, total) -} - -func (d *ListControllerDelegate) getList(c *gin.Context) (l interfaces.List, total int, err error) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - l, err = d.svc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // total count - total, err = d.svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - return l, total, nil -} diff --git a/core/controllers/delegate_list_action.go b/core/controllers/delegate_list_action.go deleted file mode 100644 index deacf347..00000000 --- a/core/controllers/delegate_list_action.go +++ /dev/null @@ -1,17 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" -) - -func NewListPostActionControllerDelegate(id ControllerId, svc interfaces.ModelBaseService, actions []Action) (d *ListActionControllerDelegate) { - return &ListActionControllerDelegate{ - NewListControllerDelegate(id, svc), - NewActionControllerDelegate(id, actions), - } -} - -type ListActionControllerDelegate struct { - ListController - ActionController -} diff --git a/core/controllers/demo.go b/core/controllers/demo.go deleted file mode 100644 index 7a09d988..00000000 --- a/core/controllers/demo.go +++ /dev/null @@ -1,73 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "net/http" -) - -func getDemoActions() []Action { - ctx := newDemoContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/import", - HandlerFunc: ctx.import_, - }, - { - Method: http.MethodGet, - Path: "/reimport", - HandlerFunc: ctx.reimport, - }, - { - Method: http.MethodGet, - Path: "/cleanup", - HandlerFunc: ctx.cleanup, - }, - } -} - -type demoContext struct { -} - -func (ctx *demoContext) import_(c *gin.Context) { - if err := utils.ImportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) reimport(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *demoContext) cleanup(c *gin.Context) { - if err := utils.ReimportDemo(); err != nil { - trace.PrintError(err) - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -var _demoCtx *demoContext - -func newDemoContext() *demoContext { - if _demoCtx != nil { - return _demoCtx - } - - _demoCtx = &demoContext{} - - return _demoCtx -} - -var DemoController ActionController diff --git a/core/controllers/environment.go b/core/controllers/environment.go deleted file mode 100644 index 560b02c9..00000000 --- a/core/controllers/environment.go +++ /dev/null @@ -1,57 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" -) - -var EnvironmentController *environmentController - -var EnvironmentActions []Action - -type environmentController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *environmentContext -} - -type environmentContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func newEnvironmentContext() *environmentContext { - // context - ctx := &environmentContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newEnvironmentController() *environmentController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) - d := NewListPostActionControllerDelegate(ControllerIdEnvironment, modelSvc.GetBaseService(interfaces.ModelIdEnvironment), EnvironmentActions) - ctx := newEnvironmentContext() - - return &environmentController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/export.go b/core/controllers/export.go deleted file mode 100644 index b99ffcb9..00000000 --- a/core/controllers/export.go +++ /dev/null @@ -1,127 +0,0 @@ -package controllers - -import ( - "errors" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/export" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "net/http" -) - -var ExportController ActionController - -func getExportActions() []Action { - ctx := newExportContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:type", - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id", - HandlerFunc: ctx.getExport, - }, - { - Method: http.MethodGet, - Path: "/:type/:id/download", - HandlerFunc: ctx.getExportDownload, - }, - } -} - -type exportContext struct { - csvSvc interfaces.ExportService - jsonSvc interfaces.ExportService -} - -func (ctx *exportContext) postExport(c *gin.Context) { - exportType := c.Param("type") - exportTarget := c.Query("target") - exportFilter, _ := GetFilter(c) - - var exportId string - var err error - switch exportType { - case constants.ExportTypeCsv: - exportId, err = ctx.csvSvc.Export(exportType, exportTarget, exportFilter) - case constants.ExportTypeJson: - exportId, err = ctx.jsonSvc.Export(exportType, exportTarget, exportFilter) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exportId) -} - -func (ctx *exportContext) getExport(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, exp) -} - -func (ctx *exportContext) getExportDownload(c *gin.Context) { - exportType := c.Param("type") - exportId := c.Param("id") - - var exp interfaces.Export - var err error - switch exportType { - case constants.ExportTypeCsv: - exp, err = ctx.csvSvc.GetExport(exportId) - case constants.ExportTypeJson: - exp, err = ctx.jsonSvc.GetExport(exportId) - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - switch exportType { - case constants.ExportTypeCsv: - c.Header("Content-Type", "text/csv") - case constants.ExportTypeJson: - c.Header("Content-Type", "text/plain") - default: - HandleErrorBadRequest(c, errors.New(fmt.Sprintf("invalid export type: %s", exportType))) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", exp.GetDownloadPath())) - c.File(exp.GetDownloadPath()) -} - -func newExportContext() *exportContext { - return &exportContext{ - csvSvc: export.GetCsvService(), - jsonSvc: export.GetJsonService(), - } -} diff --git a/core/controllers/filer.go b/core/controllers/filer.go deleted file mode 100644 index 4af5d142..00000000 --- a/core/controllers/filer.go +++ /dev/null @@ -1,130 +0,0 @@ -package controllers - -import ( - "bufio" - "fmt" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/gin-gonic/gin" - "github.com/imroc/req" - "github.com/spf13/viper" - "net/http" - "strings" -) - -var FilerController ActionController - -func getFilerActions() []Action { - filerCtx := newFilerContext() - return []Action{ - { - Method: http.MethodGet, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPost, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodPut, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - { - Method: http.MethodDelete, - Path: "*path", - HandlerFunc: filerCtx.do, - }, - } -} - -type filerContext struct { - endpoint string -} - -func (ctx *filerContext) do(c *gin.Context) { - // request path - requestPath := strings.Replace(c.Request.URL.Path, "/filer", "", 1) - - // request url - requestUrl := fmt.Sprintf("%s%s", ctx.endpoint, requestPath) - if c.Request.URL.RawQuery != "" { - requestUrl += "?" + c.Request.URL.RawQuery - } - - // request body - bufR := bufio.NewScanner(c.Request.Body) - requestBody := req.BodyJSON(bufR.Bytes()) - - // request file uploads - var requestFileUploads []req.FileUpload - form, err := c.MultipartForm() - if err == nil { - for k, v := range form.File { - for _, fh := range v { - f, err := fh.Open() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - requestFileUploads = append(requestFileUploads, req.FileUpload{ - FileName: fh.Filename, - FieldName: k, - File: f, - }) - } - } - } - - // request header - requestHeader := req.Header{} - for k, v := range c.Request.Header { - if len(v) > 0 { - requestHeader[k] = v[0] - } - } - - // perform request - res, err := req.Do(c.Request.Method, requestUrl, requestHeader, requestBody, requestFileUploads) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // status code check - statusCode := res.Response().StatusCode - if statusCode == http.StatusNotFound { - HandleErrorNotFoundNoPrint(c, errors.ErrorControllerFilerNotFound) - return - } - - // response - for k, v := range res.Response().Header { - if len(v) > 0 { - c.Header(k, v[0]) - } - } - _, _ = c.Writer.Write(res.Bytes()) - c.AbortWithStatus(statusCode) -} - -var _filerCtx *filerContext - -func newFilerContext() *filerContext { - if _filerCtx != nil { - return _filerCtx - } - - ctx := &filerContext{ - endpoint: "http://localhost:8888", - } - - if viper.GetString("fs.filer.proxy") != "" { - ctx.endpoint = viper.GetString("fs.filer.proxy") - } - - _filerCtx = ctx - - return ctx -} diff --git a/core/controllers/filter.go b/core/controllers/filter.go deleted file mode 100644 index b609f6e0..00000000 --- a/core/controllers/filter.go +++ /dev/null @@ -1,100 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var FilterController ActionController - -func getFilterActions() []Action { - ctx := newFilterContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:col", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value", - HandlerFunc: ctx.getColFieldOptions, - }, - { - Method: http.MethodGet, - Path: "/:col/:value/:label", - HandlerFunc: ctx.getColFieldOptions, - }, - } -} - -type filterContext struct { -} - -func (ctx *filterContext) getColFieldOptions(c *gin.Context) { - colName := c.Param("col") - value := c.Param("value") - if value == "" { - value = "_id" - } - label := c.Param("label") - if label == "" { - label = "name" - } - query := MustGetFilterQuery(c) - pipelines := mongo2.Pipeline{} - if query != nil { - pipelines = append(pipelines, bson.D{{"$match", query}}) - } - pipelines = append( - pipelines, - bson.D{ - { - "$group", - bson.M{ - "_id": bson.M{ - "value": "$" + value, - "label": "$" + label, - }, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$project", - bson.M{ - "value": "$_id.value", - "label": bson.M{"$toString": "$_id.label"}, - }, - }, - }, - ) - pipelines = append( - pipelines, - bson.D{ - { - "$sort", - bson.D{ - {"value", 1}, - }, - }, - }, - ) - var options []entity.FilterSelectOption - if err := mongo.GetMongoCol(colName).Aggregate(pipelines, nil).All(&options); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, options) -} - -func newFilterContext() *filterContext { - return &filterContext{} -} diff --git a/core/controllers/git.go b/core/controllers/git.go deleted file mode 100644 index 128323d4..00000000 --- a/core/controllers/git.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var GitController ListController diff --git a/core/controllers/init.go b/core/controllers/init.go deleted file mode 100644 index e927b9c3..00000000 --- a/core/controllers/init.go +++ /dev/null @@ -1,42 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" -) - -func InitControllers() (err error) { - modelSvc, err := service.GetService() - if err != nil { - return err - } - - NodeController = newNodeController() - ProjectController = newProjectController() - SpiderController = newSpiderController() - TaskController = newTaskController() - UserController = newUserController() - TagController = NewListControllerDelegate(ControllerIdTag, modelSvc.GetBaseService(interfaces.ModelIdTag)) - SettingController = newSettingController() - LoginController = NewActionControllerDelegate(ControllerIdLogin, getLoginActions()) - DataCollectionController = newDataCollectionController() - ResultController = NewActionControllerDelegate(ControllerIdResult, getResultActions()) - ScheduleController = newScheduleController() - StatsController = NewActionControllerDelegate(ControllerIdStats, getStatsActions()) - TokenController = newTokenController() - FilerController = NewActionControllerDelegate(ControllerIdFiler, getFilerActions()) - GitController = NewListControllerDelegate(ControllerIdGit, modelSvc.GetBaseService(interfaces.ModelIdGit)) - VersionController = NewActionControllerDelegate(ControllerIdVersion, getVersionActions()) - SystemInfoController = NewActionControllerDelegate(ControllerIdSystemInfo, getSystemInfoActions()) - DemoController = NewActionControllerDelegate(ControllerIdDemo, getDemoActions()) - RoleController = NewListControllerDelegate(ControllerIdRole, modelSvc.GetBaseService(interfaces.ModelIdRole)) - PermissionController = NewListControllerDelegate(ControllerIdPermission, modelSvc.GetBaseService(interfaces.ModelIdPermission)) - ExportController = NewActionControllerDelegate(ControllerIdExport, getExportActions()) - NotificationController = NewActionControllerDelegate(ControllerIdNotification, getNotificationActions()) - FilterController = NewActionControllerDelegate(ControllerIdFilter, getFilterActions()) - SyncController = NewActionControllerDelegate(ControllerIdSync, getSyncActions()) - DataSourceController = newDataSourceController() - EnvironmentController = newEnvironmentController() - - return nil -} diff --git a/core/controllers/login.go b/core/controllers/login.go deleted file mode 100644 index 42926fd5..00000000 --- a/core/controllers/login.go +++ /dev/null @@ -1,64 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gin-gonic/gin" - "net/http" -) - -var LoginController ActionController - -func getLoginActions() []Action { - loginCtx := newLoginContext() - return []Action{ - {Method: http.MethodPost, Path: "/login", HandlerFunc: loginCtx.login}, - {Method: http.MethodPost, Path: "/logout", HandlerFunc: loginCtx.logout}, - } -} - -type loginContext struct { - userSvc interfaces.UserService -} - -func (ctx *loginContext) login(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - token, loggedInUser, err := ctx.userSvc.Login(&interfaces.UserLoginOptions{ - Username: u.Username, - Password: u.Password, - }) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - c.Set(constants.UserContextKey, loggedInUser) - HandleSuccessWithData(c, token) -} - -func (ctx *loginContext) logout(c *gin.Context) { - c.Set(constants.UserContextKey, nil) - HandleSuccess(c) -} - -func newLoginContext() *loginContext { - // context - ctx := &loginContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - userSvc interfaces.UserService, - ) { - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/node.go b/core/controllers/node.go deleted file mode 100644 index 236ebafc..00000000 --- a/core/controllers/node.go +++ /dev/null @@ -1,94 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var NodeController *nodeController - -type nodeController struct { - ListControllerDelegate -} - -func (ctr *nodeController) Post(c *gin.Context) { - var n models.Node - if err := c.ShouldBindJSON(&n); err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := ctr._post(c, &n); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctr *nodeController) PostList(c *gin.Context) { - // bind - var docs []models.Node - if err := c.ShouldBindJSON(&docs); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // success ids - var ids []primitive.ObjectID - - // iterate nodes - for _, n := range docs { - if err := ctr._post(c, &n); err != nil { - trace.PrintError(err) - continue - } - ids = append(ids, n.Id) - } - - // success - HandleSuccessWithData(c, docs) -} - -func (ctr *nodeController) _post(c *gin.Context, n *models.Node) (err error) { - // set default key - if n.Key == "" { - id, err := uuid.NewUUID() - if err != nil { - return trace.TraceError(err) - } - n.Key = id.String() - } - - // set default status - if n.Status == "" { - n.Status = constants.NodeStatusUnregistered - } - - // add - if err := delegate.NewModelDelegate(n, GetUserFromContext(c)).Add(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func newNodeController() *nodeController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdNode, modelSvc.GetBaseService(interfaces.ModelIdNode)) - - return &nodeController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/notification.go b/core/controllers/notification.go deleted file mode 100644 index 31b96c04..00000000 --- a/core/controllers/notification.go +++ /dev/null @@ -1,158 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/notification" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var NotificationController ActionController - -func getNotificationActions() []Action { - ctx := newNotificationContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/settings", - HandlerFunc: ctx.GetSettingList, - }, - { - Method: http.MethodGet, - Path: "/settings/:id", - HandlerFunc: ctx.GetSetting, - }, - { - Method: http.MethodPost, - Path: "/settings", - HandlerFunc: ctx.PostSetting, - }, - { - Method: http.MethodPut, - Path: "/settings/:id", - HandlerFunc: ctx.PutSetting, - }, - { - Method: http.MethodDelete, - Path: "/settings/:id", - HandlerFunc: ctx.DeleteSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/enable", - HandlerFunc: ctx.EnableSetting, - }, - { - Method: http.MethodPost, - Path: "/settings/:id/disable", - HandlerFunc: ctx.DisableSetting, - }, - } -} - -type notificationContext struct { - svc *notification.Service -} - -func (ctx *notificationContext) GetSettingList(c *gin.Context) { - query := MustGetFilterQuery(c) - pagination := MustGetPagination(c) - sort := MustGetSortOption(c) - res, total, err := ctx.svc.GetSettingList(query, pagination, sort) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithListData(c, res, total) -} - -func (ctx *notificationContext) GetSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - res, err := ctx.svc.GetSetting(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, res) -} - -func (ctx *notificationContext) PostSetting(c *gin.Context) { - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PosSetting(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) PutSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s notification.NotificationSetting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.PutSetting(id, s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DeleteSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DeleteSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) EnableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.EnableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *notificationContext) DisableSetting(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctx.svc.DisableSetting(id); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func newNotificationContext() *notificationContext { - ctx := ¬ificationContext{ - svc: notification.GetService(), - } - return ctx -} diff --git a/core/controllers/permission.go b/core/controllers/permission.go deleted file mode 100644 index 42be45dc..00000000 --- a/core/controllers/permission.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var PermissionController ListController diff --git a/core/controllers/project.go b/core/controllers/project.go deleted file mode 100644 index fe13a9e8..00000000 --- a/core/controllers/project.go +++ /dev/null @@ -1,103 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" -) - -var ProjectController *projectController - -type projectController struct { - ListControllerDelegate -} - -func (ctr *projectController) GetList(c *gin.Context) { - // get all if query field "all" is set true - all := MustGetFilterAll(c) - if all { - ctr.getAll(c) - return - } - - // get list - list, total, err := ctr.getList(c) - if err != nil { - return - } - data := list.GetModels() - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // project ids - var ids []primitive.ObjectID - - // count cache - cache := map[primitive.ObjectID]int{} - - // iterate - for _, d := range data { - p, ok := d.(*models.Project) - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerInvalidType) - return - } - ids = append(ids, p.Id) - cache[p.Id] = 0 - } - - // spiders - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - spiders, err := modelSvc.GetSpiderList(bson.M{ - "project_id": bson.M{ - "$in": ids, - }, - }, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - for _, s := range spiders { - _, ok := cache[s.ProjectId] - if !ok { - HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) - return - } - cache[s.ProjectId]++ - } - - // assign - var projects []models.Project - for _, d := range data { - p := d.(*models.Project) - p.Spiders = cache[p.Id] - projects = append(projects, *p) - } - - HandleSuccessWithListData(c, projects, total) -} - -func newProjectController() *projectController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdProject, modelSvc.GetBaseService(interfaces.ModelIdProject)) - - return &projectController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/project_v2.go b/core/controllers/project_v2.go new file mode 100644 index 00000000..03396fd5 --- /dev/null +++ b/core/controllers/project_v2.go @@ -0,0 +1,90 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/errors" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/db/mongo" + "github.com/gin-gonic/gin" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +func GetProjectList(c *gin.Context) { + // get all list + all := MustGetFilterAll(c) + if all { + NewControllerV2[models2.ProjectV2]().getAll(c) + return + } + + // params + pagination := MustGetPagination(c) + query := MustGetFilterQuery(c) + sort := MustGetSortOption(c) + + // get list + projects, err := service.NewModelServiceV2[models2.ProjectV2]().GetMany(query, &mongo.FindOptions{ + Sort: sort, + Skip: pagination.Size * (pagination.Page - 1), + Limit: pagination.Size, + }) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + HandleErrorInternalServerError(c, err) + } + return + } + if len(projects) == 0 { + HandleSuccessWithListData(c, []models2.ProjectV2{}, 0) + return + } + + // total count + total, err := service.NewModelServiceV2[models2.ProjectV2]().Count(query) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // project ids + var ids []primitive.ObjectID + + // count cache + cache := map[primitive.ObjectID]int{} + + // iterate + for _, p := range projects { + ids = append(ids, p.Id) + cache[p.Id] = 0 + } + + // spiders + spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(bson.M{ + "project_id": bson.M{ + "$in": ids, + }, + }, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + for _, s := range spiders { + _, ok := cache[s.ProjectId] + if !ok { + HandleErrorInternalServerError(c, errors.ErrorControllerMissingInCache) + return + } + cache[s.ProjectId]++ + } + + // assign + var data []models2.ProjectV2 + for _, p := range projects { + p.Spiders = cache[p.Id] + data = append(data, p) + } + + HandleSuccessWithListData(c, data, total) +} diff --git a/core/controllers/result.go b/core/controllers/result.go deleted file mode 100644 index 0324a417..00000000 --- a/core/controllers/result.go +++ /dev/null @@ -1,150 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" -) - -var ResultController ActionController - -func getResultActions() []Action { - var resultCtx = newResultContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id", - HandlerFunc: resultCtx.getList, - }, - } -} - -type resultContext struct { - modelSvc service.ModelService -} - -func (ctx *resultContext) getList(c *gin.Context) { - // data collection id - dcId, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - var dsId primitive.ObjectID - dsIdStr := c.Query("data_source_id") - if dsIdStr != "" { - dsId, err = primitive.ObjectIDFromHex(dsIdStr) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } - - // data collection - dc, err := ctx.modelSvc.GetDataCollectionById(dcId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models.DataSource{} - } else { - HandleErrorInternalServerError(c, err) - return - } - } - - // spider - sq := bson.M{ - "col_id": dc.Id, - "data_source_id": ds.Id, - } - s, err := ctx.modelSvc.GetSpider(sq, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // service - svc, err := result.GetResultService(s.Id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // params - pagination := MustGetPagination(c) - query := ctx.getListQuery(c) - - // get results - data, err := svc.List(query, &generic.ListOptions{ - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccessWithListData(c, nil, 0) - return - } - HandleErrorInternalServerError(c, err) - return - } - - // validate results - if len(data) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // total count - total, err := svc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *resultContext) getListQuery(c *gin.Context) (q generic.ListQuery) { - f, err := GetFilter(c) - if err != nil { - return q - } - for _, cond := range f.Conditions { - q = append(q, generic.ListQueryCondition{ - Key: cond.Key, - Op: cond.Op, - Value: utils.NormalizeObjectId(cond.Value), - }) - } - return q -} - -func newResultContext() *resultContext { - // context - ctx := &resultContext{} - - var err error - ctx.modelSvc, err = service.NewService() - if err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/result_v2.go b/core/controllers/result_v2.go index 9033367f..662ef1f4 100644 --- a/core/controllers/result_v2.go +++ b/core/controllers/result_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/utils" @@ -32,17 +32,17 @@ func GetResultList(c *gin.Context) { } // data collection - dc, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(dcId) + dc, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(dcId) if err != nil { HandleErrorInternalServerError(c, err) return } // data source - ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { - ds = &models.DataSourceV2{} + ds = &models2.DatabaseV2{} } else { HandleErrorInternalServerError(c, err) return @@ -54,7 +54,7 @@ func GetResultList(c *gin.Context) { "col_id": dc.Id, "data_source_id": ds.Id, } - s, err := service.NewModelServiceV2[models.SpiderV2]().GetOne(sq, nil) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetOne(sq, nil) if err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/role.go b/core/controllers/role.go deleted file mode 100644 index d4a286e5..00000000 --- a/core/controllers/role.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var RoleController ListController diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go index 91e42ae8..36ee2a26 100644 --- a/core/controllers/router_v2.go +++ b/core/controllers/router_v2.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" "net/http" ) @@ -55,235 +55,212 @@ func InitRoutes(app *gin.Engine) (err error) { // routes groups groups := NewRouterGroups(app) - RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models.DataCollectionV2]()) - RegisterController(groups.AuthGroup, "/data-sources", NewControllerV2[models.DataSourceV2]()) - RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models.EnvironmentV2]()) - RegisterController(groups.AuthGroup, "/gits", NewControllerV2[models.GitV2]()) - RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models.NodeV2]()) - RegisterController(groups.AuthGroup, "/notifications/settings", NewControllerV2[models.SettingV2]()) - RegisterController(groups.AuthGroup, "/permissions", NewControllerV2[models.PermissionV2]()) - RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models.ProjectV2]()) - RegisterController(groups.AuthGroup, "/roles", NewControllerV2[models.RoleV2]()) - RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models.ScheduleV2]( - Action{ + RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]()) + RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]()) + RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]()) + RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{ + { + Method: http.MethodGet, + Path: "", + HandlerFunc: GetProjectList, + }, + }...)) + RegisterController(groups.AuthGroup, "/schedules", NewControllerV2[models2.ScheduleV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostSchedule, }, - Action{ + { Method: http.MethodPut, Path: "/:id", HandlerFunc: PutScheduleById, }, - Action{ + { Method: http.MethodPost, Path: "/:id/enable", HandlerFunc: PostScheduleEnable, }, - Action{ + { Method: http.MethodPost, Path: "/:id/disable", HandlerFunc: PostScheduleDisable, }, - )) - RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models.SpiderV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/spiders", NewControllerV2[models2.SpiderV2]([]Action{ + { Method: http.MethodGet, Path: "/:id", HandlerFunc: GetSpiderById, }, - Action{ + { Method: http.MethodGet, Path: "", HandlerFunc: GetSpiderList, }, - Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostSpider, }, - Action{ + { Method: http.MethodPut, Path: "/:id", HandlerFunc: PutSpiderById, }, - Action{ + { Method: http.MethodDelete, Path: "/:id", HandlerFunc: DeleteSpiderById, }, - Action{ + { Method: http.MethodDelete, Path: "", HandlerFunc: DeleteSpiderList, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/list", HandlerFunc: GetSpiderListDir, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/get", HandlerFunc: GetSpiderFile, }, - Action{ + { Method: http.MethodGet, Path: "/:id/files/info", HandlerFunc: GetSpiderFileInfo, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save", HandlerFunc: PostSpiderSaveFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save/batch", HandlerFunc: PostSpiderSaveFiles, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/save/dir", HandlerFunc: PostSpiderSaveDir, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/rename", HandlerFunc: PostSpiderRenameFile, }, - Action{ + { Method: http.MethodDelete, Path: "/:id/files", HandlerFunc: DeleteSpiderFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/copy", HandlerFunc: PostSpiderCopyFile, }, - Action{ + { Method: http.MethodPost, Path: "/:id/files/export", HandlerFunc: PostSpiderExport, }, - Action{ + { Method: http.MethodPost, Path: "/:id/run", HandlerFunc: PostSpiderRun, }, - Action{ - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: GetSpiderGit, - }, - Action{ - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: GetSpiderGitRemoteRefs, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: PostSpiderGitCheckout, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: PostSpiderGitPull, - }, - Action{ - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: PostSpiderGitCommit, - }, - Action{ + + { Method: http.MethodGet, Path: "/:id/data-source", HandlerFunc: GetSpiderDataSource, }, - Action{ + { Method: http.MethodPost, Path: "/:id/data-source/:ds_id", HandlerFunc: PostSpiderDataSource, }, - )) - RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models.TaskV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/tasks", NewControllerV2[models2.TaskV2]([]Action{ + { Method: http.MethodGet, Path: "/:id", HandlerFunc: GetTaskById, }, - Action{ + { Method: http.MethodGet, Path: "", HandlerFunc: GetTaskList, }, - Action{ + { Method: http.MethodDelete, Path: "/:id", HandlerFunc: DeleteTaskById, }, - Action{ + { Method: http.MethodDelete, Path: "", HandlerFunc: DeleteList, }, - Action{ + { Method: http.MethodPost, Path: "/run", HandlerFunc: PostTaskRun, }, - Action{ + { Method: http.MethodPost, Path: "/:id/restart", HandlerFunc: PostTaskRestart, }, - Action{ + { Method: http.MethodPost, Path: "/:id/cancel", HandlerFunc: PostTaskCancel, }, - Action{ + { Method: http.MethodGet, Path: "/:id/logs", HandlerFunc: GetTaskLogs, }, - Action{ + { Method: http.MethodGet, Path: "/:id/data", HandlerFunc: GetTaskData, }, - )) - RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models.TokenV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/tokens", NewControllerV2[models2.TokenV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostToken, }, - )) - RegisterController(groups.AuthGroup, "/users", NewControllerV2[models.UserV2]( - Action{ + }...)) + RegisterController(groups.AuthGroup, "/users", NewControllerV2[models2.UserV2]([]Action{ + { Method: http.MethodPost, Path: "", HandlerFunc: PostUser, }, - Action{ + { Method: http.MethodPost, Path: "/:id/change-password", HandlerFunc: PostUserChangePassword, }, - Action{ + { Method: http.MethodGet, Path: "/me", HandlerFunc: GetUserMe, }, - Action{ + { Method: http.MethodPut, Path: "/me", HandlerFunc: PutUserById, }, - )) + }...)) RegisterActions(groups.AuthGroup, "/results", []Action{ { @@ -332,6 +309,11 @@ func InitRoutes(app *gin.Engine) (err error) { Path: "/:id", HandlerFunc: GetSetting, }, + { + Method: http.MethodPost, + Path: "/:id", + HandlerFunc: PostSetting, + }, { Method: http.MethodPut, Path: "/:id", @@ -363,13 +345,6 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: GetSystemInfo, }, }) - RegisterActions(groups.AnonymousGroup, "/version", []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - }) RegisterActions(groups.AnonymousGroup, "/", []Action{ { Method: http.MethodPost, @@ -382,6 +357,18 @@ func InitRoutes(app *gin.Engine) (err error) { HandlerFunc: PostLogout, }, }) + RegisterActions(groups.AnonymousGroup, "/sync", []Action{ + { + Method: http.MethodGet, + Path: "/:id/scan", + HandlerFunc: GetSyncScan, + }, + { + Method: http.MethodGet, + Path: "/:id/download", + HandlerFunc: GetSyncDownload, + }, + }) return nil } diff --git a/core/controllers/router_v2_test.go b/core/controllers/router_v2_test.go index 47642324..e0e6f530 100644 --- a/core/controllers/router_v2_test.go +++ b/core/controllers/router_v2_test.go @@ -2,7 +2,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "testing" @@ -28,7 +28,7 @@ func TestRouterGroups(t *testing.T) { func TestRegisterController_Routes(t *testing.T) { router := gin.Default() groups := controllers.NewRouterGroups(router) - ctr := controllers.NewControllerV2[models.TestModel]() + ctr := controllers.NewControllerV2[models.TestModelV2]() basePath := "/testmodels" controllers.RegisterController(groups.AuthGroup, basePath, ctr) diff --git a/core/controllers/schedule.go b/core/controllers/schedule.go deleted file mode 100644 index 71ba2b67..00000000 --- a/core/controllers/schedule.go +++ /dev/null @@ -1,221 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var ScheduleController *scheduleController - -func getScheduleActions() []Action { - scheduleCtx := newScheduleContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/enable", - HandlerFunc: scheduleCtx.enable, - }, - { - Method: http.MethodPost, - Path: "/:id/disable", - HandlerFunc: scheduleCtx.disable, - }, - } -} - -type scheduleController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *scheduleContext -} - -func (ctr *scheduleController) Post(c *gin.Context) { - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := delegate.NewModelDelegate(&s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Put(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var s models.Schedule - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return - } - if s.GetId() != oid { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - if err := delegate.NewModelDelegate(&s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if s.Enabled { - if err := ctr.ctx.scheduleSvc.Disable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Enable(&s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - HandleSuccessWithData(c, s) -} - -func (ctr *scheduleController) Delete(c *gin.Context) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctr.ctx.modelSvc.GetScheduleById(oid) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - HandleErrorInternalServerError(c, err) - return - } -} - -func (ctr *scheduleController) DeleteList(c *gin.Context) { - payload, err := NewJsonBinder(interfaces.ModelIdSchedule).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - for _, id := range payload.Ids { - s, err := ctr.ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := ctr.ctx.scheduleSvc.Disable(s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdSchedule).DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) enable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Enable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) disable(c *gin.Context) { - s, err := ctx._getSchedule(c) - if err != nil { - return - } - if err := ctx.scheduleSvc.Disable(s, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *scheduleContext) _getSchedule(c *gin.Context) (s *models.Schedule, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - s, err = ctx.modelSvc.GetScheduleById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - return s, nil -} - -type scheduleContext struct { - modelSvc service.ModelService - scheduleSvc interfaces.ScheduleService -} - -func newScheduleContext() *scheduleContext { - // context - ctx := &scheduleContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - scheduleSvc interfaces.ScheduleService, - ) { - ctx.modelSvc = modelSvc - ctx.scheduleSvc = scheduleSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newScheduleController() *scheduleController { - actions := getScheduleActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - d := NewListPostActionControllerDelegate(ControllerIdSchedule, modelSvc.GetBaseService(interfaces.ModelIdSchedule), actions) - ctx := newScheduleContext() - - return &scheduleController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/schedule_v2.go b/core/controllers/schedule_v2.go index 3e19da76..9149f2a1 100644 --- a/core/controllers/schedule_v2.go +++ b/core/controllers/schedule_v2.go @@ -2,7 +2,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/schedule" "github.com/gin-gonic/gin" diff --git a/core/controllers/setting.go b/core/controllers/setting.go deleted file mode 100644 index 068e2379..00000000 --- a/core/controllers/setting.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var SettingController *settingController - -type settingController struct { - ListControllerDelegate -} - -func (ctr *settingController) Get(c *gin.Context) { - // key - key := c.Param("id") - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, s) -} - -func (ctr *settingController) Put(c *gin.Context) { - // key - key := c.Param("id") - - // settings - var s models.Setting - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // model service - modelSvc, err := service.NewService() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // setting - _s, err := modelSvc.GetSettingByKey(key, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // save - _s.Value = s.Value - if err := delegate.NewModelDelegate(_s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func newSettingController() *settingController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListControllerDelegate(ControllerIdSetting, modelSvc.GetBaseService(interfaces.ModelIdSetting)) - - return &settingController{ - ListControllerDelegate: *ctr, - } -} diff --git a/core/controllers/setting_v2.go b/core/controllers/setting_v2.go index 326873b9..c4be44cc 100644 --- a/core/controllers/setting_v2.go +++ b/core/controllers/setting_v2.go @@ -1,10 +1,12 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + "errors" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" ) func GetSetting(c *gin.Context) { @@ -14,6 +16,10 @@ func GetSetting(c *gin.Context) { // setting s, err := service.NewModelServiceV2[models.SettingV2]().GetOne(bson.M{"key": key}, nil) if err != nil { + if errors.Is(err, mongo.ErrNoDocuments) { + HandleSuccess(c) + return + } HandleErrorInternalServerError(c, err) return } @@ -21,12 +27,42 @@ func GetSetting(c *gin.Context) { HandleSuccessWithData(c, s) } +func PostSetting(c *gin.Context) { + // key + key := c.Param("id") + + // settings + var s models.SettingV2 + if err := c.ShouldBindJSON(&s); err != nil { + HandleErrorInternalServerError(c, err) + return + } + + if s.Key == "" { + s.Key = key + } + + u := GetUserFromContextV2(c) + + s.SetCreated(u.Id) + s.SetUpdated(u.Id) + + id, err := service.NewModelServiceV2[models.SettingV2]().InsertOne(s) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + s.Id = id + + HandleSuccessWithData(c, s) +} + func PutSetting(c *gin.Context) { // key key := c.Param("id") // settings - var s models.Setting + var s models.SettingV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorInternalServerError(c, err) return diff --git a/core/controllers/spider.go b/core/controllers/spider.go deleted file mode 100644 index f05da372..00000000 --- a/core/controllers/spider.go +++ /dev/null @@ -1,1333 +0,0 @@ -package controllers - -import ( - "bytes" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - fs2 "github.com/crawlab-team/crawlab/core/fs" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" - "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" - "math" - "net/http" - "os" - "path/filepath" - "strings" -) - -var SpiderController *spiderController - -func getSpiderActions() []Action { - ctx := newSpiderContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/files/list", - HandlerFunc: ctx.listDir, - }, - { - Method: http.MethodGet, - Path: "/:id/files/get", - HandlerFunc: ctx.getFile, - }, - { - Method: http.MethodGet, - Path: "/:id/files/info", - HandlerFunc: ctx.getFileInfo, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save", - HandlerFunc: ctx.saveFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/save/dir", - HandlerFunc: ctx.saveDir, - }, - { - Method: http.MethodPost, - Path: "/:id/files/rename", - HandlerFunc: ctx.renameFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/delete", - HandlerFunc: ctx.deleteFile, - }, - { - Method: http.MethodPost, - Path: "/:id/files/copy", - HandlerFunc: ctx.copyFile, - }, - { - Path: "/:id/files/export", - Method: http.MethodPost, - HandlerFunc: ctx.postExport, - }, - { - Method: http.MethodPost, - Path: "/:id/run", - HandlerFunc: ctx.run, - }, - { - Method: http.MethodGet, - Path: "/:id/git", - HandlerFunc: ctx.getGit, - }, - { - Method: http.MethodGet, - Path: "/:id/git/remote-refs", - HandlerFunc: ctx.getGitRemoteRefs, - }, - { - Method: http.MethodPost, - Path: "/:id/git/checkout", - HandlerFunc: ctx.gitCheckout, - }, - { - Method: http.MethodPost, - Path: "/:id/git/pull", - HandlerFunc: ctx.gitPull, - }, - { - Method: http.MethodPost, - Path: "/:id/git/commit", - HandlerFunc: ctx.gitCommit, - }, - //{ - // Method: http.MethodPost, - // Path: "/:id/clone", - // HandlerFunc: ctx.clone, - //}, - { - Path: "/:id/data-source", - Method: http.MethodGet, - HandlerFunc: ctx.getDataSource, - }, - { - Path: "/:id/data-source/:ds_id", - Method: http.MethodPost, - HandlerFunc: ctx.postDataSource, - }, - } -} - -type spiderController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *spiderContext -} - -func (ctr *spiderController) Get(c *gin.Context) { - ctr.ctx._get(c) -} - -func (ctr *spiderController) Post(c *gin.Context) { - s, err := ctr.ctx._post(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Put(c *gin.Context) { - s, err := ctr.ctx._put(c) - if err != nil { - return - } - HandleSuccessWithData(c, s) -} - -func (ctr *spiderController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *spiderController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx._getListWithStats(c) -} - -func (ctr *spiderController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type spiderContext struct { - modelSvc service.ModelService - modelSpiderSvc interfaces.ModelBaseService - modelSpiderStatSvc interfaces.ModelBaseService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService -} - -func (ctx *spiderContext) listDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - files, err := fsSvc.List(payload.Path) - if err != nil { - if err.Error() != "response status code: 404" { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccessWithData(c, files) -} - -func (ctx *spiderContext) getFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - data, err := fsSvc.GetFile(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - data = utils.TrimFileData(data) - - HandleSuccessWithData(c, string(data)) -} - -func (ctx *spiderContext) getFileInfo(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodGet) - if err != nil { - return - } - - info, err := fsSvc.GetFileInfo(payload.Path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, info) -} - -func (ctx *spiderContext) saveFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) saveDir(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) renameFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) deleteFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) copyFile(c *gin.Context) { - _, payload, fsSvc, err := ctx._processFileRequest(c, http.MethodPost) - if err != nil { - return - } - - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) run(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // options - var opts interfaces.SpiderRunOptions - if err := c.ShouldBindJSON(&opts); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // schedule - taskIds, err := ctx.adminSvc.Schedule(id, &opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *spiderContext) getGit(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := ctx._getGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func (ctx *spiderContext) getGitRemoteRefs(c *gin.Context) { - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func (ctx *spiderContext) gitCheckout(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := ctx._gitCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitPull(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - _ = ctx.adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func (ctx *spiderContext) gitCommit(c *gin.Context) { - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider id - id, err := ctx._processActionRequest(c) - if err != nil { - return - } - - // git client - gitClient, err := ctx._getGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) getDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - ds, err := ctx.modelSvc.GetDataSourceById(s.DataSourceId) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, ds) -} - -func (ctx *spiderContext) postDataSource(c *gin.Context) { - // spider id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // data source id - dsId, err := primitive.ObjectIDFromHex(c.Param("ds_id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data source - if !dsId.IsZero() { - _, err = ctx.modelSvc.GetDataSourceById(dsId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // save data source id - s.DataSourceId = dsId - if err := delegate2.NewModelDelegate(s).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *spiderContext) postExport(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // zip file path - zipFilePath, err := ctx.adminSvc.Export(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // download - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) - c.File(zipFilePath) -} - -func (ctx *spiderContext) _get(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - s, err := ctx.modelSvc.GetSpiderById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat - s.Stat, err = ctx.modelSvc.GetSpiderStatById(s.GetId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // data collection - if !s.ColId.IsZero() { - col, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - if err != mongo2.ErrNoDocuments { - HandleErrorInternalServerError(c, err) - return - } - } else { - s.ColName = col.Name - } - } - - HandleSuccessWithData(c, s) -} - -func (ctx *spiderContext) _post(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // add stat - st := &models.SpiderStat{ - Id: s.GetId(), - } - if err := delegate2.NewModelDelegate(st, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _put(c *gin.Context) (s *models.Spider, err error) { - // bind - s = &models.Spider{} - if err := c.ShouldBindJSON(&s); err != nil { - HandleErrorBadRequest(c, err) - return nil, err - } - - // upsert data collection - if err := ctx._upsertDataCollection(c, s); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - // save - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return nil, err - } - - return s, nil -} - -func (ctx *spiderContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spider - s, err := ctx.modelSvc.GetSpiderById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(s, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete spider stat - ss, err := ctx.modelSvc.GetSpiderStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(ss, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": oid}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - l, err := ctx.modelSpiderSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(l.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range l.GetModels() { - s := d.(*models.Spider) - ids = append(ids, s.GetId()) - } - - // total count - total, err := ctx.modelSpiderSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetSpiderStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.SpiderStat{} - var tids []primitive.ObjectID - for _, st := range stats { - if st.Tasks > 0 { - taskCount := int64(st.Tasks) - st.AverageWaitDuration = int64(math.Round(float64(st.WaitDuration) / float64(taskCount))) - st.AverageRuntimeDuration = int64(math.Round(float64(st.RuntimeDuration) / float64(taskCount))) - st.AverageTotalDuration = int64(math.Round(float64(st.TotalDuration) / float64(taskCount))) - } - dict[st.GetId()] = st - - if !st.LastTaskId.IsZero() { - tids = append(tids, st.LastTaskId) - } - } - - // task list and stats - var tasks []models.Task - dictTask := map[primitive.ObjectID]models.Task{} - dictTaskStat := map[primitive.ObjectID]models.TaskStat{} - if len(tids) > 0 { - // task list - queryTask := bson.M{ - "_id": bson.M{ - "$in": tids, - }, - } - tasks, err = ctx.modelSvc.GetTaskList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // task stats list - taskStats, err := ctx.modelSvc.GetTaskStatList(queryTask, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache task stats to dict - for _, st := range taskStats { - dictTaskStat[st.GetId()] = st - } - - // cache task list to dict - for _, t := range tasks { - st, ok := dictTaskStat[t.GetId()] - if ok { - t.Stat = &st - } - dictTask[t.GetSpiderId()] = t - } - } - - // iterate list again - var data []interface{} - for _, d := range l.GetModels() { - s := d.(*models.Spider) - - // spider stat - st, ok := dict[s.GetId()] - if ok { - s.Stat = &st - - // last task - t, ok := dictTask[s.GetId()] - if ok { - s.Stat.LastTask = &t - } - } - - // add to list - data = append(data, *s) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *spiderContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdSpider).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete spiders - if err := ctx.modelSpiderSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete spider stats - if err := ctx.modelSpiderStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // related tasks - tasks, err := ctx.modelSvc.GetTaskList(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) - if err != nil { - return err - } - - // task ids - var taskIds []primitive.ObjectID - for _, t := range tasks { - taskIds = append(taskIds, t.Id) - } - - // delete related tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - // delete related task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *spiderContext) _processFileRequest(c *gin.Context, method string) (id primitive.ObjectID, payload entity.FileRequestPayload, fsSvc interfaces.FsServiceV2, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - contentType := c.GetHeader("Content-Type") - if strings.HasPrefix(contentType, "multipart/form-data") { - // multipart/form-data - payload, err = ctx._getFileRequestMultipartPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - } else { - // query or application/json - switch method { - case http.MethodGet: - err = c.ShouldBindQuery(&payload) - default: - err = c.ShouldBindJSON(&payload) - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // fs service - workspacePath := viper.GetString("workspace") - fsSvc = fs2.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) - - return -} - -func (ctx *spiderContext) _getFileRequestMultipartPayload(c *gin.Context) (payload entity.FileRequestPayload, err error) { - fh, err := c.FormFile("file") - if err != nil { - return - } - f, err := fh.Open() - if err != nil { - return - } - buf := bytes.NewBuffer(nil) - if _, err = io.Copy(buf, f); err != nil { - return - } - payload.Path = c.PostForm("path") - payload.Data = buf.String() - return -} - -func (ctx *spiderContext) _processActionRequest(c *gin.Context) (id primitive.ObjectID, err error) { - // id - id, err = primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - return -} - -func (ctx *spiderContext) _upsertDataCollection(c *gin.Context, s *models.Spider) (err error) { - if s.ColId.IsZero() { - // validate - if s.ColName == "" { - return trace.TraceError(errors.ErrorControllerMissingRequestFields) - } - // no id - dc, err := ctx.modelSvc.GetDataCollectionByName(s.ColName, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - // not exists, add new - dc = &models.DataCollection{Name: s.ColName} - if err := delegate2.NewModelDelegate(dc, GetUserFromContext(c)).Add(); err != nil { - return err - } - } else { - // error - return err - } - } - s.ColId = dc.Id - - // create index - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) - } else { - // with id - dc, err := ctx.modelSvc.GetDataCollectionById(s.ColId) - if err != nil { - return err - } - s.ColId = dc.Id - } - return nil -} - -func (ctx *spiderContext) _getGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func (ctx *spiderContext) _gitCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return ctx._gitPull(gitClient, remote, branch) -} - -func (ctx *spiderContext) _gitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func (ctx *spiderContext) _getGitClient(id primitive.ObjectID) (gitClient *vcs.GitClient, err error) { - // git - g, err := ctx.modelSvc.GetGitById(id) - if err != nil { - if err != mongo2.ErrNoDocuments { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) - if err != nil { - return nil, err - } - - // set auth - utils.InitGitClientAuth(g, gitClient) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := gitClient.GetRemote(remoteName) - if err == git.ErrRemoteNotFound { - // remote not exists, create - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := gitClient.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := gitClient.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - gitClient.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = gitClient.GetRepository().Head() - if err == nil { - return gitClient, nil - } - - // align master/main branch - ctx._alignBranch(gitClient) - - return gitClient, nil -} - -func (ctx *spiderContext) _alignBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - trace.PrintError(err) - return - } - - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) - if err != nil { - trace.PrintError(err) - return - } - - // main branch - defaultRemoteBranch, err := ctx._getDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return - } - - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) - } -} - -func (ctx *spiderContext) _getDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -var _spiderCtx *spiderContext - -func newSpiderContext() *spiderContext { - if _spiderCtx != nil { - return _spiderCtx - } - - // context - ctx := &spiderContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - }); err != nil { - panic(err) - } - - // model spider service - ctx.modelSpiderSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpider) - - // model spider stat service - ctx.modelSpiderStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdSpiderStat) - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - _spiderCtx = ctx - - return ctx -} - -func newSpiderController() *spiderController { - actions := getSpiderActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - d := NewListPostActionControllerDelegate(ControllerIdSpider, modelSvc.GetBaseService(interfaces.ModelIdSpider), actions) - ctx := newSpiderContext() - - return &spiderController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/spider_v2.go b/core/controllers/spider_v2.go index 9e81c6a2..76d27038 100644 --- a/core/controllers/spider_v2.go +++ b/core/controllers/spider_v2.go @@ -2,31 +2,23 @@ package controllers import ( "errors" - "fmt" - log2 "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/fs" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" "github.com/gin-gonic/gin" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/config" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" "math" "os" "path/filepath" - "strings" "sync" ) @@ -36,7 +28,7 @@ func GetSpiderById(c *gin.Context) { HandleErrorBadRequest(c, err) return } - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorNotFound(c, err) return @@ -47,7 +39,7 @@ func GetSpiderById(c *gin.Context) { } // stat - s.Stat, err = service.NewModelServiceV2[models.SpiderStatV2]().GetById(s.Id) + s.Stat, err = service.NewModelServiceV2[models2.SpiderStatV2]().GetById(s.Id) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -55,9 +47,9 @@ func GetSpiderById(c *gin.Context) { } } - // data collection - if !s.ColId.IsZero() { - col, err := service.NewModelServiceV2[models.DataCollectionV2]().GetById(s.ColId) + // data collection (compatible to old version) # TODO: remove in the future + if s.ColName == "" && !s.ColId.IsZero() { + col, err := service.NewModelServiceV2[models2.DataCollectionV2]().GetById(s.ColId) if err != nil { if !errors.Is(err, mongo2.ErrNoDocuments) { HandleErrorInternalServerError(c, err) @@ -68,23 +60,47 @@ func GetSpiderById(c *gin.Context) { } } + // git + if utils.IsPro() && !s.GitId.IsZero() { + s.Git, err = service.NewModelServiceV2[models2.GitV2]().GetById(s.GitId) + if err != nil { + if !errors.Is(err, mongo2.ErrNoDocuments) { + HandleErrorInternalServerError(c, err) + return + } + } + } + HandleSuccessWithData(c, s) } func GetSpiderList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - NewControllerV2[models.SpiderV2]().GetList(c) + // get all list + all := MustGetFilterAll(c) + if all { + NewControllerV2[models2.SpiderV2]().getAll(c) return } + // get list + withStats := c.Query("stats") + if withStats == "" { + NewControllerV2[models2.SpiderV2]().GetList(c) + return + } + + // get list with stats + getSpiderListWithStats(c) +} + +func getSpiderListWithStats(c *gin.Context) { // params pagination := MustGetPagination(c) query := MustGetFilterQuery(c) sort := MustGetSortOption(c) // get list - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(query, &mongo.FindOptions{ + spiders, err := service.NewModelServiceV2[models2.SpiderV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -96,32 +112,36 @@ func GetSpiderList(c *gin.Context) { return } if len(spiders) == 0 { - HandleSuccessWithListData(c, []models.SpiderV2{}, 0) + HandleSuccessWithListData(c, []models2.SpiderV2{}, 0) return } // ids var ids []primitive.ObjectID + var gitIds []primitive.ObjectID for _, s := range spiders { ids = append(ids, s.Id) + if !s.GitId.IsZero() { + gitIds = append(gitIds, s.GitId) + } } // total count - total, err := service.NewModelServiceV2[models.SpiderV2]().Count(query) + total, err := service.NewModelServiceV2[models2.SpiderV2]().Count(query) if err != nil { HandleErrorInternalServerError(c, err) return } // stat list - spiderStats, err := service.NewModelServiceV2[models.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) + spiderStats, err := service.NewModelServiceV2[models2.SpiderStatV2]().GetMany(bson.M{"_id": bson.M{"$in": ids}}, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // cache stat list to dict - dict := map[primitive.ObjectID]models.SpiderStatV2{} + dict := map[primitive.ObjectID]models2.SpiderStatV2{} var taskIds []primitive.ObjectID for _, st := range spiderStats { if st.Tasks > 0 { @@ -138,9 +158,9 @@ func GetSpiderList(c *gin.Context) { } // task list and stats - var tasks []models.TaskV2 - dictTask := map[primitive.ObjectID]models.TaskV2{} - dictTaskStat := map[primitive.ObjectID]models.TaskStatV2{} + var tasks []models2.TaskV2 + dictTask := map[primitive.ObjectID]models2.TaskV2{} + dictTaskStat := map[primitive.ObjectID]models2.TaskStatV2{} if len(taskIds) > 0 { // task list queryTask := bson.M{ @@ -148,14 +168,14 @@ func GetSpiderList(c *gin.Context) { "$in": taskIds, }, } - tasks, err = service.NewModelServiceV2[models.TaskV2]().GetMany(queryTask, nil) + tasks, err = service.NewModelServiceV2[models2.TaskV2]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // task stats list - taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(queryTask, nil) + taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(queryTask, nil) if err != nil { HandleErrorInternalServerError(c, err) return @@ -176,8 +196,24 @@ func GetSpiderList(c *gin.Context) { } } + // git list + var gits []models2.GitV2 + if len(gitIds) > 0 && utils.IsPro() { + gits, err = service.NewModelServiceV2[models2.GitV2]().GetMany(bson.M{"_id": bson.M{"$in": gitIds}}, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + } + + // cache git list to dict + dictGit := map[primitive.ObjectID]models2.GitV2{} + for _, g := range gits { + dictGit[g.Id] = g + } + // iterate list again - var data []models.SpiderV2 + var data []models2.SpiderV2 for _, s := range spiders { // spider stat st, ok := dict[s.Id] @@ -191,6 +227,14 @@ func GetSpiderList(c *gin.Context) { } } + // git + if !s.GitId.IsZero() && utils.IsPro() { + g, ok := dictGit[s.GitId] + if ok { + s.Git = &g + } + } + // add to list data = append(data, s) } @@ -201,24 +245,19 @@ func GetSpiderList(c *gin.Context) { func PostSpider(c *gin.Context) { // bind - var s models.SpiderV2 + var s models2.SpiderV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return } - // upsert data collection - if err := upsertSpiderDataCollection(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - + // user u := GetUserFromContextV2(c) // add s.SetCreated(u.Id) s.SetUpdated(u.Id) - id, err := service.NewModelServiceV2[models.SpiderV2]().InsertOne(s) + id, err := service.NewModelServiceV2[models2.SpiderV2]().InsertOne(s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -226,18 +265,23 @@ func PostSpider(c *gin.Context) { s.SetId(id) // add stat - st := models.SpiderStatV2{} + st := models2.SpiderStatV2{} st.SetId(id) st.SetCreated(u.Id) st.SetUpdated(u.Id) - _, err = service.NewModelServiceV2[models.SpiderStatV2]().InsertOne(st) + _, err = service.NewModelServiceV2[models2.SpiderStatV2]().InsertOne(st) if err != nil { HandleErrorInternalServerError(c, err) return } // create folder - err = getSpiderFsSvcById(id).CreateDir(".") + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + err = fsSvc.CreateDir(".") if err != nil { HandleErrorInternalServerError(c, err) return @@ -254,21 +298,15 @@ func PutSpiderById(c *gin.Context) { } // bind - var s models.SpiderV2 + var s models2.SpiderV2 if err := c.ShouldBindJSON(&s); err != nil { HandleErrorBadRequest(c, err) return } - // upsert data collection - if err := upsertSpiderDataCollection(&s); err != nil { - HandleErrorInternalServerError(c, err) - return - } - u := GetUserFromContextV2(c) - modelSvc := service.NewModelServiceV2[models.SpiderV2]() + modelSvc := service.NewModelServiceV2[models2.SpiderV2]() // save s.SetUpdated(u.Id) @@ -297,19 +335,19 @@ func DeleteSpiderById(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spider - err = service.NewModelServiceV2[models.SpiderV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.SpiderV2]().DeleteById(id) if err != nil { return err } // delete spider stat - err = service.NewModelServiceV2[models.SpiderStatV2]().DeleteById(id) + err = service.NewModelServiceV2[models2.SpiderStatV2]().DeleteById(id) if err != nil { return err } // related tasks - tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) + tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": id}, nil) if err != nil { return err } @@ -325,13 +363,13 @@ func DeleteSpiderById(c *gin.Context) { } // delete related tasks - err = service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } // delete related task stats - err = service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) + err = service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}) if err != nil { return err } @@ -344,7 +382,7 @@ func DeleteSpiderById(c *gin.Context) { // delete task logs logPath := filepath.Join(viper.GetString("log.path"), id) if err := os.RemoveAll(logPath); err != nil { - log2.Warnf("failed to remove task log directory: %s", logPath) + log.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() }(id.Hex()) @@ -357,6 +395,35 @@ func DeleteSpiderById(c *gin.Context) { return } + go func() { + // spider + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) + if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) + return + } + + // skip spider with git + if !s.GitId.IsZero() { + return + } + + // delete spider directory + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + log.Errorf("failed to get spider fs service: %s", err.Error()) + trace.PrintError(err) + return + } + err = fsSvc.Delete(".") + if err != nil { + log.Errorf("failed to delete spider directory: %s", err.Error()) + trace.PrintError(err) + return + } + }() + HandleSuccess(c) } @@ -371,7 +438,7 @@ func DeleteSpiderList(c *gin.Context) { if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { // delete spiders - if err := service.NewModelServiceV2[models.SpiderV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.SpiderV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -380,7 +447,7 @@ func DeleteSpiderList(c *gin.Context) { } // delete spider stats - if err := service.NewModelServiceV2[models.SpiderStatV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.SpiderStatV2]().DeleteMany(bson.M{ "_id": bson.M{ "$in": payload.Ids, }, @@ -389,7 +456,7 @@ func DeleteSpiderList(c *gin.Context) { } // related tasks - tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) + tasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{"spider_id": bson.M{"$in": payload.Ids}}, nil) if err != nil { return err } @@ -405,12 +472,12 @@ func DeleteSpiderList(c *gin.Context) { } // delete related tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } // delete related task stats - if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { + if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{"_id": bson.M{"$in": taskIds}}); err != nil { return err } @@ -422,7 +489,7 @@ func DeleteSpiderList(c *gin.Context) { // delete task logs logPath := filepath.Join(viper.GetString("log.path"), id) if err := os.RemoveAll(logPath); err != nil { - log2.Warnf("failed to remove task log directory: %s", logPath) + log.Warnf("failed to remove task log directory: %s", logPath) } wg.Done() }(id.Hex()) @@ -435,290 +502,137 @@ func DeleteSpiderList(c *gin.Context) { return } + // delete spider directories + go func() { + wg := sync.WaitGroup{} + wg.Add(len(payload.Ids)) + for _, id := range payload.Ids { + go func(id primitive.ObjectID) { + defer wg.Done() + + // spider + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) + if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) + return + } + + // skip spider with git + if !s.GitId.IsZero() { + return + } + + // delete spider directory + fsSvc, err := getSpiderFsSvcById(id) + if err != nil { + log.Errorf("failed to get spider fs service: %s", err.Error()) + trace.PrintError(err) + return + } + err = fsSvc.Delete(".") + if err != nil { + log.Errorf("failed to delete spider directory: %s", err.Error()) + trace.PrintError(err) + return + } + }(id) + } + wg.Wait() + }() + HandleSuccess(c) } func GetSpiderListDir(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - files, err := fsSvc.List(path) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccessWithData(c, files) + GetBaseFileListDir(rootPath, c) } func GetSpiderFile(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - data, err := fsSvc.GetFile(path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, string(data)) + GetBaseFileFile(rootPath, c) } func GetSpiderFileInfo(c *gin.Context) { - path := c.Query("path") - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - info, err := fsSvc.GetFileInfo(path) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, info) + GetBaseFileFileInfo(rootPath, c) } func PostSpiderSaveFile(c *gin.Context) { - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - - if c.GetHeader("Content-Type") == "application/json" { - var payload struct { - Path string `json:"path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(payload.Path, []byte(payload.Data)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - path, ok := c.GetPostForm("path") - if !ok { - HandleErrorBadRequest(c, errors.New("missing required field 'path'")) - return - } - file, err := c.FormFile("file") - if err != nil { - HandleErrorBadRequest(c, err) - return - } - f, err := file.Open() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - fileData, err := io.ReadAll(f) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := fsSvc.Save(path, fileData); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - HandleSuccess(c) + PostBaseFileSaveFile(rootPath, c) } func PostSpiderSaveFiles(c *gin.Context) { - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } - - form, err := c.MultipartForm() - if err != nil { - HandleErrorBadRequest(c, err) - return - } - wg := sync.WaitGroup{} - wg.Add(len(form.File)) - for path := range form.File { - go func(path string) { - file, err := c.FormFile(path) - if err != nil { - log2.Warnf("invalid file header: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - f, err := file.Open() - if err != nil { - log2.Warnf("unable to open file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - fileData, err := io.ReadAll(f) - if err != nil { - log2.Warnf("unable to read file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - if err := fsSvc.Save(path, fileData); err != nil { - log2.Warnf("unable to save file: %s", path) - log2.Error(err.Error()) - wg.Done() - return - } - wg.Done() - }(path) - } - wg.Wait() - - HandleSuccess(c) + targetDirectory := c.PostForm("targetDirectory") + PostBaseFileSaveFiles(filepath.Join(rootPath, targetDirectory), c) } func PostSpiderSaveDir(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - Data string `json:"data"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - if err := fsSvc.CreateDir(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) + PostBaseFileSaveDir(rootPath, c) } func PostSpiderRenameFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := fsSvc.Rename(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) + HandleErrorForbidden(c, err) return } + PostBaseFileRenameFile(rootPath, c) } func DeleteSpiderFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - if payload.Path == "~" { - payload.Path = "." - } - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - if err := fsSvc.Delete(payload.Path); err != nil { - HandleErrorInternalServerError(c, err) - return - } - _, err = fsSvc.GetFileInfo(".") - if err != nil { - _ = fsSvc.CreateDir("/") - } - - HandleSuccess(c) + DeleteBaseFileFile(rootPath, c) } func PostSpiderCopyFile(c *gin.Context) { - var payload struct { - Path string `json:"path"` - NewPath string `json:"new_path"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - fsSvc, err := getSpiderFsSvc(c) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - if err := fsSvc.Copy(payload.Path, payload.NewPath); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) + PostBaseFileCopyFile(rootPath, c) } func PostSpiderExport(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) + rootPath, err := getSpiderRootPath(c) if err != nil { - HandleErrorBadRequest(c, err) + HandleErrorForbidden(c, err) return } - - adminSvc, err := admin.GetSpiderAdminServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // zip file path - zipFilePath, err := adminSvc.Export(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // download - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFilePath)) - c.File(zipFilePath) + PostBaseFileExport(rootPath, c) } func PostSpiderRun(c *gin.Context) { @@ -756,266 +670,6 @@ func PostSpiderRun(c *gin.Context) { HandleSuccessWithData(c, taskIds) } -func GetSpiderGit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // current branch - currentBranch, err := gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // branches - branches, err := gitClient.GetBranches() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if branches == nil || len(branches) == 0 && currentBranch != "" { - branches = []vcs.GitRef{{Name: currentBranch}} - } - - // changes - changes, err := gitClient.GetStatus() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // logs - logs, err := gitClient.GetLogsWithRefs() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // ignore - ignore, err := getSpiderGitIgnore(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // git - _git, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if err.Error() != mongo2.ErrNoDocuments.Error() { - HandleErrorInternalServerError(c, err) - return - } - } - - // response - res := bson.M{ - "current_branch": currentBranch, - "branches": branches, - "changes": changes, - "logs": logs, - "ignore": ignore, - "git": _git, - } - - HandleSuccessWithData(c, res) -} - -func GetSpiderGitRemoteRefs(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // remote name - remoteName := c.Query("remote") - if remoteName == "" { - remoteName = vcs.GitRemoteNameOrigin - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // refs - refs, err := gitClient.GetRemoteRefs(remoteName) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, refs) -} - -func PostSpiderGitCheckout(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // branch to pull - var branch string - if payload.Branch == "" { - // by default current branch - branch, err = gitClient.GetCurrentBranch() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - } else { - // payload branch - branch = payload.Branch - } - - // checkout - if err := gitSpiderCheckout(gitClient, constants.GitRemoteNameOrigin, branch); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func PostSpiderGitPull(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload struct { - Paths []string `json:"paths"` - CommitMessage string `json:"commit_message"` - Branch string `json:"branch"` - Tag string `json:"tag"` - } - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // attempt to sync git - adminSvc, err := admin.GetSpiderAdminServiceV2() - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - _ = adminSvc.SyncGitOne(g) - - HandleSuccess(c) -} - -func PostSpiderGitCommit(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // payload - var payload entity.GitPayload - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // git client - gitClient, err := getSpiderGitClient(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // return null if git client is empty - if gitClient == nil { - HandleSuccess(c) - return - } - - // add - for _, p := range payload.Paths { - if err := gitClient.Add(p); err != nil { - HandleErrorInternalServerError(c, err) - return - } - } - - // commit - if err := gitClient.Commit(payload.CommitMessage); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // push - if err := gitClient.Push( - vcs.WithRemoteNamePush(vcs.GitRemoteNameOrigin), - ); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - func GetSpiderDataSource(c *gin.Context) { // spider id id, err := primitive.ObjectIDFromHex(c.Param("id")) @@ -1025,14 +679,14 @@ func GetSpiderDataSource(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return } // data source - ds, err := service.NewModelServiceV2[models.DataSourceV2]().GetById(s.DataSourceId) + ds, err := service.NewModelServiceV2[models2.DatabaseV2]().GetById(s.DataSourceId) if err != nil { if err.Error() == mongo2.ErrNoDocuments.Error() { HandleSuccess(c) @@ -1061,7 +715,7 @@ func PostSpiderDataSource(c *gin.Context) { } // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { HandleErrorInternalServerError(c, err) return @@ -1069,7 +723,7 @@ func PostSpiderDataSource(c *gin.Context) { // data source if !dsId.IsZero() { - _, err = service.NewModelServiceV2[models.DataSourceV2]().GetById(dsId) + _, err = service.NewModelServiceV2[models2.DatabaseV2]().GetById(dsId) if err != nil { HandleErrorInternalServerError(c, err) return @@ -1080,7 +734,7 @@ func PostSpiderDataSource(c *gin.Context) { u := GetUserFromContextV2(c) s.DataSourceId = dsId s.SetUpdatedBy(u.Id) - _, err = service.NewModelServiceV2[models.SpiderV2]().InsertOne(*s) + _, err = service.NewModelServiceV2[models2.SpiderV2]().InsertOne(*s) if err != nil { HandleErrorInternalServerError(c, err) return @@ -1089,221 +743,45 @@ func PostSpiderDataSource(c *gin.Context) { HandleSuccess(c) } -func getSpiderFsSvc(c *gin.Context) (svc interfaces.FsServiceV2, err error) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - return nil, err - } - +func getSpiderFsSvc(s *models2.SpiderV2) (svc interfaces.FsServiceV2, err error) { workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) + fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, s.Id.Hex())) return fsSvc, nil } -func getSpiderFsSvcById(id primitive.ObjectID) interfaces.FsServiceV2 { - workspacePath := viper.GetString("workspace") - fsSvc := fs.NewFsServiceV2(filepath.Join(workspacePath, id.Hex())) - return fsSvc -} - -func getSpiderGitClient(id primitive.ObjectID) (client *vcs.GitClient, err error) { - // git - g, err := service.NewModelServiceV2[models.GitV2]().GetById(id) - if err != nil { - if !errors.Is(err, mongo2.ErrNoDocuments) { - return nil, trace.TraceError(err) - } - return nil, nil - } - - // git client - workspacePath := viper.GetString("workspace") - client, err = vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, id.Hex()))) +func getSpiderFsSvcById(id primitive.ObjectID) (svc interfaces.FsServiceV2, err error) { + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { + log.Errorf("failed to get spider: %s", err.Error()) + trace.PrintError(err) return nil, err } - - // set auth - utils.InitGitClientAuthV2(g, client) - - // remote name - remoteName := vcs.GitRemoteNameOrigin - - // update remote - r, err := client.GetRemote(remoteName) - if errors.Is(err, git.ErrRemoteNotFound) { - // remote not exists, create - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } else if err == nil { - // remote exists, update if different - if g.Url != r.Config().URLs[0] { - if err := client.DeleteRemote(remoteName); err != nil { - return nil, trace.TraceError(err) - } - if _, err := client.CreateRemote(&config.RemoteConfig{ - Name: remoteName, - URLs: []string{g.Url}, - }); err != nil { - return nil, trace.TraceError(err) - } - } - client.SetRemoteUrl(g.Url) - } else { - // error - return nil, trace.TraceError(err) - } - - // check if head reference exists - _, err = client.GetRepository().Head() - if err == nil { - return client, nil - } - - // align master/main branch - alignSpiderGitBranch(client) - - return client, nil + return getSpiderFsSvc(s) } -func alignSpiderGitBranch(gitClient *vcs.GitClient) { - // current branch - currentBranch, err := gitClient.GetCurrentBranch() +func getSpiderRootPath(c *gin.Context) (rootPath string, err error) { + // spider id + id, err := primitive.ObjectIDFromHex(c.Param("id")) if err != nil { - trace.PrintError(err) - return + return "", err } - // skip if current branch is not master - if currentBranch != vcs.GitBranchNameMaster { - return - } - - // remote refs - refs, err := gitClient.GetRemoteRefs(vcs.GitRemoteNameOrigin) + // spider + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { - trace.PrintError(err) - return + return "", err } - // main branch - defaultRemoteBranch, err := getSpiderDefaultRemoteBranch(refs) - if err != nil || defaultRemoteBranch == "" { - return + // check git permission + if !utils.IsPro() && !s.GitId.IsZero() { + return "", errors.New("git is not allowed in the community version") } - // move branch - if err := gitClient.MoveBranch(vcs.GitBranchNameMaster, defaultRemoteBranch); err != nil { - trace.PrintError(err) + // if git id is zero, return spider id as root path + if s.GitId.IsZero() { + return id.Hex(), nil } -} - -func getSpiderDefaultRemoteBranch(refs []vcs.GitRef) (defaultRemoteBranchName string, err error) { - // remote branch name - for _, r := range refs { - if r.Type != vcs.GitRefTypeBranch { - continue - } - - if r.Name == vcs.GitBranchNameMain { - defaultRemoteBranchName = r.Name - break - } - - if r.Name == vcs.GitBranchNameMaster { - defaultRemoteBranchName = r.Name - continue - } - - if defaultRemoteBranchName == "" { - defaultRemoteBranchName = r.Name - continue - } - } - - return defaultRemoteBranchName, nil -} - -func getSpiderGitIgnore(id primitive.ObjectID) (ignore []string, err error) { - workspacePath := viper.GetString("workspace") - filePath := filepath.Join(workspacePath, id.Hex(), ".gitignore") - if !utils.Exists(filePath) { - return nil, nil - } - data, err := os.ReadFile(filePath) - if err != nil { - return nil, trace.TraceError(err) - } - ignore = strings.Split(string(data), "\n") - return ignore, nil -} - -func gitSpiderCheckout(gitClient *vcs.GitClient, remote, branch string) (err error) { - if err := gitClient.CheckoutBranch(branch, vcs.WithBranch(branch)); err != nil { - return trace.TraceError(err) - } - - // pull - return spiderGitPull(gitClient, remote, branch) -} - -func spiderGitPull(gitClient *vcs.GitClient, remote, branch string) (err error) { - // pull - if err := gitClient.Pull( - vcs.WithRemoteNamePull(remote), - vcs.WithBranchNamePull(branch), - ); err != nil { - return trace.TraceError(err) - } - - // reset - if err := gitClient.Reset(); err != nil { - return trace.TraceError(err) - } - - return nil -} - -func upsertSpiderDataCollection(s *models.SpiderV2) (err error) { - modelSvc := service.NewModelServiceV2[models.DataCollectionV2]() - if s.ColId.IsZero() { - // validate - if s.ColName == "" { - return errors.New("data collection name is required") - } - // no id - dc, err := modelSvc.GetOne(bson.M{"name": s.ColName}, nil) - if err != nil { - if errors.Is(err, mongo2.ErrNoDocuments) { - // not exists, add new - dc = &models.DataCollectionV2{Name: s.ColName} - dcId, err := modelSvc.InsertOne(*dc) - if err != nil { - return err - } - dc.SetId(dcId) - } else { - // error - return err - } - } - s.ColId = dc.Id - - // create index - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.TaskKey: 1}}) - _ = mongo.GetMongoCol(dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{constants.HashKey: 1}}) - } else { - // with id - dc, err := modelSvc.GetById(s.ColId) - if err != nil { - return err - } - s.ColId = dc.Id - } - return nil + + return filepath.Join(s.GitId.Hex(), s.GitRootPath), nil } diff --git a/core/controllers/spider_v2_test.go b/core/controllers/spider_v2_test.go index 69c50f28..a744ba85 100644 --- a/core/controllers/spider_v2_test.go +++ b/core/controllers/spider_v2_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" diff --git a/core/controllers/stats.go b/core/controllers/stats.go deleted file mode 100644 index 3d474956..00000000 --- a/core/controllers/stats.go +++ /dev/null @@ -1,87 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "time" -) - -var StatsController ActionController - -func getStatsActions() []Action { - statsCtx := newStatsContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/overview", - HandlerFunc: statsCtx.getOverview, - }, - { - Method: http.MethodGet, - Path: "/daily", - HandlerFunc: statsCtx.getDaily, - }, - { - Method: http.MethodGet, - Path: "/tasks", - HandlerFunc: statsCtx.getTasks, - }, - } -} - -type statsContext struct { - statsSvc interfaces.StatsService - defaultQuery bson.M -} - -func (svc *statsContext) getOverview(c *gin.Context) { - data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getDaily(c *gin.Context) { - data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func (svc *statsContext) getTasks(c *gin.Context) { - data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccessWithData(c, data) -} - -func newStatsContext() *statsContext { - // context - ctx := &statsContext{ - defaultQuery: bson.M{ - "create_ts": bson.M{ - "$gte": time.Now().Add(-30 * 24 * time.Hour), - }, - }, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - statsSvc interfaces.StatsService, - ) { - ctx.statsSvc = statsSvc - }); err != nil { - panic(err) - } - - return ctx -} diff --git a/core/controllers/sync.go b/core/controllers/sync.go deleted file mode 100644 index 674bff76..00000000 --- a/core/controllers/sync.go +++ /dev/null @@ -1,57 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/utils" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" - "path/filepath" -) - -var SyncController ActionController - -func getSyncActions() []Action { - var ctx = newSyncContext() - return []Action{ - { - Method: http.MethodGet, - Path: "/:id/scan", - HandlerFunc: ctx.scan, - }, - { - Method: http.MethodGet, - Path: "/:id/download", - HandlerFunc: ctx.download, - }, - } -} - -type syncContext struct { -} - -func (ctx *syncContext) scan(c *gin.Context) { - id := c.Param("id") - dir := ctx._getDir(id) - files, err := utils.ScanDirectory(dir) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - c.AbortWithStatusJSON(http.StatusOK, files) -} - -func (ctx *syncContext) download(c *gin.Context) { - id := c.Param("id") - filePath := c.Query("path") - dir := ctx._getDir(id) - c.File(filepath.Join(dir, filePath)) -} - -func (ctx *syncContext) _getDir(id string) string { - workspacePath := viper.GetString("workspace") - return filepath.Join(workspacePath, id) -} - -func newSyncContext() syncContext { - return syncContext{} -} diff --git a/core/controllers/sync_v2.go b/core/controllers/sync_v2.go new file mode 100644 index 00000000..5e44fcd4 --- /dev/null +++ b/core/controllers/sync_v2.go @@ -0,0 +1,31 @@ +package controllers + +import ( + "github.com/crawlab-team/crawlab/core/utils" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + "net/http" + "path/filepath" +) + +func GetSyncScan(c *gin.Context) { + id := c.Param("id") + path := c.Query("path") + + workspacePath := viper.GetString("workspace") + dirPath := filepath.Join(workspacePath, id, path) + files, err := utils.ScanDirectory(dirPath) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + c.AbortWithStatusJSON(http.StatusOK, files) +} + +func GetSyncDownload(c *gin.Context) { + id := c.Param("id") + path := c.Query("path") + workspacePath := viper.GetString("workspace") + filePath := filepath.Join(workspacePath, id, path) + c.File(filePath) +} diff --git a/core/controllers/system_info.go b/core/controllers/system_info.go deleted file mode 100644 index a89edc15..00000000 --- a/core/controllers/system_info.go +++ /dev/null @@ -1,28 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/gin-gonic/gin" - "github.com/spf13/viper" - "net/http" -) - -func getSystemInfo(c *gin.Context) { - info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), - } - HandleSuccessWithData(c, info) -} - -func getSystemInfoActions() []Action { - return []Action{ - { - Path: "", - Method: http.MethodGet, - HandlerFunc: getSystemInfo, - }, - } -} - -var SystemInfoController ActionController diff --git a/core/controllers/system_info_v2.go b/core/controllers/system_info_v2.go index fc01ea6f..9b4fad83 100644 --- a/core/controllers/system_info_v2.go +++ b/core/controllers/system_info_v2.go @@ -8,8 +8,8 @@ import ( func GetSystemInfo(c *gin.Context) { info := &entity.SystemInfo{ - Edition: viper.GetString("info.edition"), - Version: viper.GetString("info.version"), + Edition: viper.GetString("edition"), + Version: viper.GetString("version"), } HandleSuccessWithData(c, info) } diff --git a/core/controllers/tag.go b/core/controllers/tag.go deleted file mode 100644 index 3f74abf2..00000000 --- a/core/controllers/tag.go +++ /dev/null @@ -1,3 +0,0 @@ -package controllers - -var TagController ListController diff --git a/core/controllers/task.go b/core/controllers/task.go deleted file mode 100644 index 9a66b9ef..00000000 --- a/core/controllers/task.go +++ /dev/null @@ -1,534 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/generic" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "net/http" - "strings" -) - -var TaskController *taskController - -func getTaskActions() []Action { - taskCtx := newTaskContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/run", - HandlerFunc: taskCtx.run, - }, - { - Method: http.MethodPost, - Path: "/:id/restart", - HandlerFunc: taskCtx.restart, - }, - { - Method: http.MethodPost, - Path: "/:id/cancel", - HandlerFunc: taskCtx.cancel, - }, - { - Method: http.MethodGet, - Path: "/:id/logs", - HandlerFunc: taskCtx.getLogs, - }, - { - Method: http.MethodGet, - Path: "/:id/data", - HandlerFunc: taskCtx.getData, - }, - } -} - -type taskController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *taskContext -} - -func (ctr *taskController) Get(c *gin.Context) { - ctr.ctx.getWithStatsSpider(c) -} - -func (ctr *taskController) Delete(c *gin.Context) { - if err := ctr.ctx._delete(c); err != nil { - return - } - HandleSuccess(c) -} - -func (ctr *taskController) GetList(c *gin.Context) { - withStats := c.Query("stats") - if withStats == "" { - ctr.d.GetList(c) - return - } - ctr.ctx.getListWithStats(c) -} - -func (ctr *taskController) DeleteList(c *gin.Context) { - if err := ctr.ctx._deleteList(c); err != nil { - return - } - HandleSuccess(c) -} - -type taskContext struct { - modelSvc service.ModelService - modelTaskSvc interfaces.ModelBaseService - modelTaskStatSvc interfaces.ModelBaseService - adminSvc interfaces.SpiderAdminService - schedulerSvc interfaces.TaskSchedulerService - l log.Driver -} - -func (ctx *taskContext) run(c *gin.Context) { - // task - var t models.Task - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // validate spider id - if t.GetSpiderId().IsZero() { - HandleErrorBadRequest(c, errors.ErrorTaskEmptySpiderId) - return - } - - // spider - s, err := ctx.modelSvc.GetSpiderById(t.GetSpiderId()) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(s.GetId(), opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) restart(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // options - opts := &interfaces.SpiderRunOptions{ - Mode: t.Mode, - NodeIds: t.NodeIds, - Cmd: t.Cmd, - Param: t.Param, - Priority: t.Priority, - } - - // user - if u := GetUserFromContext(c); u != nil { - opts.UserId = u.GetId() - } - - // run - taskIds, err := ctx.adminSvc.Schedule(t.SpiderId, opts) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, taskIds) -} - -func (ctx *taskContext) cancel(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // validate - if !utils.IsCancellable(t.Status) { - HandleErrorInternalServerError(c, errors.ErrorControllerNotCancellable) - return - } - - // cancel - if err := ctx.schedulerSvc.Cancel(id, GetUserFromContext(c)); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccess(c) -} - -func (ctx *taskContext) getLogs(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // logs - logs, err := ctx.l.Find(id.Hex(), "", (p.Page-1)*p.Size, p.Size) - if err != nil { - if strings.HasSuffix(err.Error(), "Status:404 Not Found") { - HandleSuccess(c) - return - } - HandleErrorInternalServerError(c, err) - return - } - total, err := ctx.l.Count(id.Hex(), "") - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, logs, total) -} - -func (ctx *taskContext) getListWithStats(c *gin.Context) { - // params - pagination := MustGetPagination(c) - query := MustGetFilterQuery(c) - sort := MustGetSortOption(c) - - // get list - list, err := ctx.modelTaskSvc.GetList(query, &mongo.FindOptions{ - Sort: sort, - Skip: pagination.Size * (pagination.Page - 1), - Limit: pagination.Size, - }) - if err != nil { - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - } else { - HandleErrorInternalServerError(c, err) - } - return - } - - // check empty list - if len(list.GetModels()) == 0 { - HandleSuccessWithListData(c, nil, 0) - return - } - - // ids - var ids []primitive.ObjectID - for _, d := range list.GetModels() { - t := d.(interfaces.Model) - ids = append(ids, t.GetId()) - } - - // total count - total, err := ctx.modelTaskSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // stat list - query = bson.M{ - "_id": bson.M{ - "$in": ids, - }, - } - stats, err := ctx.modelSvc.GetTaskStatList(query, nil) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // cache stat list to dict - dict := map[primitive.ObjectID]models.TaskStat{} - for _, s := range stats { - dict[s.GetId()] = s - } - - // iterate list again - var data []interface{} - for _, d := range list.GetModels() { - t := d.(*models.Task) - s, ok := dict[t.GetId()] - if ok { - t.Stat = &s - } - data = append(data, *t) - } - - // response - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) getWithStatsSpider(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err == mongo2.ErrNoDocuments { - HandleErrorNotFound(c, err) - return - } - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // spider - t.Spider, _ = ctx.modelSvc.GetSpiderById(t.SpiderId) - - // skip if task status is pending - if t.Status == constants.TaskStatusPending { - HandleSuccessWithData(c, t) - return - } - - // task stat - t.Stat, _ = ctx.modelSvc.GetTaskStatById(id) - - HandleSuccessWithData(c, t) -} - -func (ctx *taskContext) getData(c *gin.Context) { - // id - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // pagination - p, err := GetPagination(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - // task - t, err := ctx.modelSvc.GetTaskById(id) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // result service - resultSvc, err := result.GetResultService(t.SpiderId) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // query - query := generic.ListQuery{ - generic.ListQueryCondition{ - Key: constants.TaskKey, - Op: generic.OpEqual, - Value: t.Id, - }, - } - - // list - data, err := resultSvc.List(query, &generic.ListOptions{ - Skip: (p.Page - 1) * p.Size, - Limit: p.Size, - Sort: []generic.ListSort{{"_id", generic.SortDirectionDesc}}, - }) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // total - total, err := resultSvc.Count(query) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithListData(c, data, total) -} - -func (ctx *taskContext) _delete(c *gin.Context) (err error) { - id := c.Param("id") - oid, err := primitive.ObjectIDFromHex(id) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) (err error) { - // delete task - task, err := ctx.modelSvc.GetTaskById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(task, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - // delete task stat - taskStat, err := ctx.modelSvc.GetTaskStatById(oid) - if err != nil { - return err - } - if err := delegate2.NewModelDelegate(taskStat, GetUserFromContext(c)).Delete(); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func (ctx *taskContext) _deleteList(c *gin.Context) (err error) { - payload, err := NewJsonBinder(ControllerIdTask).BindBatchRequestPayload(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - - if err := mongo.RunTransaction(func(context mongo2.SessionContext) error { - // delete tasks - if err := ctx.modelTaskSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - // delete task stats - if err := ctx.modelTaskStatSvc.DeleteList(bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - }); err != nil { - return err - } - - return nil - }); err != nil { - HandleErrorInternalServerError(c, err) - return err - } - - return nil -} - -func newTaskContext() *taskContext { - // context - ctx := &taskContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - adminSvc interfaces.SpiderAdminService, - schedulerSvc interfaces.TaskSchedulerService, - ) { - ctx.modelSvc = modelSvc - ctx.adminSvc = adminSvc - ctx.schedulerSvc = schedulerSvc - }); err != nil { - panic(err) - } - - // model task service - ctx.modelTaskSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTask) - - // model task stat service - ctx.modelTaskStatSvc = ctx.modelSvc.GetBaseService(interfaces.ModelIdTaskStat) - - // log driver - l, err := log.GetLogDriver(log.DriverTypeFile) - if err != nil { - panic(err) - } - ctx.l = l - - return ctx -} - -func newTaskController() *taskController { - actions := getTaskActions() - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - d := NewListPostActionControllerDelegate(ControllerIdTask, modelSvc.GetBaseService(interfaces.ModelIdTask), actions) - ctx := newTaskContext() - - return &taskController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/task_v2.go b/core/controllers/task_v2.go index 95c58de6..4334d008 100644 --- a/core/controllers/task_v2.go +++ b/core/controllers/task_v2.go @@ -5,7 +5,7 @@ import ( log2 "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/spider/admin" @@ -71,8 +71,8 @@ func GetTaskList(c *gin.Context) { query := MustGetFilterQuery(c) sort := MustGetSortOption(c) - // get list - list, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ + // get tasks + tasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(query, &mongo.FindOptions{ Sort: sort, Skip: pagination.Size * (pagination.Page - 1), Limit: pagination.Size, @@ -87,15 +87,17 @@ func GetTaskList(c *gin.Context) { } // check empty list - if len(list) == 0 { + if len(tasks) == 0 { HandleSuccessWithListData(c, nil, 0) return } // ids - var ids []primitive.ObjectID - for _, t := range list { - ids = append(ids, t.Id) + var taskIds []primitive.ObjectID + var spiderIds []primitive.ObjectID + for _, t := range tasks { + taskIds = append(taskIds, t.Id) + spiderIds = append(spiderIds, t.SpiderId) } // total count @@ -106,33 +108,56 @@ func GetTaskList(c *gin.Context) { } // stat list - query = bson.M{ + stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ "_id": bson.M{ - "$in": ids, + "$in": taskIds, }, - } - stats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(query, nil) + }, nil) if err != nil { HandleErrorInternalServerError(c, err) return } // cache stat list to dict - dict := map[primitive.ObjectID]models.TaskStatV2{} + statsDict := map[primitive.ObjectID]models.TaskStatV2{} for _, s := range stats { - dict[s.Id] = s + statsDict[s.Id] = s + } + + // spider list + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + }, nil) + if err != nil { + HandleErrorInternalServerError(c, err) + return + } + + // cache spider list to dict + spiderDict := map[primitive.ObjectID]models.SpiderV2{} + for _, s := range spiders { + spiderDict[s.Id] = s } // iterate list again - for i, t := range list { - ts, ok := dict[t.Id] + for i, t := range tasks { + // task stat + ts, ok := statsDict[t.Id] if ok { - list[i].Stat = &ts + tasks[i].Stat = &ts + } + + // spider + s, ok := spiderDict[t.SpiderId] + if ok { + tasks[i].Spider = &s } } // response - HandleSuccessWithListData(c, list, total) + HandleSuccessWithListData(c, tasks, total) } func DeleteTaskById(c *gin.Context) { diff --git a/core/controllers/test/base.go b/core/controllers/test/base.go deleted file mode 100644 index 1e3c2906..00000000 --- a/core/controllers/test/base.go +++ /dev/null @@ -1,104 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/routes" - "github.com/crawlab-team/crawlab/trace" - "github.com/gavv/httpexpect/v2" - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "go.uber.org/dig" - "net/http/httptest" - "testing" - "time" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -type Test struct { - // dependencies - modelSvc service.ModelService - - // internals - app *gin.Engine - svr *httptest.Server - - // test data - TestUsername string - TestPassword string - TestToken string -} - -func (t *Test) Setup(t2 *testing.T) { - //if err := controllers.InitControllers(); err != nil { - // panic(err) - //} - //t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.DropAll() - time.Sleep(200 * time.Millisecond) -} - -func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { - e = httpexpect.New(t2, t.svr.URL) - res := e.POST("/login").WithJSON(map[string]string{ - "username": t.TestUsername, - "password": t.TestPassword, - }).Expect().JSON().Object() - t.TestToken = res.Path("$.data").String().Raw() - require.NotEmpty(t2, t.TestToken) - return e -} - -func (t *Test) WithAuth(req *httpexpect.Request) *httpexpect.Request { - return req.WithHeader("Authorization", t.TestToken) -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // gin app - t.app = gin.New() - - // http test server - t.svr = httptest.NewServer(t.app) - - // init controllers - if err := controllers.InitControllers(); err != nil { - return nil, err - } - - // init routes - if err := routes.InitRoutes(t.app); err != nil { - return nil, err - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.NewService); err != nil { - return nil, trace.TraceError(err) - } - if err := c.Invoke(func(modelSvc service.ModelService) { - t.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // test data - t.TestUsername = "admin" - t.TestPassword = "admin" - - return t, nil -} diff --git a/core/controllers/test/export_test.go b/core/controllers/test/export_test.go deleted file mode 100644 index 7623feb6..00000000 --- a/core/controllers/test/export_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "net/http" - "testing" - "time" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestExportController_Csv(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_export" - col := mongo.GetMongoCol(colName) - - // insert test data to mongo collection - for i := 0; i < 10; i++ { - _, err := col.Insert(bson.M{ - "field1": i + 1, - "field2": i + 2, - "field3": i + 3, - "field4": i + 4, - }) - require.Nil(t, err) - } - - // export from mongo collection - res := T.WithAuth(e.POST("/export/csv")). - WithQuery("target", colName). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - - // export id - exportId := res.Path("$.data").String().Raw() - - // poll export with export id - for i := 0; i < 10; i++ { - res = T.WithAuth(e.GET("/export/csv/" + exportId)).Expect().Status(http.StatusOK).JSON().Object() - status := res.Path("$.data.status").String().Raw() - if status == constants.TaskStatusFinished { - break - } - time.Sleep(1 * time.Second) - } - - // download exported csv file - csvFileBody := T.WithAuth(e.GET("/export/csv/" + exportId + "/download")).Expect().Status(http.StatusOK).Body() - csvFileBody.NotEmpty() -} diff --git a/core/controllers/test/filter_test.go b/core/controllers/test/filter_test.go deleted file mode 100644 index e7579e0f..00000000 --- a/core/controllers/test/filter_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") -} - -func TestFilterController_GetColFieldOptions(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // mongo collection - colName := "test_collection_for_filter" - field1 := "field1" - field2 := "field2" - value1 := "value1" - col := mongo.GetMongoCol(colName) - n := 10 - var ids []primitive.ObjectID - var names []string - for i := 0; i < n; i++ { - _id := primitive.NewObjectID() - ids = append(ids, _id) - name := fmt.Sprintf("name_%d", i) - names = append(names, name) - _, err := col.Insert(bson.M{field1: value1, field2: i % 2, "name": name, "_id": _id}) - require.Nil(t, err) - } - - // validate filter options field 1 - res := T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field1, field1))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - res.Path("$.data").Array().Element(0).Path("$.value").Equal(value1) - - // validate filter options field 2 - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(2) - - // validate filter options with query - conditions := []entity.Condition{{field2, constants.FilterOpEqual, 0}} - conditionsJson, err := json.Marshal(conditions) - conditionsJsonStr := string(conditionsJson) - require.Nil(t, err) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s/%s/%s", colName, field2, field2))). - WithQuery(constants.FilterQueryFieldConditions, conditionsJsonStr). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(1) - - // validate filter options (basic path) - res = T.WithAuth(e.GET(fmt.Sprintf("/filters/%s", colName))). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").NotNull() - res.Path("$.data").Array().Length().Equal(n) - for i := 0; i < n; i++ { - res.Path("$.data").Array().Element(i).Object().Value("value").Equal(ids[i]) - res.Path("$.data").Array().Element(i).Object().Value("label").Equal(names[i]) - } -} diff --git a/core/controllers/test/main_test.go b/core/controllers/test/main_test.go deleted file mode 100644 index 2b9bb76b..00000000 --- a/core/controllers/test/main_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/user" - "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestMain(m *testing.M) { - // init user - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - _, err = modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) - if err != nil { - if err.Error() != mongo.ErrNoDocuments.Error() { - panic(err) - } - userSvc, err := user.GetUserService() - if err != nil { - panic(err) - } - if err := userSvc.Create(&interfaces.UserCreateOptions{ - Username: constants.DefaultAdminUsername, - Password: constants.DefaultAdminPassword, - Role: constants.RoleAdmin, - }); err != nil { - panic(err) - } - } - - if err := controllers.InitControllers(); err != nil { - panic(err) - } - - m.Run() - - T.Cleanup() -} diff --git a/core/controllers/test/project_test.go b/core/controllers/test/project_test.go deleted file mode 100644 index b0a1b2c6..00000000 --- a/core/controllers/test/project_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" - "time" -) - -func TestProjectController_Get(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - } - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - res = T.WithAuth(e.GET("/projects/" + id)).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") -} - -func TestProjectController_Put(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "old name", - Description: "old description", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // change object - p.Id = oid - p.Name = "new name" - p.Description = "new description" - - // update - T.WithAuth(e.PUT("/projects/" + id)). - WithJSON(p). - Expect().Status(http.StatusOK) - - // check - res = T.WithAuth(e.GET("/projects/" + id)).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").Equal(id) - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") -} - -func TestProjectController_Post(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - res := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data._id").NotNull() - res.Path("$.data.name").Equal("test project") - res.Path("$.data.description").Equal("this is a test project") -} - -func TestProjectController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - p := models.Project{ - Name: "test project", - Description: "this is a test project", - } - - // add - res := T.WithAuth(e.POST("/projects")). - WithJSON(p). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // get - res = T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id = res.Path("$.data._id").String().Raw() - oid, err = primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // delete - T.WithAuth(e.DELETE("/projects/" + id)). - Expect().Status(http.StatusOK). - JSON().Object() - - // get - T.WithAuth(e.GET("/projects/" + id)). - Expect().Status(http.StatusNotFound) -} - -func TestProjectController_GetList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 100 // total - bn := 10 // batch - - for i := 0; i < n; i++ { - p := models.Project{ - Name: fmt.Sprintf("test name %d", i+1), - } - obj := T.WithAuth(e.POST("/projects")).WithJSON(p).Expect().Status(http.StatusOK).JSON().Object() - obj.Path("$.data._id").NotNull() - } - - f := entity.Filter{ - //IsOr: false, - Conditions: []*entity.Condition{ - {Key: "name", Op: constants.FilterOpContains, Value: "test name"}, - }, - } - condBytes, err := json.Marshal(&f.Conditions) - require.Nil(t, err) - - pagination := entity.Pagination{ - Page: 1, - Size: bn, - } - - // get list with pagination - res := T.WithAuth(e.GET("/projects")). - WithQuery("conditions", string(condBytes)). - WithQueryObject(pagination). - Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data").Array().Length().Equal(bn) - res.Path("$.total").Number().Equal(n) - - data := res.Path("$.data").Array() - for i := 0; i < bn; i++ { - obj := data.Element(i) - obj.Path("$.name").Equal(fmt.Sprintf("test name %d", i+1)) - } - -} - -func TestProjectController_PostList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK) - - res := T.WithAuth(e.GET("/projects")). - WithQueryObject(entity.Pagination{Page: 1, Size: 10}). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data").Array().Length().Equal(n) -} - -func TestProjectController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: fmt.Sprintf("project %d", i+1), - Description: "this is a project", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // delete - payload := entity.BatchRequestPayload{ - Ids: ids, - } - T.WithAuth(e.DELETE("/projects")). - WithJSON(payload). - Expect().Status(http.StatusOK) - - // check - for _, id := range ids { - T.WithAuth(e.GET("/projects/" + id.Hex())). - Expect().Status(http.StatusNotFound) - } - -} - -func TestProjectController_PutList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - // now - now := time.Now() - - n := 10 - var docs []models.Project - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Name: "old name", - Description: "old description", - }) - } - - // add - res := T.WithAuth(e.POST("/projects/batch")).WithJSON(docs).Expect().Status(http.StatusOK). - JSON().Object() - var ids []primitive.ObjectID - data := res.Path("$.data").Array() - for i := 0; i < n; i++ { - obj := data.Element(i) - id := obj.Path("$._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - ids = append(ids, oid) - } - - // wait for 100 millisecond - time.Sleep(100 * time.Millisecond) - - // update - p := models.Project{ - Name: "new name", - Description: "new description", - } - dataBytes, err := json.Marshal(&p) - require.Nil(t, err) - payload := entity.BatchRequestPayloadWithStringData{ - Ids: ids, - Data: string(dataBytes), - Fields: []string{ - "name", - "description", - }, - } - T.WithAuth(e.PUT("/projects")).WithJSON(payload).Expect().Status(http.StatusOK) - - // check response data - for i := 0; i < n; i++ { - res = T.WithAuth(e.GET("/projects/" + ids[i].Hex())).Expect().Status(http.StatusOK).JSON().Object() - res.Path("$.data.name").Equal("new name") - res.Path("$.data.description").Equal("new description") - } - - // check artifacts - pl, err := T.modelSvc.GetProjectList(bson.M{"_id": bson.M{"$in": ids}}, nil) - require.Nil(t, err) - for _, p := range pl { - a, err := delegate.NewModelDelegate(&p).GetArtifact() - require.Nil(t, err) - require.True(t, a.GetSys().GetUpdateTs().After(now)) - require.True(t, a.GetSys().GetUpdateTs().After(a.GetSys().GetCreateTs())) - } -} diff --git a/core/controllers/test/spider_test.go b/core/controllers/test/spider_test.go deleted file mode 100644 index b8c4dc06..00000000 --- a/core/controllers/test/spider_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestSpiderController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - s := models.Spider{ - Name: "test spider", - Description: "this is a test spider", - ColName: "test col name", - } - - // add spider - res := T.WithAuth(e.POST("/spiders")). - WithJSON(s). - Expect().Status(http.StatusOK). - JSON().Object() - res.Path("$.data._id").NotNull() - id := res.Path("$.data._id").String().Raw() - oid, err := primitive.ObjectIDFromHex(id) - require.Nil(t, err) - require.False(t, oid.IsZero()) - - // add tasks - var taskIds []primitive.ObjectID - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - { - Id: primitive.NewObjectID(), - SpiderId: oid, - }, - } - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete - T.WithAuth(e.DELETE("/spiders/" + id)). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/spiders/" + id)). - Expect().Status(http.StatusNotFound) - - // get tasks - for _, task := range tasks { - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": oid, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestSpiderController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - spiders := []models.Spider{ - { - Id: primitive.NewObjectID(), - Name: "test spider 1", - Description: "this is a test spider 1", - ColName: "test col name 1", - }, - { - Id: primitive.NewObjectID(), - Name: "test spider 2", - Description: "this is a test spider 2", - ColName: "test col name 2", - }, - } - - // add spiders - for _, spider := range spiders { - T.WithAuth(e.POST("/spiders")). - WithJSON(spider). - Expect().Status(http.StatusOK) - } - - var spiderIds []primitive.ObjectID - var taskIds []primitive.ObjectID - for _, spider := range spiders { - // task id - taskId := primitive.NewObjectID() - - // add task - err := delegate.NewModelDelegate(&models.Task{ - Id: taskId, - SpiderId: spider.Id, - }).Add() - require.Nil(t, err) - - // add task stats - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: taskId, - }).Add() - require.Nil(t, err) - - spiderIds = append(spiderIds, spider.Id) - taskIds = append(taskIds, taskId) - } - - // delete spiders - T.WithAuth(e.DELETE("/spiders")). - WithJSON(entity.BatchRequestPayload{ - Ids: spiderIds, - }).Expect().Status(http.StatusOK) - - // get spiders - for _, spider := range spiders { - // get - T.WithAuth(e.GET("/spiders/" + spider.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // get tasks - for _, taskId := range taskIds { - T.WithAuth(e.GET("/tasks/" + taskId.Hex())). - Expect().Status(http.StatusNotFound) - } - - // spider stat - modelSpiderStatSvc := service.NewBaseService(interfaces.ModelIdSpiderStat) - spiderStatCount, err := modelSpiderStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - }) - require.Nil(t, err) - require.Zero(t, spiderStatCount) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/test/task_test.go b/core/controllers/test/task_test.go deleted file mode 100644 index 4d81738b..00000000 --- a/core/controllers/test/task_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" - "testing" -) - -func TestTaskController_Delete(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - task := models.Task{ - Id: primitive.NewObjectID(), - } - - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - // delete - T.WithAuth(e.DELETE("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusOK) - - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": task.Id, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} - -func TestTaskController_DeleteList(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - - tasks := []models.Task{ - { - Id: primitive.NewObjectID(), - }, - { - Id: primitive.NewObjectID(), - }, - } - - // add spiders - var taskIds []primitive.ObjectID - for _, task := range tasks { - // add task - err := delegate.NewModelDelegate(&task).Add() - require.Nil(t, err) - - // add task stat - err = delegate.NewModelDelegate(&models.TaskStat{ - Id: task.Id, - }).Add() - require.Nil(t, err) - - taskIds = append(taskIds, task.Id) - } - - // delete tasks - T.WithAuth(e.DELETE("/tasks")). - WithJSON(entity.BatchRequestPayload{ - Ids: taskIds, - }).Expect().Status(http.StatusOK) - - // get tasks - for _, task := range tasks { - // get - T.WithAuth(e.GET("/tasks/" + task.Id.Hex())). - Expect().Status(http.StatusNotFound) - } - - // task stats - modelTaskStatSvc := service.NewBaseService(interfaces.ModelIdTaskStat) - taskStatCount, err := modelTaskStatSvc.Count(bson.M{ - "_id": bson.M{ - "$in": taskIds, - }, - }) - require.Nil(t, err) - require.Zero(t, taskStatCount) -} diff --git a/core/controllers/token.go b/core/controllers/token.go deleted file mode 100644 index f94f3b68..00000000 --- a/core/controllers/token.go +++ /dev/null @@ -1,84 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/gin-gonic/gin" -) - -var TokenController *tokenController - -var TokenActions []Action - -type tokenController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *tokenContext -} - -func (ctr *tokenController) Post(c *gin.Context) { - var err error - var t models.Token - if err := c.ShouldBindJSON(&t); err != nil { - HandleErrorBadRequest(c, err) - return - } - u, err := ctr.ctx.userSvc.GetCurrentUser(c) - if err != nil { - HandleErrorUnauthorized(c, err) - return - } - t.Token, err = ctr.ctx.userSvc.MakeToken(u) - if err != nil { - HandleErrorInternalServerError(c, err) - return - } - if err := delegate.NewModelDelegate(&t, GetUserFromContext(c)).Add(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -type tokenContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func newTokenContext() *tokenContext { - // context - ctx := &tokenContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newTokenController() *tokenController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - d := NewListPostActionControllerDelegate(ControllerIdToken, modelSvc.GetBaseService(interfaces.ModelIdToken), TokenActions) - ctx := newTokenContext() - - return &tokenController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/token_v2.go b/core/controllers/token_v2.go index bac880b2..b66f873b 100644 --- a/core/controllers/token_v2.go +++ b/core/controllers/token_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/user" "github.com/gin-gonic/gin" diff --git a/core/controllers/user.go b/core/controllers/user.go deleted file mode 100644 index 5c22a729..00000000 --- a/core/controllers/user.go +++ /dev/null @@ -1,244 +0,0 @@ -package controllers - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - delegate2 "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "github.com/gin-gonic/gin" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http" -) - -var UserController *userController - -func getUserActions() []Action { - userCtx := newUserContext() - return []Action{ - { - Method: http.MethodPost, - Path: "/:id/change-password", - HandlerFunc: userCtx.changePassword, - }, - { - Method: http.MethodGet, - Path: "/me", - HandlerFunc: userCtx.getMe, - }, - { - Method: http.MethodPut, - Path: "/me", - HandlerFunc: userCtx.putMe, - }, - } -} - -type userController struct { - ListActionControllerDelegate - d ListActionControllerDelegate - ctx *userContext -} - -func (ctr *userController) Post(c *gin.Context) { - var u models.User - if err := c.ShouldBindJSON(&u); err != nil { - HandleErrorBadRequest(c, err) - return - } - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctr *userController) PostList(c *gin.Context) { - // users - var users []models.User - if err := c.ShouldBindJSON(&users); err != nil { - HandleErrorBadRequest(c, err) - return - } - - for _, u := range users { - if err := ctr.ctx.userSvc.Create(&interfaces.UserCreateOptions{ - Username: u.Username, - Password: u.Password, - Email: u.Email, - Role: u.Role, - }); err != nil { - trace.PrintError(err) - } - } - - HandleSuccess(c) -} - -func (ctr *userController) PutList(c *gin.Context) { - // payload - var payload entity.BatchRequestPayloadWithStringData - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // doc to update - var doc models.User - if err := json.Unmarshal([]byte(payload.Data), &doc); err != nil { - HandleErrorBadRequest(c, err) - return - } - - // query - query := bson.M{ - "_id": bson.M{ - "$in": payload.Ids, - }, - } - - // update users - if err := ctr.ctx.modelSvc.GetBaseService(interfaces.ModelIdUser).UpdateDoc(query, &doc, payload.Fields); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - // update passwords - if utils.Contains(payload.Fields, "password") { - for _, id := range payload.Ids { - if err := ctr.ctx.userSvc.ChangePassword(id, doc.Password); err != nil { - trace.PrintError(err) - } - } - } - - HandleSuccess(c) -} - -type userContext struct { - modelSvc service.ModelService - userSvc interfaces.UserService -} - -func (ctx *userContext) changePassword(c *gin.Context) { - id, err := primitive.ObjectIDFromHex(c.Param("id")) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - var payload map[string]string - if err := c.ShouldBindJSON(&payload); err != nil { - HandleErrorBadRequest(c, err) - return - } - password, ok := payload["password"] - if !ok { - HandleErrorBadRequest(c, errors.ErrorUserMissingRequiredFields) - return - } - if len(password) < 5 { - HandleErrorBadRequest(c, errors.ErrorUserInvalidPassword) - return - } - if err := ctx.userSvc.ChangePassword(id, password); err != nil { - HandleErrorInternalServerError(c, err) - return - } - HandleSuccess(c) -} - -func (ctx *userContext) getMe(c *gin.Context) { - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - HandleSuccessWithData(c, u) -} - -func (ctx *userContext) putMe(c *gin.Context) { - // current user - u, err := ctx._getMe(c) - if err != nil { - HandleErrorUnauthorized(c, errors.ErrorUserUnauthorized) - return - } - - // payload - doc, err := NewJsonBinder(ControllerIdUser).Bind(c) - if err != nil { - HandleErrorBadRequest(c, err) - return - } - if doc.GetId() != u.GetId() { - HandleErrorBadRequest(c, errors.ErrorHttpBadRequest) - return - } - - // save to db - if err := delegate2.NewModelDelegate(doc, GetUserFromContext(c)).Save(); err != nil { - HandleErrorInternalServerError(c, err) - return - } - - HandleSuccessWithData(c, doc) -} - -func (ctx *userContext) _getMe(c *gin.Context) (u interfaces.User, err error) { - res, ok := c.Get(constants.UserContextKey) - if !ok { - return nil, trace.TraceError(errors.ErrorUserNotExistsInContext) - } - u, ok = res.(interfaces.User) - if !ok { - return nil, trace.TraceError(errors.ErrorUserInvalidType) - } - return u, nil -} - -func newUserContext() *userContext { - // context - ctx := &userContext{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - userSvc interfaces.UserService, - ) { - ctx.modelSvc = modelSvc - ctx.userSvc = userSvc - }); err != nil { - panic(err) - } - - return ctx -} - -func newUserController() *userController { - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - - ctr := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - d := NewListPostActionControllerDelegate(ControllerIdUser, modelSvc.GetBaseService(interfaces.ModelIdUser), getUserActions()) - ctx := newUserContext() - - return &userController{ - ListActionControllerDelegate: *ctr, - d: *d, - ctx: ctx, - } -} diff --git a/core/controllers/user_v2.go b/core/controllers/user_v2.go index 434b2061..152dcb35 100644 --- a/core/controllers/user_v2.go +++ b/core/controllers/user_v2.go @@ -1,7 +1,7 @@ package controllers import ( - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" "github.com/gin-gonic/gin" diff --git a/core/controllers/user_v2_test.go b/core/controllers/user_v2_test.go index c86739fc..88901537 100644 --- a/core/controllers/user_v2_test.go +++ b/core/controllers/user_v2_test.go @@ -3,7 +3,7 @@ package controllers_test import ( "github.com/crawlab-team/crawlab/core/controllers" "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" diff --git a/core/controllers/utils_context.go b/core/controllers/utils_context.go index 9365bd0d..00e2cc9c 100644 --- a/core/controllers/utils_context.go +++ b/core/controllers/utils_context.go @@ -3,7 +3,7 @@ package controllers import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/gin-gonic/gin" ) diff --git a/core/controllers/utils_http.go b/core/controllers/utils_http.go index 4726f288..a26f4ea1 100644 --- a/core/controllers/utils_http.go +++ b/core/controllers/utils_http.go @@ -31,6 +31,10 @@ func HandleErrorBadRequest(c *gin.Context, err error) { HandleError(http.StatusBadRequest, c, err) } +func HandleErrorForbidden(c *gin.Context, err error) { + HandleError(http.StatusForbidden, c, err) +} + func HandleErrorUnauthorized(c *gin.Context, err error) { HandleError(http.StatusUnauthorized, c, err) } diff --git a/core/controllers/version.go b/core/controllers/version.go deleted file mode 100644 index 83e3fe95..00000000 --- a/core/controllers/version.go +++ /dev/null @@ -1,23 +0,0 @@ -package controllers - -import ( - "github.com/crawlab-team/crawlab/core/config" - "github.com/gin-gonic/gin" - "net/http" -) - -func GetVersion(c *gin.Context) { - HandleSuccessWithData(c, config.GetVersion()) -} - -func getVersionActions() []Action { - return []Action{ - { - Method: http.MethodGet, - Path: "", - HandlerFunc: GetVersion, - }, - } -} - -var VersionController ActionController diff --git a/core/controllers/ws_writer.go b/core/controllers/ws_writer.go new file mode 100644 index 00000000..798b162c --- /dev/null +++ b/core/controllers/ws_writer.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/trace" + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "io" + http2 "net/http" +) + +type WsWriter struct { + io.Writer + io.Closer + conn *websocket.Conn +} + +func (w *WsWriter) Write(data []byte) (n int, err error) { + log.Infof("websocket write: %s", string(data)) + err = w.conn.WriteMessage(websocket.TextMessage, data) + if err != nil { + return 0, err + } + return len(data), nil +} + +func (w *WsWriter) Close() (err error) { + return w.conn.Close() +} + +func (w *WsWriter) CloseWithText(text string) { + _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, text)) +} + +func (w *WsWriter) CloseWithError(err error) { + _ = w.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error())) +} + +func NewWsWriter(c *gin.Context) (writer *WsWriter, err error) { + upgrader := websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http2.Request) bool { + return true + }, + } + + conn, err := upgrader.Upgrade(c.Writer, c.Request, nil) + if err != nil { + log.Errorf("websocket open connection error: %v", err) + trace.PrintError(err) + } + + return &WsWriter{ + conn: conn, + }, nil +} diff --git a/core/database/entity/database.go b/core/database/entity/database.go new file mode 100644 index 00000000..9c0d7569 --- /dev/null +++ b/core/database/entity/database.go @@ -0,0 +1,61 @@ +package entity + +type DatabaseMetadata struct { + Databases []Database `json:"databases"` +} + +type Database struct { + Name string `json:"name"` + Tables []DatabaseTable `json:"tables"` +} + +type DatabaseTable struct { + Name string `json:"name"` + Columns []DatabaseColumn `json:"columns"` + Indexes []DatabaseIndex `json:"indexes"` +} + +type DatabaseColumn struct { + Name string `json:"name"` + Type string `json:"type"` + Primary bool `json:"primary,omitempty"` + NotNull bool `json:"not_null,omitempty"` + Key string `json:"key,omitempty"` + Default string `json:"default,omitempty"` + Extra string `json:"extra,omitempty"` + AutoIncrement bool `json:"auto_increment,omitempty"` + Children []DatabaseColumn `json:"children,omitempty"` + Hash string `json:"hash,omitempty"` + OriginalName string `json:"original_name,omitempty"` + Status string `json:"status,omitempty"` +} + +type DatabaseIndex struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Columns []DatabaseIndexColumn `json:"columns"` + Unique bool `json:"unique"` + Hash string `json:"hash,omitempty"` + OriginalName string `json:"original_name,omitempty"` + Status string `json:"status,omitempty"` +} + +type DatabaseIndexColumn struct { + Name string `json:"name"` + Order int `json:"order"` +} + +func (col *DatabaseIndexColumn) OrderString() string { + if col.Order < 0 { + return "DESC" + } else { + return "ASC" + } +} + +type DatabaseQueryResults struct { + Columns []DatabaseColumn `json:"columns,omitempty"` + Rows []map[string]interface{} `json:"rows,omitempty"` + Output string `json:"output,omitempty"` + Error string `json:"error,omitempty"` +} diff --git a/core/database/interfaces/database_registry_service.go b/core/database/interfaces/database_registry_service.go new file mode 100644 index 00000000..5fc66682 --- /dev/null +++ b/core/database/interfaces/database_registry_service.go @@ -0,0 +1,11 @@ +package interfaces + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DatabaseRegistryService interface { + Start() + CheckStatus() + GetDatabaseService(id primitive.ObjectID) (res DatabaseService, err error) +} diff --git a/core/database/interfaces/database_service.go b/core/database/interfaces/database_service.go new file mode 100644 index 00000000..621829c2 --- /dev/null +++ b/core/database/interfaces/database_service.go @@ -0,0 +1,27 @@ +package interfaces + +import ( + "github.com/crawlab-team/crawlab/core/database/entity" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DatabaseService interface { + TestConnection(id primitive.ObjectID) (err error) + GetMetadata(id primitive.ObjectID) (m *entity.DatabaseMetadata, err error) + GetMetadataAllDb(id primitive.ObjectID) (m *entity.DatabaseMetadata, err error) + CreateDatabase(id primitive.ObjectID, databaseName string) (err error) + DropDatabase(id primitive.ObjectID, databaseName string) (err error) + GetTableMetadata(id primitive.ObjectID, databaseName, tableName string) (table *entity.DatabaseTable, err error) + CreateTable(id primitive.ObjectID, databaseName string, table *entity.DatabaseTable) (err error) + ModifyTable(id primitive.ObjectID, databaseName string, table *entity.DatabaseTable) (err error) + DropTable(id primitive.ObjectID, databaseName, tableName string) (err error) + RenameTable(id primitive.ObjectID, databaseName, oldTableName, newTableName string) (err error) + GetColumnTypes(query string) (types []string) + ReadRows(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}, skip, limit int) ([]map[string]interface{}, int64, error) + CreateRow(id primitive.ObjectID, databaseName, tableName string, row map[string]interface{}) error + UpdateRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}, update map[string]interface{}) error + DeleteRow(id primitive.ObjectID, databaseName, tableName string, filter map[string]interface{}) error + Query(id primitive.ObjectID, databaseName, query string) (results *entity.DatabaseQueryResults, err error) + GetCurrentMetric(id primitive.ObjectID) (m *models.DatabaseMetricV2, err error) +} diff --git a/core/database/registry_service.go b/core/database/registry_service.go new file mode 100644 index 00000000..e05b7809 --- /dev/null +++ b/core/database/registry_service.go @@ -0,0 +1,15 @@ +package database + +import ( + "github.com/crawlab-team/crawlab/core/database/interfaces" +) + +var serviceInstance interfaces.DatabaseRegistryService + +func SetDatabaseRegistryService(svc interfaces.DatabaseRegistryService) { + serviceInstance = svc +} + +func GetDatabaseRegistryService() interfaces.DatabaseRegistryService { + return serviceInstance +} diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go index 44aca263..4af04720 100644 --- a/core/ds/cockroachdb.go +++ b/core/ds/cockroachdb.go @@ -38,7 +38,7 @@ func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.Ob if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultCockroachdbPort } diff --git a/core/ds/es.go b/core/ds/es.go index e3910d19..de87ee7c 100644 --- a/core/ds/es.go +++ b/core/ds/es.go @@ -179,7 +179,7 @@ func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive. if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultElasticsearchPort } diff --git a/core/ds/kafka.go b/core/ds/kafka.go index 4931b632..0bf54940 100644 --- a/core/ds/kafka.go +++ b/core/ds/kafka.go @@ -81,7 +81,7 @@ func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultKafkaPort } diff --git a/core/ds/mongo.go b/core/ds/mongo.go index d6e69d3d..29d912fa 100644 --- a/core/ds/mongo.go +++ b/core/ds/mongo.go @@ -71,7 +71,7 @@ func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMongoPort } diff --git a/core/ds/mssql.go b/core/ds/mssql.go index f73e4be6..516f7a32 100644 --- a/core/ds/mssql.go +++ b/core/ds/mssql.go @@ -39,7 +39,7 @@ func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMssqlPort } diff --git a/core/ds/mysql.go b/core/ds/mysql.go index b9f84018..fc2f3bae 100644 --- a/core/ds/mysql.go +++ b/core/ds/mysql.go @@ -38,7 +38,7 @@ func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultMysqlPort } diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go index 174d004b..485a3a96 100644 --- a/core/ds/postgresql.go +++ b/core/ds/postgresql.go @@ -39,7 +39,7 @@ func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.Obj if svc.ds.Host == "" { svc.ds.Host = constants.DefaultHost } - if svc.ds.Port == "" { + if svc.ds.Port == 0 { svc.ds.Port = constants.DefaultPostgresqlPort } diff --git a/core/entity/dependency.go b/core/entity/dependency.go new file mode 100644 index 00000000..55f9c6ad --- /dev/null +++ b/core/entity/dependency.go @@ -0,0 +1,14 @@ +package entity + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyResult struct { + Name string `json:"name,omitempty" bson:"name,omitempty"` + NodeIds []primitive.ObjectID `json:"node_ids,omitempty" bson:"node_ids,omitempty"` + Versions []string `json:"versions,omitempty" bson:"versions,omitempty"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + Count int `json:"count,omitempty" bson:"count,omitempty"` + Upgradable bool `json:"upgradable" bson:"upgradable"` + Downgradable bool `json:"downgradable" bson:"downgradable"` + Installable bool `json:"installable" bson:"installable"` +} diff --git a/core/entity/notification_variable.go b/core/entity/notification_variable.go new file mode 100644 index 00000000..631bcc95 --- /dev/null +++ b/core/entity/notification_variable.go @@ -0,0 +1,12 @@ +package entity + +import "fmt" + +type NotificationVariable struct { + Category string `json:"category"` + Name string `json:"name"` +} + +func (v *NotificationVariable) GetKey() string { + return fmt.Sprintf("${%s:%s}", v.Category, v.Name) +} diff --git a/core/fs/service_v2.go b/core/fs/service_v2.go index 6ca1f057..918a3593 100644 --- a/core/fs/service_v2.go +++ b/core/fs/service_v2.go @@ -4,6 +4,8 @@ import ( "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/trace" + "github.com/google/uuid" "io" "os" "path/filepath" @@ -159,6 +161,15 @@ func (svc *ServiceV2) Copy(path, newPath string) (err error) { } } +func (svc *ServiceV2) Export() (resultPath string, err error) { + zipFilePath := filepath.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(svc.rootPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + func NewFsServiceV2(path string) (svc interfaces.FsServiceV2) { return &ServiceV2{ rootPath: path, diff --git a/core/fs/service_v2_test.go b/core/fs/service_v2_test.go index 09191305..f3006f5d 100644 --- a/core/fs/service_v2_test.go +++ b/core/fs/service_v2_test.go @@ -1,7 +1,7 @@ package fs import ( - "io/ioutil" + "github.com/apex/log" "os" "path/filepath" "testing" @@ -10,20 +10,25 @@ import ( ) func TestServiceV2_List(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() testDir := filepath.Join(rootDir, "dir") - os.Mkdir(testDir, 0755) - ioutil.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644) - ioutil.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644) + _ = os.Mkdir(testDir, 0755) + _ = os.WriteFile(filepath.Join(testDir, "file1.txt"), []byte("hello world"), 0644) + _ = os.WriteFile(filepath.Join(testDir, "file2.txt"), []byte("hello again"), 0644) subDir := filepath.Join(testDir, "subdir") - os.Mkdir(subDir, 0755) - ioutil.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) - os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion + _ = os.Mkdir(subDir, 0755) + _ = os.WriteFile(filepath.Join(subDir, "file3.txt"), []byte("subdir file"), 0644) + _ = os.Mkdir(filepath.Join(testDir, "empty"), 0755) // explicitly testing empty dir inclusion svc := NewFsServiceV2(rootDir) @@ -56,14 +61,19 @@ func TestServiceV2_List(t *testing.T) { } func TestServiceV2_GetFile(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() expectedContent := []byte("hello world") - ioutil.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) + _ = os.WriteFile(filepath.Join(rootDir, "file.txt"), expectedContent, 0644) svc := NewFsServiceV2(rootDir) @@ -75,14 +85,19 @@ func TestServiceV2_GetFile(t *testing.T) { } func TestServiceV2_Delete(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() filePath := filepath.Join(rootDir, "file.txt") - ioutil.WriteFile(filePath, []byte("hello world"), 0644) + _ = os.WriteFile(filePath, []byte("hello world"), 0644) svc := NewFsServiceV2(rootDir) @@ -98,11 +113,16 @@ func TestServiceV2_Delete(t *testing.T) { } func TestServiceV2_CreateDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) @@ -118,11 +138,16 @@ func TestServiceV2_CreateDir(t *testing.T) { } func TestServiceV2_Save(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) @@ -133,22 +158,27 @@ func TestServiceV2_Save(t *testing.T) { } // Verify the file was saved - data, err := ioutil.ReadFile(filepath.Join(rootDir, "newFile.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "newFile.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } func TestServiceV2_Rename(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a file to rename - ioutil.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) + _ = os.WriteFile(filepath.Join(rootDir, "oldName.txt"), []byte("Hello, world!"), 0644) // Rename the file err = svc.Rename("oldName.txt", "newName.txt") @@ -162,16 +192,21 @@ func TestServiceV2_Rename(t *testing.T) { } func TestServiceV2_RenameDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a directory to rename - os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) + _ = os.Mkdir(filepath.Join(rootDir, "oldName"), 0755) // Rename the directory err = svc.Rename("oldName", "newName") @@ -185,16 +220,21 @@ func TestServiceV2_RenameDir(t *testing.T) { } func TestServiceV2_Copy(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a file to copy - ioutil.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) + _ = os.WriteFile(filepath.Join(rootDir, "source.txt"), []byte("Hello, world!"), 0644) // Copy the file err = svc.Copy("source.txt", "copy.txt") @@ -203,23 +243,28 @@ func TestServiceV2_Copy(t *testing.T) { } // Verify the file was copied - data, err := ioutil.ReadFile(filepath.Join(rootDir, "copy.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "copy.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } func TestServiceV2_CopyDir(t *testing.T) { - rootDir, err := ioutil.TempDir("", "fsTest") + rootDir, err := os.MkdirTemp("", "fsTest") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } - defer os.RemoveAll(rootDir) // clean up + defer func() { + err := os.RemoveAll(rootDir) // clean up + if err != nil { + log.Errorf("Failed to remove temp dir: %v", err) + } + }() svc := NewFsServiceV2(rootDir) // Create a directory to copy - os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) - ioutil.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644) + _ = os.Mkdir(filepath.Join(rootDir, "sourceDir"), 0755) + _ = os.WriteFile(filepath.Join(rootDir, "sourceDir", "file.txt"), []byte("Hello, world!"), 0644) // Copy the directory err = svc.Copy("sourceDir", "copyDir") @@ -232,7 +277,7 @@ func TestServiceV2_CopyDir(t *testing.T) { assert.NoError(t, err) // Verify the file inside the directory was copied - data, err := ioutil.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt")) + data, err := os.ReadFile(filepath.Join(rootDir, "copyDir", "file.txt")) assert.NoError(t, err) assert.Equal(t, "Hello, world!", string(data)) } diff --git a/core/go.mod b/core/go.mod index 35a77a51..5291ef40 100644 --- a/core/go.mod +++ b/core/go.mod @@ -4,50 +4,49 @@ go 1.22 replace ( github.com/crawlab-team/crawlab/db => ../db + github.com/crawlab-team/crawlab/fs => ../fs github.com/crawlab-team/crawlab/grpc => ../grpc - github.com/crawlab-team/crawlab/template-parser => ../template-parser github.com/crawlab-team/crawlab/trace => ../trace github.com/crawlab-team/crawlab/vcs => ../vcs - github.com/crawlab-team/crawlab/fs => ../fs ) require ( + github.com/PuerkitoBio/goquery v1.9.2 github.com/ReneKroon/ttlcache v1.7.0 github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/fs v0.6.3-2 - github.com/crawlab-team/crawlab/db v0.6.0-1 - github.com/crawlab-team/crawlab/grpc v0.0.0 - github.com/crawlab-team/crawlab/template-parser v0.0.4-0.20221006034646-9bb77a7ae86e - github.com/crawlab-team/crawlab/trace v0.1.0 - github.com/crawlab-team/crawlab/vcs v0.1.1 - github.com/elastic/go-elasticsearch/v8 v8.7.0 + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/crawlab-team/crawlab/db v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/fs v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/grpc v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/trace v0.0.0-20240731075841-7fe770ae9d15 + github.com/crawlab-team/crawlab/vcs v0.0.0-20240731075841-7fe770ae9d15 + github.com/elastic/go-elasticsearch/v8 v8.14.0 github.com/emirpasic/gods v1.18.1 - github.com/fsnotify/fsnotify v1.5.1 - github.com/gavv/httpexpect/v2 v2.16.0 - github.com/gin-gonic/gin v1.9.1 - github.com/go-git/go-git/v5 v5.12.0 + github.com/fsnotify/fsnotify v1.7.0 + github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 - github.com/hashicorp/go-uuid v1.0.1 - github.com/imroc/req v0.3.0 - github.com/matcornic/hermes/v2 v2.1.0 + github.com/hashicorp/go-uuid v1.0.3 + github.com/imroc/req v0.3.2 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 github.com/segmentio/kafka-go v0.4.39 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.3.0 - github.com/spf13/viper v1.10.0 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.9.1 github.com/upper/db/v4 v4.6.0 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.15.1 go.uber.org/dig v1.10.0 - google.golang.org/grpc v1.64.0 + golang.org/x/oauth2 v0.21.0 + google.golang.org/api v0.189.0 + google.golang.org/grpc v1.65.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df ) @@ -57,132 +56,116 @@ require ( ) require ( - cloud.google.com/go/compute/metadata v0.2.3 // indirect - github.com/Masterminds/semver v1.4.2 // indirect - github.com/Masterminds/sprig v2.16.0+incompatible // indirect + cloud.google.com/go/auth v0.7.2 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.0.0 // indirect - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect - github.com/ajg/form v1.5.1 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/aokoli/goutils v1.0.1 // indirect - github.com/bytedance/sonic v1.9.1 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/crawlab-team/goseaweedfs v0.6.3 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denisenkom/go-mssqldb v0.11.0 // indirect - github.com/elastic/elastic-transport-go/v8 v8.2.0 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/fatih/structs v1.1.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect - github.com/gorilla/css v1.0.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huandu/xstrings v1.2.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect - github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.11.0 // indirect + github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.2.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.10.0 // indirect - github.com/jackc/pgx/v4 v4.15.0 // indirect - github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect + github.com/jackc/pgx/v4 v4.18.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.3 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/olekukonko/tablewriter v0.0.1 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sanity-io/litter v1.5.5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/smartystreets/assertions v1.0.0 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.34.0 // indirect - github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 // indirect - github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - github.com/yudai/gojsondiff v1.0.0 // indirect - github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/ztrue/tracerr v0.4.0 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/protobuf v1.34.1 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/goleak v1.1.11 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - moul.io/http2curl/v2 v2.3.0 // indirect ) diff --git a/core/go.sum b/core/go.sum index a6e708a6..ee5b1249 100644 --- a/core/go.sum +++ b/core/go.sum @@ -3,7 +3,6 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -16,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= @@ -28,18 +26,19 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= -cloud.google.com/go v0.99.0 h1:y/cM2iqGgGi5D5DQZl6D9STN/3dR/Vx5Mp8s752oJTY= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= +cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= +cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= +cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -52,48 +51,33 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= -github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= -github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= +github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= github.com/ReneKroon/ttlcache v1.7.0 h1:8BkjFfrzVFXyrqnMtezAaJ6AHPSsVV10m6w28N/Fgkk= github.com/ReneKroon/ttlcache v1.7.0/go.mod h1:8BGGzdumrIjWxdRx8zpK6L3oGMWvIXdvB2GD1cfvd+I= -github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 h1:ZBbLwSJqkHBuFDA6DUhhse0IGJ7T5bemHyNILUjvOq4= -github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2/go.mod h1:VSw57q4QFiWDbRnjdX8Cb3Ow0SFncRw+bA/ofY6Q83w= -github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= -github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= @@ -113,19 +97,17 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= -github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= -github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -135,6 +117,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -152,26 +138,22 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crawlab-team/crawlab/fs v0.6.3-2 h1:GAovTF1R1PLoQgj+0F1GpePrlp1k7RtW/jK8p0L9HhA= -github.com/crawlab-team/crawlab/fs v0.6.3-2/go.mod h1:Nob0uFr82IPbkk6LEYG0BAB2NgJ3PKoNVhtcbf5fLf0= -github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg= -github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8= github.com/crawlab-team/goseaweedfs v0.6.3 h1:f96H2QCLrZpof9na1mhIKouKrv8p32XRUyouSVm4YHU= github.com/crawlab-team/goseaweedfs v0.6.3/go.mod h1:Anqw9QErRJpTeVAVdcSfzprGzUz7OW4MVCHLJjKeO1U= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elastic/elastic-transport-go/v8 v8.2.0 h1:hkK5IIs/15mpSXzd5THWVlWTKJyMw6cbCWM3T/B2S5E= -github.com/elastic/elastic-transport-go/v8 v8.2.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= -github.com/elastic/go-elasticsearch/v8 v8.7.0 h1:ZvbT1YHppBC0QxGnMmaDUxoDa26clwhRaB3Gp5E3UcY= -github.com/elastic/go-elasticsearch/v8 v8.7.0/go.mod h1:lVb8SvJV8McVkdswpL8YR5QKIkhlWaoSq60YpHilOLI= +github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= +github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= +github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow= +github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -190,22 +172,21 @@ github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= -github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= @@ -219,13 +200,17 @@ github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXY github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -234,13 +219,11 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -289,6 +272,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 h1:saBP362Qm7zDdDXqv61kI4rzhmLFq3Z1gx34xpl6cWE= +github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -305,8 +290,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -321,29 +304,30 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= -github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -365,8 +349,9 @@ github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR3 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -379,22 +364,12 @@ github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8= -github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= -github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= -github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= +github.com/imroc/req v0.3.2 h1:M/JkeU6RPmX+WYvT2vaaOL0K+q8ufL5LxwvJc4xeB4o= +github.com/imroc/req v0.3.2/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -409,8 +384,9 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -426,28 +402,30 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= +github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0 h1:xqgexXAGQgY3HAjNPSaCqn5Aahbo5TKsmhp8VRfr1iQ= -github.com/jaytaylor/html2text v0.0.0-20180606194806-57d518f124b0/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -466,13 +444,13 @@ github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -486,20 +464,18 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/matcornic/hermes/v2 v2.1.0 h1:9TDYFBPFv6mcXanaDmRDEp/RTWj0dTTi+LpFnnnfNWc= -github.com/matcornic/hermes/v2 v2.1.0/go.mod h1:2+ziJeoyRfaLiATIL8VZ7f9hpzH4oDHqTmn0bhrsgVI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -507,8 +483,6 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -517,11 +491,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -533,13 +504,12 @@ github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXx github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -550,35 +520,28 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= -github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -595,8 +558,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452 h1:ewTtJ72GFy2e0e8uyiDwMG3pKCS5mBh+hdSTYsPKEP8= -github.com/robertkrimen/otto v0.0.0-20210614181706-373ff5438452/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -607,13 +568,13 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= -github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= @@ -628,7 +589,6 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -645,23 +605,24 @@ github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.10.0 h1:EaGW2JJh15aKOejeuJ+wpFSHnbd7GE6Wvp3TsNhb6LY= -github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= -github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -669,7 +630,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -679,14 +639,12 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= @@ -703,19 +661,10 @@ github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcy github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.34.0 h1:d3AAQJ2DRcxJYHm7OXNXtXt2as1vMDfxeIcFvhmGGm4= -github.com/valyala/fasthttp v1.34.0/go.mod h1:epZA5N+7pY6ZaEKRmstzOuYJx9HI8DI1oaCGZpdH4h0= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= -github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04 h1:L0rPdfzq43+NV8rfIx2kA4iSSLRj2jN5ijYHoeXRwvQ= -github.com/vanng822/css v0.0.0-20190504095207-a21e860bcd04/go.mod h1:tcnB1voG49QhCrwq1W0w5hhGasvOg+VQp9i9H1rCM1w= -github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe h1:9YnI5plmy+ad6BM+JCLJb2ZV7/TNiE5l7SNKfumYKgc= -github.com/vanng822/go-premailer v0.0.0-20191214114701-be27abe028fe/go.mod h1:JTFJA/t820uFDoyPpErFQ3rb3amdZoPtxcKervG0OE4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= @@ -728,23 +677,8 @@ github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -754,14 +688,13 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww= github.com/ztrue/tracerr v0.4.0 h1:vT5PFxwIGs7rCg9ZgJ/y0NmOpJkPCPFK8x0vVIYzd04= github.com/ztrue/tracerr v0.4.0/go.mod h1:PaFfYlas0DfmXNpo7Eay4MFhZUONqvXM+T2HyGPpngk= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= -go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= -go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.mongodb.org/mongo-driver v1.15.1 h1:l+RvoUOoMXFmADTLfYDm7On9dRm7p4T80/lEQM+r7HU= +go.mongodb.org/mongo-driver v1.15.1/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -769,31 +702,47 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY= go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= -go.uber.org/goleak v0.10.0 h1:G3eWbSNIskeRqtsN/1uI5B+eP73y3JUuBsv9AZjehb4= go.uber.org/goleak v0.10.0/go.mod h1:VCZuO8V8mFPlL0F5J5GK1rtHV3DrFcQ1R8ryq7FK0aI= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029175232-7e6ffbd03851/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -805,19 +754,16 @@ golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -829,6 +775,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -858,9 +806,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -897,7 +844,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -905,16 +851,15 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -932,8 +877,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -947,8 +892,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -957,7 +902,6 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190225065934-cc5685c2db12/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -999,7 +943,6 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1007,7 +950,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1024,26 +966,25 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1057,8 +998,9 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1115,10 +1057,8 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1127,8 +1067,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1167,6 +1107,8 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= +google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= +google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1174,8 +1116,6 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1211,9 +1151,7 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1241,8 +1179,11 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCeFn7MoLy+wBDQZ1UQIJD4UNM1Qvkg= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= +google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f h1:RARaIm8pxYuxyNPbBQf5igT7XdOyCNtat1qAT2ZxjU4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1270,8 +1211,8 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1286,8 +1227,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1297,18 +1238,15 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= -gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1318,7 +1256,6 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1350,8 +1287,7 @@ modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= -moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= -moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/core/grpc/client/client_v2.go b/core/grpc/client/client_v2.go index fd40cc9c..a5f32ee0 100644 --- a/core/grpc/client/client_v2.go +++ b/core/grpc/client/client_v2.go @@ -19,6 +19,7 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" "io" + "sync" "time" ) @@ -35,36 +36,38 @@ type GrpcClientV2 struct { stream grpc2.NodeService_SubscribeClient msgCh chan *grpc2.StreamMessage err error + once sync.Once // clients NodeClient grpc2.NodeServiceClient TaskClient grpc2.TaskServiceClient ModelBaseServiceV2Client grpc2.ModelBaseServiceV2Client - DependenciesClient grpc2.DependencyServiceV2Client -} - -func (c *GrpcClientV2) Init() (err error) { - return nil + DependenciesClient grpc2.DependenciesServiceV2Client + MetricsClient grpc2.MetricsServiceV2Client } func (c *GrpcClientV2) Start() (err error) { - // connect - if err := c.connect(); err != nil { - return err - } + c.once.Do(func() { + // connect + err = c.connect() + if err != nil { + return + } - // register rpc services - c.Register() + // register rpc services + c.Register() - // subscribe - if err := c.subscribe(); err != nil { - return err - } + // subscribe + err = c.subscribe() + if err != nil { + return + } - // handle stream message - go c.handleStreamMessage() + // handle stream message + go c.handleStreamMessage() + }) - return nil + return err } func (c *GrpcClientV2) Stop() (err error) { @@ -95,12 +98,8 @@ func (c *GrpcClientV2) Register() { c.NodeClient = grpc2.NewNodeServiceClient(c.conn) c.ModelBaseServiceV2Client = grpc2.NewModelBaseServiceV2Client(c.conn) c.TaskClient = grpc2.NewTaskServiceClient(c.conn) - c.DependenciesClient = grpc2.NewDependencyServiceV2Client(c.conn) - - // log - log.Infof("[GrpcClient] grpc client registered client services") - log.Debugf("[GrpcClient] NodeClient: %v", c.NodeClient) - log.Debugf("[GrpcClient] ModelBaseServiceV2Client: %v", c.ModelBaseServiceV2Client) + c.DependenciesClient = grpc2.NewDependenciesServiceV2Client(c.conn) + c.MetricsClient = grpc2.NewMetricsServiceV2Client(c.conn) } func (c *GrpcClientV2) Context() (ctx context.Context, cancel context.CancelFunc) { @@ -248,7 +247,7 @@ func (c *GrpcClientV2) handleStreamMessage() { } } -func NewGrpcClientV2() (c *GrpcClientV2, err error) { +func newGrpcClientV2() (c *GrpcClientV2) { client := &GrpcClientV2{ address: entity.NewAddress(&entity.AddressOptions{ Host: constants.DefaultGrpcClientRemoteHost, @@ -260,28 +259,23 @@ func NewGrpcClientV2() (c *GrpcClientV2, err error) { client.nodeCfgSvc = nodeconfig.GetNodeConfigService() if viper.GetString("grpc.address") != "" { - client.address, err = entity.NewAddressFromString(viper.GetString("grpc.address")) + address, err := entity.NewAddressFromString(viper.GetString("grpc.address")) if err != nil { - return nil, trace.TraceError(err) + log.Errorf("failed to parse grpc address: %s", viper.GetString("grpc.address")) + panic(err) } + client.address = address } - if err := client.Init(); err != nil { - return nil, err - } - - return client, nil + return client } -var _clientV2 *GrpcClientV2 +var clientV2 *GrpcClientV2 +var clientV2Once sync.Once -func GetGrpcClientV2() (client *GrpcClientV2, err error) { - if _clientV2 != nil { - return _clientV2, nil - } - _clientV2, err = NewGrpcClientV2() - if err != nil { - return nil, err - } - return _clientV2, nil +func GetGrpcClientV2() *GrpcClientV2 { + clientV2Once.Do(func() { + clientV2 = newGrpcClientV2() + }) + return clientV2 } diff --git a/core/grpc/server/dependencies_server_v2.go b/core/grpc/server/dependencies_server_v2.go index aca7e008..32c39760 100644 --- a/core/grpc/server/dependencies_server_v2.go +++ b/core/grpc/server/dependencies_server_v2.go @@ -2,31 +2,179 @@ package server import ( "context" - grpc "github.com/crawlab-team/crawlab/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "errors" + "github.com/apex/log" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" + mongo2 "github.com/crawlab-team/crawlab/db/mongo" + "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "io" + "sync" + "time" ) type DependenciesServerV2 struct { - grpc.UnimplementedDependencyServiceV2Server + grpc.UnimplementedDependenciesServiceV2Server + mu *sync.Mutex + streams map[string]*grpc.DependenciesServiceV2_ConnectServer } -func (svr DependenciesServerV2) Connect(stream grpc.DependencyServiceV2_ConnectServer) (err error) { - return status.Errorf(codes.Unimplemented, "method Connect not implemented") +func (svr DependenciesServerV2) Connect(req *grpc.DependenciesServiceV2ConnectRequest, stream grpc.DependenciesServiceV2_ConnectServer) (err error) { + svr.mu.Lock() + svr.streams[req.NodeKey] = &stream + svr.mu.Unlock() + log.Info("[DependenciesServerV2] connected: " + req.NodeKey) + + // Keep this scope alive because once this scope exits - the stream is closed + for { + select { + case <-stream.Context().Done(): + log.Info("[DependenciesServerV2] disconnected: " + req.NodeKey) + return nil + } + } } func (svr DependenciesServerV2) Sync(ctx context.Context, request *grpc.DependenciesServiceV2SyncRequest) (response *grpc.Response, err error) { - return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": request.NodeKey}, nil) + if err != nil { + return nil, err + } + + depsDb, err := service.NewModelServiceV2[models2.DependencyV2]().GetMany(bson.M{ + "node_id": n.Id, + "type": request.Lang, + }, nil) + if err != nil { + if !errors.Is(err, mongo.ErrNoDocuments) { + log.Errorf("[DependenciesServiceV2] get dependencies from db error: %v", err) + return nil, err + } + } + + depsDbMap := make(map[string]*models2.DependencyV2) + for _, d := range depsDb { + depsDbMap[d.Name] = &d + } + + var depsToInsert []models2.DependencyV2 + depsMap := make(map[string]*models2.DependencyV2) + for _, dep := range request.Dependencies { + d := models2.DependencyV2{ + Name: dep.Name, + NodeId: n.Id, + Type: request.Lang, + Version: dep.Version, + } + d.SetCreatedAt(time.Now()) + + depsMap[d.Name] = &d + + _, ok := depsDbMap[d.Name] + if !ok { + depsToInsert = append(depsToInsert, d) + } + } + + var depIdsToDelete []primitive.ObjectID + for _, d := range depsDb { + _, ok := depsMap[d.Name] + if !ok { + depIdsToDelete = append(depIdsToDelete, d.Id) + } + } + + err = mongo2.RunTransaction(func(ctx mongo.SessionContext) (err error) { + if len(depIdsToDelete) > 0 { + err = service.NewModelServiceV2[models2.DependencyV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": depIdsToDelete}, + }) + if err != nil { + log.Errorf("[DependenciesServerV2] delete dependencies in db error: %v", err) + trace.PrintError(err) + return err + } + } + + if len(depsToInsert) > 0 { + _, err = service.NewModelServiceV2[models2.DependencyV2]().InsertMany(depsToInsert) + if err != nil { + log.Errorf("[DependenciesServerV2] insert dependencies in db error: %v", err) + trace.PrintError(err) + return err + } + } + + return nil + }) + + return nil, err } -func (svr DependenciesServerV2) Install(stream grpc.DependencyServiceV2_InstallServer) (err error) { - return status.Errorf(codes.Unimplemented, "method Install not implemented") +func (svr DependenciesServerV2) UpdateTaskLog(stream grpc.DependenciesServiceV2_UpdateTaskLogServer) (err error) { + var t *models2.DependencyTaskV2 + for { + req, err := stream.Recv() + if err == io.EOF { + // all messages have been received + return stream.SendAndClose(&grpc.Response{Message: "update task log finished"}) + } + if err != nil { + return err + } + taskId, err := primitive.ObjectIDFromHex(req.TaskId) + if err != nil { + return err + } + if t == nil { + t, err = service.NewModelServiceV2[models2.DependencyTaskV2]().GetById(taskId) + if err != nil { + return err + } + } + var logs []models2.DependencyLogV2 + for _, line := range req.LogLines { + l := models2.DependencyLogV2{ + TaskId: taskId, + Content: line, + } + l.SetCreated(t.CreatedBy) + logs = append(logs, l) + } + _, err = service.NewModelServiceV2[models2.DependencyLogV2]().InsertMany(logs) + if err != nil { + return err + } + } } -func (svr DependenciesServerV2) UninstallDependencies(stream grpc.DependencyServiceV2_UninstallDependenciesServer) (err error) { - return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") +func (svr DependenciesServerV2) GetStream(key string) (stream *grpc.DependenciesServiceV2_ConnectServer, err error) { + svr.mu.Lock() + defer svr.mu.Unlock() + stream, ok := svr.streams[key] + if !ok { + return nil, errors.New("stream not found") + } + return stream, nil } func NewDependenciesServerV2() *DependenciesServerV2 { - return &DependenciesServerV2{} + return &DependenciesServerV2{ + mu: new(sync.Mutex), + streams: make(map[string]*grpc.DependenciesServiceV2_ConnectServer), + } +} + +var depSvc *DependenciesServerV2 + +func GetDependenciesServerV2() *DependenciesServerV2 { + if depSvc != nil { + return depSvc + } + depSvc = NewDependenciesServerV2() + return depSvc } diff --git a/core/grpc/server/metrics_server_v2.go b/core/grpc/server/metrics_server_v2.go new file mode 100644 index 00000000..60a92569 --- /dev/null +++ b/core/grpc/server/metrics_server_v2.go @@ -0,0 +1,66 @@ +package server + +import ( + "context" + "github.com/apex/log" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/grpc" + "go.mongodb.org/mongo-driver/bson" + "sync" + "time" +) + +type MetricsServerV2 struct { + grpc.UnimplementedMetricsServiceV2Server +} + +func (svr MetricsServerV2) Send(_ context.Context, req *grpc.MetricsServiceV2SendRequest) (res *grpc.Response, err error) { + log.Info("[MetricsServerV2] received metric from node: " + req.NodeKey) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + if err != nil { + log.Errorf("[MetricsServerV2] error getting node: %v", err) + return HandleError(err) + } + metric := models2.MetricV2{ + Type: req.Type, + NodeId: n.Id, + CpuUsagePercent: req.CpuUsagePercent, + TotalMemory: req.TotalMemory, + AvailableMemory: req.AvailableMemory, + UsedMemory: req.UsedMemory, + UsedMemoryPercent: req.UsedMemoryPercent, + TotalDisk: req.TotalDisk, + AvailableDisk: req.AvailableDisk, + UsedDisk: req.UsedDisk, + UsedDiskPercent: req.UsedDiskPercent, + DiskReadBytesRate: req.DiskReadBytesRate, + DiskWriteBytesRate: req.DiskWriteBytesRate, + NetworkBytesSentRate: req.NetworkBytesSentRate, + NetworkBytesRecvRate: req.NetworkBytesRecvRate, + } + metric.CreatedAt = time.Unix(req.Timestamp, 0) + _, err = service.NewModelServiceV2[models2.MetricV2]().InsertOne(metric) + if err != nil { + log.Errorf("[MetricsServerV2] error inserting metric: %v", err) + return HandleError(err) + } + return HandleSuccess() +} + +func newMetricsServerV2() *MetricsServerV2 { + return &MetricsServerV2{} +} + +var metricsServerV2 *MetricsServerV2 +var metricsServerV2Once = &sync.Once{} + +func GetMetricsServerV2() *MetricsServerV2 { + if metricsServerV2 != nil { + return metricsServerV2 + } + metricsServerV2Once.Do(func() { + metricsServerV2 = newMetricsServerV2() + }) + return metricsServerV2 +} diff --git a/core/grpc/server/model_base_service_v2_server.go b/core/grpc/server/model_base_service_v2_server.go index 79385602..d695d1de 100644 --- a/core/grpc/server/model_base_service_v2_server.go +++ b/core/grpc/server/model_base_service_v2_server.go @@ -3,10 +3,10 @@ package server import ( "context" "encoding/json" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "reflect" @@ -16,26 +16,35 @@ var ( typeNameColNameMap = make(map[string]string) typeOneNameModelMap = make(map[string]any) typeOneInstances = []any{ - *new(models.TestModel), - *new(models.DataCollectionV2), - *new(models.DataSourceV2), - *new(models.DependencySettingV2), - *new(models.EnvironmentV2), - *new(models.GitV2), - *new(models.NodeV2), - *new(models.PermissionV2), - *new(models.ProjectV2), - *new(models.RolePermissionV2), - *new(models.RoleV2), - *new(models.ScheduleV2), - *new(models.SettingV2), - *new(models.SpiderV2), - *new(models.TaskQueueItemV2), - *new(models.TaskStatV2), - *new(models.TaskV2), - *new(models.TokenV2), - *new(models.UserRoleV2), - *new(models.UserV2), + *new(models2.TestModelV2), + *new(models2.DataCollectionV2), + *new(models2.DatabaseV2), + *new(models2.DatabaseMetricV2), + *new(models2.DependencyV2), + *new(models2.DependencyLogV2), + *new(models2.DependencySettingV2), + *new(models2.DependencyTaskV2), + *new(models2.EnvironmentV2), + *new(models2.GitV2), + *new(models2.MetricV2), + *new(models2.NodeV2), + *new(models2.NotificationChannelV2), + *new(models2.NotificationRequestV2), + *new(models2.NotificationSettingV2), + *new(models2.PermissionV2), + *new(models2.ProjectV2), + *new(models2.RolePermissionV2), + *new(models2.RoleV2), + *new(models2.ScheduleV2), + *new(models2.SettingV2), + *new(models2.SpiderV2), + *new(models2.SpiderStatV2), + *new(models2.TaskQueueItemV2), + *new(models2.TaskStatV2), + *new(models2.TaskV2), + *new(models2.TokenV2), + *new(models2.UserRoleV2), + *new(models2.UserV2), } ) @@ -57,7 +66,7 @@ type ModelBaseServiceServerV2 struct { grpc.UnimplementedModelBaseServiceV2Server } -func (svr ModelBaseServiceServerV2) GetById(ctx context.Context, req *grpc.ModelServiceV2GetByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetById(_ context.Context, req *grpc.ModelServiceV2GetByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -70,7 +79,7 @@ func (svr ModelBaseServiceServerV2) GetById(ctx context.Context, req *grpc.Model return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) GetOne(ctx context.Context, req *grpc.ModelServiceV2GetOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetOne(_ context.Context, req *grpc.ModelServiceV2GetOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -89,7 +98,7 @@ func (svr ModelBaseServiceServerV2) GetOne(ctx context.Context, req *grpc.ModelS return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) GetMany(ctx context.Context, req *grpc.ModelServiceV2GetManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) GetMany(_ context.Context, req *grpc.ModelServiceV2GetManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -108,7 +117,7 @@ func (svr ModelBaseServiceServerV2) GetMany(ctx context.Context, req *grpc.Model return HandleSuccessWithData(data) } -func (svr ModelBaseServiceServerV2) DeleteById(ctx context.Context, req *grpc.ModelServiceV2DeleteByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteById(_ context.Context, req *grpc.ModelServiceV2DeleteByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -121,7 +130,7 @@ func (svr ModelBaseServiceServerV2) DeleteById(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) DeleteOne(ctx context.Context, req *grpc.ModelServiceV2DeleteOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteOne(_ context.Context, req *grpc.ModelServiceV2DeleteOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -135,7 +144,7 @@ func (svr ModelBaseServiceServerV2) DeleteOne(ctx context.Context, req *grpc.Mod return HandleSuccess() } -func (svr ModelBaseServiceServerV2) DeleteMany(ctx context.Context, req *grpc.ModelServiceV2DeleteManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) DeleteMany(_ context.Context, req *grpc.ModelServiceV2DeleteManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -149,7 +158,7 @@ func (svr ModelBaseServiceServerV2) DeleteMany(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateById(ctx context.Context, req *grpc.ModelServiceV2UpdateByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateById(_ context.Context, req *grpc.ModelServiceV2UpdateByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -167,7 +176,7 @@ func (svr ModelBaseServiceServerV2) UpdateById(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateOne(ctx context.Context, req *grpc.ModelServiceV2UpdateOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateOne(_ context.Context, req *grpc.ModelServiceV2UpdateOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -186,7 +195,7 @@ func (svr ModelBaseServiceServerV2) UpdateOne(ctx context.Context, req *grpc.Mod return HandleSuccess() } -func (svr ModelBaseServiceServerV2) UpdateMany(ctx context.Context, req *grpc.ModelServiceV2UpdateManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) UpdateMany(_ context.Context, req *grpc.ModelServiceV2UpdateManyRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -205,7 +214,7 @@ func (svr ModelBaseServiceServerV2) UpdateMany(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) ReplaceById(ctx context.Context, req *grpc.ModelServiceV2ReplaceByIdRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) ReplaceById(_ context.Context, req *grpc.ModelServiceV2ReplaceByIdRequest) (res *grpc.Response, err error) { id, err := primitive.ObjectIDFromHex(req.Id) if err != nil { return HandleError(err) @@ -225,7 +234,7 @@ func (svr ModelBaseServiceServerV2) ReplaceById(ctx context.Context, req *grpc.M return HandleSuccess() } -func (svr ModelBaseServiceServerV2) ReplaceOne(ctx context.Context, req *grpc.ModelServiceV2ReplaceOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) ReplaceOne(_ context.Context, req *grpc.ModelServiceV2ReplaceOneRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { @@ -246,7 +255,7 @@ func (svr ModelBaseServiceServerV2) ReplaceOne(ctx context.Context, req *grpc.Mo return HandleSuccess() } -func (svr ModelBaseServiceServerV2) InsertOne(ctx context.Context, req *grpc.ModelServiceV2InsertOneRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) InsertOne(_ context.Context, req *grpc.ModelServiceV2InsertOneRequest) (res *grpc.Response, err error) { model := GetOneInstanceModel(req.ModelType) modelType := reflect.TypeOf(model) modelValuePtr := reflect.New(modelType).Interface() @@ -262,7 +271,7 @@ func (svr ModelBaseServiceServerV2) InsertOne(ctx context.Context, req *grpc.Mod return HandleSuccessWithData(r.InsertedID) } -func (svr ModelBaseServiceServerV2) InsertMany(ctx context.Context, req *grpc.ModelServiceV2InsertManyRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) InsertMany(_ context.Context, req *grpc.ModelServiceV2InsertManyRequest) (res *grpc.Response, err error) { model := GetOneInstanceModel(req.ModelType) modelType := reflect.TypeOf(model) modelsSliceType := reflect.SliceOf(modelType) @@ -284,7 +293,7 @@ func (svr ModelBaseServiceServerV2) InsertMany(ctx context.Context, req *grpc.Mo return HandleSuccessWithData(r.InsertedIDs) } -func (svr ModelBaseServiceServerV2) Count(ctx context.Context, req *grpc.ModelServiceV2CountRequest) (res *grpc.Response, err error) { +func (svr ModelBaseServiceServerV2) Count(_ context.Context, req *grpc.ModelServiceV2CountRequest) (res *grpc.Response, err error) { var query bson.M err = json.Unmarshal(req.Query, &query) if err != nil { diff --git a/core/grpc/server/node_server.go b/core/grpc/server/node_server.go index c548ea7f..ce1c9929 100644 --- a/core/grpc/server/node_server.go +++ b/core/grpc/server/node_server.go @@ -193,10 +193,7 @@ func NewNodeServer() (res *NodeServer, err error) { if err != nil { return nil, err } - svr.cfgSvc, err = nodeconfig.NewNodeConfigService() - if err != nil { - return nil, err - } + svr.cfgSvc = nodeconfig.GetNodeConfigService() return svr, nil } diff --git a/core/grpc/server/node_server_v2.go b/core/grpc/server/node_server_v2.go index ac9499ca..d1936e3e 100644 --- a/core/grpc/server/node_server_v2.go +++ b/core/grpc/server/node_server_v2.go @@ -2,20 +2,22 @@ package server import ( "context" - "encoding/json" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/notification" + "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/grpc" errors2 "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "sync" "time" ) @@ -30,84 +32,70 @@ type NodeServerV2 struct { } // Register from handler/worker to master -func (svr NodeServerV2) Register(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { +func (svr NodeServerV2) Register(_ context.Context, req *grpc.NodeServiceRegisterRequest) (res *grpc.Response, err error) { // unmarshall data - var node models.NodeV2 - if req.Data != nil { - if err := json.Unmarshal(req.Data, &node); err != nil { - return HandleError(err) - } - - if node.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } + if req.IsMaster { + // error: cannot register master node + return HandleError(errors.ErrorGrpcNotAllowed) } // node key - var nodeKey string - if req.NodeKey != "" { - nodeKey = req.NodeKey - } else { - nodeKey = node.Key - } - if nodeKey == "" { + if req.Key == "" { return HandleError(errors.ErrorModelMissingRequiredData) } // find in db - nodeDb, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + var node *models.NodeV2 + node, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.Key}, nil) if err == nil { - if node.IsMaster { - // error: cannot register master node - return HandleError(errors.ErrorGrpcNotAllowed) - } else { - // register existing - nodeDb.Status = constants.NodeStatusRegistered - nodeDb.Active = true - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(nodeDb.Id, *nodeDb) - if err != nil { - return HandleError(err) - } - log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) - } - } else if errors2.Is(err, mongo.ErrNoDocuments) { - // register new - node.Key = nodeKey + // register existing node.Status = constants.NodeStatusRegistered node.Active = true node.ActiveAt = time.Now() - node.Enabled = true - if node.Name == "" { - node.Name = nodeKey - } - node.SetCreated(primitive.NilObjectID) - node.SetUpdated(primitive.NilObjectID) - _, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(*nodeDb) + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return HandleError(err) } - log.Infof("[NodeServerV2] added worker[%s] in db. id: %s", nodeKey, node.Id.Hex()) + log.Infof("[NodeServerV2] updated worker[%s] in db. id: %s", req.Key, node.Id.Hex()) + } else if errors2.Is(err, mongo.ErrNoDocuments) { + // register new + node = &models.NodeV2{ + Key: req.Key, + Name: req.Name, + Status: constants.NodeStatusRegistered, + Active: true, + ActiveAt: time.Now(), + Enabled: true, + MaxRunners: int(req.MaxRunners), + } + node.SetCreated(primitive.NilObjectID) + node.SetUpdated(primitive.NilObjectID) + node.Id, err = service.NewModelServiceV2[models.NodeV2]().InsertOne(*node) + if err != nil { + return HandleError(err) + } + log.Infof("[NodeServerV2] added worker[%s] in db. id: %s", req.Key, node.Id.Hex()) } else { // error return HandleError(err) } - log.Infof("[NodeServerV2] master registered worker[%s]", req.GetNodeKey()) + log.Infof("[NodeServerV2] master registered worker[%s]", req.Key) return HandleSuccessWithData(node) } // SendHeartbeat from worker to master -func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { +func (svr NodeServerV2) SendHeartbeat(_ context.Context, req *grpc.NodeServiceSendHeartbeatRequest) (res *grpc.Response, err error) { // find in db - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.NodeKey}, nil) + node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": req.Key}, nil) if err != nil { if errors2.Is(err, mongo.ErrNoDocuments) { return HandleError(errors.ErrorNodeNotExists) } return HandleError(err) } + oldStatus := node.Status // validate status if node.Status == constants.NodeStatusUnregistered { @@ -122,15 +110,18 @@ func (svr NodeServerV2) SendHeartbeat(ctx context.Context, req *grpc.Request) (r if err != nil { return HandleError(err) } + newStatus := node.Status + + // send notification if status changed + if utils.IsPro() { + if oldStatus != newStatus { + go notification.GetNotificationServiceV2().SendNodeNotification(node) + } + } return HandleSuccessWithData(node) } -// Ping from worker to master -func (svr NodeServerV2) Ping(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { - return HandleSuccess() -} - func (svr NodeServerV2) Subscribe(request *grpc.Request, stream grpc.NodeService_SubscribeServer) (err error) { log.Infof("[NodeServerV2] master received subscribe request from node[%s]", request.NodeKey) @@ -159,7 +150,7 @@ func (svr NodeServerV2) Subscribe(request *grpc.Request, stream grpc.NodeService } } -func (svr NodeServerV2) Unsubscribe(ctx context.Context, req *grpc.Request) (res *grpc.Response, err error) { +func (svr NodeServerV2) Unsubscribe(_ context.Context, req *grpc.Request) (res *grpc.Response, err error) { sub, err := svr.server.GetSubscribe("node:" + req.NodeKey) if err != nil { return nil, errors.ErrorGrpcSubscribeNotExists @@ -177,13 +168,22 @@ func (svr NodeServerV2) Unsubscribe(ctx context.Context, req *grpc.Request) (res }, nil } +var nodeSvrV2 *NodeServerV2 +var nodeSvrV2Once = new(sync.Once) + func NewNodeServerV2() (res *NodeServerV2, err error) { - // node server - svr := &NodeServerV2{} - svr.cfgSvc, err = nodeconfig.NewNodeConfigService() + if nodeSvrV2 != nil { + return nodeSvrV2, nil + } + nodeSvrV2Once.Do(func() { + nodeSvrV2 = &NodeServerV2{} + nodeSvrV2.cfgSvc = nodeconfig.GetNodeConfigService() + if err != nil { + log.Errorf("[NodeServerV2] error: %s", err.Error()) + } + }) if err != nil { return nil, err } - - return svr, nil + return nodeSvrV2, nil } diff --git a/core/grpc/server/options.go b/core/grpc/server/options.go deleted file mode 100644 index afae5dba..00000000 --- a/core/grpc/server/options.go +++ /dev/null @@ -1,43 +0,0 @@ -package server - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" -) - -type Option func(svr interfaces.GrpcServer) - -func WithConfigPath(path string) Option { - return func(svr interfaces.GrpcServer) { - svr.SetConfigPath(path) - } -} - -func WithAddress(address interfaces.Address) Option { - return func(svr interfaces.GrpcServer) { - svr.SetAddress(address) - } -} - -type NodeServerOption func(svr *NodeServer) - -func WithServerNodeServerService(server interfaces.GrpcServer) NodeServerOption { - return func(svr *NodeServer) { - svr.server = server - } -} - -type TaskServerOption func(svr *TaskServer) - -func WithServerTaskServerService(server interfaces.GrpcServer) TaskServerOption { - return func(svr *TaskServer) { - svr.server = server - } -} - -type MessageServerOption func(svr *MessageServer) - -func WithServerMessageServerService(server interfaces.GrpcServer) MessageServerOption { - return func(svr *MessageServer) { - svr.server = server - } -} diff --git a/core/grpc/server/server.go b/core/grpc/server/server.go deleted file mode 100644 index 5639e641..00000000 --- a/core/grpc/server/server.go +++ /dev/null @@ -1,265 +0,0 @@ -package server - -import ( - "encoding/json" - "fmt" - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/middlewares" - "github.com/crawlab-team/crawlab/core/interfaces" - grpc2 "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/grpc-ecosystem/go-grpc-middleware" - grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth" - "github.com/grpc-ecosystem/go-grpc-middleware/recovery" - "github.com/spf13/viper" - "go/types" - "google.golang.org/grpc" - "net" - "sync" -) - -var subs = sync.Map{} - -type Server struct { - // dependencies - nodeCfgSvc interfaces.NodeConfigService - nodeSvr *NodeServer - taskSvr *TaskServer - messageSvr *MessageServer - modelDelegateSvr *ModelDelegateServer - modelBaseServiceSvr *ModelBaseServiceServer - - // settings - cfgPath string - address interfaces.Address - - // internals - svr *grpc.Server - l net.Listener - stopped bool -} - -func (svr *Server) Init() (err error) { - // register - if err := svr.Register(); err != nil { - return err - } - - return nil -} - -func (svr *Server) Start() (err error) { - // grpc server binding address - address := svr.address.String() - - // listener - svr.l, err = net.Listen("tcp", address) - if err != nil { - _ = trace.TraceError(err) - return errors.ErrorGrpcServerFailedToListen - } - log.Infof("grpc server listens to %s", address) - - // start grpc server - go func() { - if err := svr.svr.Serve(svr.l); err != nil { - if err == grpc.ErrServerStopped { - return - } - trace.PrintError(err) - log.Error(errors.ErrorGrpcServerFailedToServe.Error()) - } - }() - - return nil -} - -func (svr *Server) Stop() (err error) { - // skip if listener is nil - if svr.l == nil { - return nil - } - - // graceful stop - log.Infof("grpc server stopping...") - svr.svr.Stop() - - // close listener - log.Infof("grpc server closing listener...") - _ = svr.l.Close() - - // mark as stopped - svr.stopped = true - - // log - log.Infof("grpc server stopped") - - return nil -} - -func (svr *Server) Register() (err error) { - grpc2.RegisterModelDelegateServer(svr.svr, *svr.modelDelegateSvr) // model delegate - grpc2.RegisterModelBaseServiceServer(svr.svr, *svr.modelBaseServiceSvr) // model base service - grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service - grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) // task service - grpc2.RegisterMessageServiceServer(svr.svr, *svr.messageSvr) // message service - - return nil -} - -func (svr *Server) SetAddress(address interfaces.Address) { - svr.address = address -} - -func (svr *Server) GetConfigPath() (path string) { - return svr.cfgPath -} - -func (svr *Server) SetConfigPath(path string) { - svr.cfgPath = path -} - -func (svr *Server) GetSubscribe(key string) (sub interfaces.GrpcSubscribe, err error) { - res, ok := subs.Load(key) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcStreamNotFound) - } - sub, ok = res.(interfaces.GrpcSubscribe) - if !ok { - return nil, trace.TraceError(errors.ErrorGrpcInvalidType) - } - return sub, nil -} - -func (svr *Server) SetSubscribe(key string, sub interfaces.GrpcSubscribe) { - subs.Store(key, sub) -} - -func (svr *Server) DeleteSubscribe(key string) { - subs.Delete(key) -} - -func (svr *Server) SendStreamMessage(key string, code grpc2.StreamMessageCode) (err error) { - return svr.SendStreamMessageWithData(key, code, nil) -} - -func (svr *Server) SendStreamMessageWithData(key string, code grpc2.StreamMessageCode, d interface{}) (err error) { - var data []byte - switch d.(type) { - case types.Nil: - // do nothing - case []byte: - data = d.([]byte) - default: - var err error - data, err = json.Marshal(d) - if err != nil { - panic(err) - } - } - sub, err := svr.GetSubscribe(key) - if err != nil { - return err - } - msg := &grpc2.StreamMessage{ - Code: code, - Key: svr.nodeCfgSvc.GetNodeKey(), - Data: data, - } - return sub.GetStream().Send(msg) -} - -func (svr *Server) IsStopped() (res bool) { - return svr.stopped -} - -func (svr *Server) recoveryHandlerFunc(p interface{}) (err error) { - err = errors.NewError(errors.ErrorPrefixGrpc, fmt.Sprintf("%v", p)) - trace.PrintError(err) - return err -} - -func NewServer() (svr2 interfaces.GrpcServer, err error) { - // server - svr := &Server{ - cfgPath: config2.GetConfigPath(), - address: entity.NewAddress(&entity.AddressOptions{ - Host: constants.DefaultGrpcServerHost, - Port: constants.DefaultGrpcServerPort, - }), - } - - if viper.GetString("grpc.server.address") != "" { - svr.address, err = entity.NewAddressFromString(viper.GetString("grpc.server.address")) - if err != nil { - return nil, err - } - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - modelDelegateSvr *ModelDelegateServer, - modelBaseServiceSvr *ModelBaseServiceServer, - nodeSvr *NodeServer, - taskSvr *TaskServer, - messageSvr *MessageServer, - ) { - // dependencies - svr.nodeCfgSvc = nodeCfgSvc - svr.modelDelegateSvr = modelDelegateSvr - svr.modelBaseServiceSvr = modelBaseServiceSvr - svr.nodeSvr = nodeSvr - svr.taskSvr = taskSvr - svr.messageSvr = messageSvr - - // server - svr.nodeSvr.server = svr - svr.taskSvr.server = svr - svr.messageSvr.server = svr - }); err != nil { - return nil, err - } - - // recovery options - recoveryOpts := []grpc_recovery.Option{ - grpc_recovery.WithRecoveryHandler(svr.recoveryHandlerFunc), - } - - // grpc server - svr.svr = grpc.NewServer( - grpc_middleware.WithUnaryServerChain( - grpc_recovery.UnaryServerInterceptor(recoveryOpts...), - grpc_auth.UnaryServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), - ), - grpc_middleware.WithStreamServerChain( - grpc_recovery.StreamServerInterceptor(recoveryOpts...), - grpc_auth.StreamServerInterceptor(middlewares.GetAuthTokenFunc(svr.nodeCfgSvc)), - ), - ) - - // initialize - if err := svr.Init(); err != nil { - return nil, err - } - - return svr, nil -} - -var _server interfaces.GrpcServer - -func GetServer() (svr interfaces.GrpcServer, err error) { - if _server != nil { - return _server, nil - } - _server, err = NewServer() - if err != nil { - return nil, err - } - return _server, nil -} diff --git a/core/grpc/server/server_v2.go b/core/grpc/server/server_v2.go index ebcb871b..e8c8ba06 100644 --- a/core/grpc/server/server_v2.go +++ b/core/grpc/server/server_v2.go @@ -42,10 +42,11 @@ type GrpcServerV2 struct { nodeCfgSvc interfaces.NodeConfigService // servers - nodeSvr *NodeServerV2 - taskSvr *TaskServerV2 - modelBaseServiceSvr *ModelBaseServiceServerV2 - dependenciesSvr *DependenciesServerV2 + NodeSvr *NodeServerV2 + TaskSvr *TaskServerV2 + ModelBaseServiceSvr *ModelBaseServiceServerV2 + DependenciesSvr *DependenciesServerV2 + MetricsSvr *MetricsServerV2 } func (svr *GrpcServerV2) GetConfigPath() (path string) { @@ -115,9 +116,11 @@ func (svr *GrpcServerV2) Stop() (err error) { } func (svr *GrpcServerV2) Register() (err error) { - grpc2.RegisterNodeServiceServer(svr.svr, *svr.nodeSvr) // node service - grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.modelBaseServiceSvr) - grpc2.RegisterTaskServiceServer(svr.svr, *svr.taskSvr) + grpc2.RegisterNodeServiceServer(svr.svr, *svr.NodeSvr) + grpc2.RegisterModelBaseServiceV2Server(svr.svr, *svr.ModelBaseServiceSvr) + grpc2.RegisterTaskServiceServer(svr.svr, *svr.TaskSvr) + grpc2.RegisterDependenciesServiceV2Server(svr.svr, *svr.DependenciesSvr) + grpc2.RegisterMetricsServiceV2Server(svr.svr, *svr.MetricsSvr) return nil } @@ -206,16 +209,17 @@ func NewGrpcServerV2() (svr *GrpcServerV2, err error) { svr.nodeCfgSvc = nodeconfig.GetNodeConfigService() - svr.nodeSvr, err = NewNodeServerV2() + svr.NodeSvr, err = NewNodeServerV2() if err != nil { return nil, err } - svr.modelBaseServiceSvr = NewModelBaseServiceV2Server() - svr.taskSvr, err = NewTaskServerV2() + svr.ModelBaseServiceSvr = NewModelBaseServiceV2Server() + svr.TaskSvr, err = NewTaskServerV2() if err != nil { return nil, err } - svr.dependenciesSvr = NewDependenciesServerV2() + svr.DependenciesSvr = GetDependenciesServerV2() + svr.MetricsSvr = GetMetricsServerV2() // recovery options recoveryOpts := []grpc_recovery.Option{ diff --git a/core/grpc/server/task_server.go b/core/grpc/server/task_server.go deleted file mode 100644 index b6d08592..00000000 --- a/core/grpc/server/task_server.go +++ /dev/null @@ -1,238 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/notification" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "io" - "strings" -) - -type TaskServer struct { - grpc.UnimplementedTaskServiceServer - - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - statsSvc interfaces.TaskStatsService - - // internals - server interfaces.GrpcServer -} - -// Subscribe to task stream when a task runner in a node starts -func (svr TaskServer) Subscribe(stream grpc.TaskService_SubscribeServer) (err error) { - for { - msg, err := stream.Recv() - utils.LogDebug(msg.String()) - if err == io.EOF { - return nil - } - if err != nil { - if strings.HasSuffix(err.Error(), "context canceled") { - return nil - } - trace.PrintError(err) - continue - } - switch msg.Code { - case grpc.StreamMessageCode_INSERT_DATA: - err = svr.handleInsertData(msg) - case grpc.StreamMessageCode_INSERT_LOGS: - err = svr.handleInsertLogs(msg) - default: - err = errors.ErrorGrpcInvalidCode - log.Errorf("invalid stream message code: %d", msg.Code) - continue - } - if err != nil { - log.Errorf("grpc error[%d]: %v", msg.Code, err) - } - } -} - -// Fetch tasks to be executed by a task handler -func (svr TaskServer) Fetch(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - nodeKey := request.GetNodeKey() - if nodeKey == "" { - return nil, trace.TraceError(errors.ErrorGrpcInvalidNodeKey) - } - n, err := svr.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil { - return nil, trace.TraceError(err) - } - var tid primitive.ObjectID - opts := &mongo.FindOptions{ - Sort: bson.D{ - {"p", 1}, - {"_id", 1}, - }, - Limit: 1, - } - if err := mongo.RunTransactionWithContext(ctx, func(sc mongo2.SessionContext) (err error) { - // get task queue item assigned to this node - tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": n.Id}, opts, n.Id) - if err != nil { - return err - } - if !tid.IsZero() { - return nil - } - - // get task queue item assigned to any node (random mode) - tid, err = svr.getTaskQueueItemIdAndDequeue(bson.M{"nid": nil}, opts, n.Id) - if !tid.IsZero() { - return nil - } - if err != nil { - return err - } - return nil - }); err != nil { - return nil, err - } - return HandleSuccessWithData(tid) -} - -func (svr TaskServer) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - svc := notification.GetService() - var t = new(models.Task) - if err := json.Unmarshal(request.Data, t); err != nil { - return nil, trace.TraceError(err) - } - t, err = svr.modelSvc.GetTaskById(t.Id) - if err != nil { - return nil, trace.TraceError(err) - } - td, err := json.Marshal(t) - if err != nil { - return nil, trace.TraceError(err) - } - var e bson.M - if err := json.Unmarshal(td, &e); err != nil { - return nil, trace.TraceError(err) - } - ts, err := svr.modelSvc.GetTaskStatById(t.Id) - if err != nil { - return nil, trace.TraceError(err) - } - settings, _, err := svc.GetSettingList(bson.M{ - "enabled": true, - }, nil, nil) - if err != nil { - return nil, trace.TraceError(err) - } - for _, s := range settings { - switch s.TaskTrigger { - case constants.NotificationTriggerTaskFinish: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - _ = svc.Send(s, e) - } - case constants.NotificationTriggerTaskError: - if t.Status == constants.TaskStatusError || t.Status == constants.TaskStatusAbnormal { - _ = svc.Send(s, e) - } - case constants.NotificationTriggerTaskEmptyResults: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - if ts.ResultCount == 0 { - _ = svc.Send(s, e) - } - } - case constants.NotificationTriggerTaskNever: - } - } - return nil, nil -} - -func (svr TaskServer) handleInsertData(msg *grpc.StreamMessage) (err error) { - data, err := svr.deserialize(msg) - if err != nil { - return err - } - var records []interface{} - for _, d := range data.Records { - res, ok := d[constants.TaskKey] - if ok { - switch res.(type) { - case string: - id, err := primitive.ObjectIDFromHex(res.(string)) - if err == nil { - d[constants.TaskKey] = id - } - } - } - records = append(records, d) - } - return svr.statsSvc.InsertData(data.TaskId, records...) -} - -func (svr TaskServer) handleInsertLogs(msg *grpc.StreamMessage) (err error) { - data, err := svr.deserialize(msg) - if err != nil { - return err - } - return svr.statsSvc.InsertLogs(data.TaskId, data.Logs...) -} - -func (svr TaskServer) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { - var tq models.TaskQueueItem - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Find(query, opts).One(&tq); err != nil { - if err == mongo2.ErrNoDocuments { - return tid, nil - } - return tid, trace.TraceError(err) - } - t, err := svr.modelSvc.GetTaskById(tq.Id) - if err == nil { - t.NodeId = nid - _ = delegate.NewModelDelegate(t).Save() - } - _ = delegate.NewModelDelegate(&tq).Delete() - return tq.Id, nil -} - -func (svr TaskServer) deserialize(msg *grpc.StreamMessage) (data entity.StreamMessageTaskData, err error) { - if err := json.Unmarshal(msg.Data, &data); err != nil { - return data, trace.TraceError(err) - } - if data.TaskId.IsZero() { - return data, trace.TraceError(errors.ErrorGrpcInvalidType) - } - return data, nil -} - -func NewTaskServer() (res *TaskServer, err error) { - // task server - svr := &TaskServer{} - - // dependency injection - if err := container.GetContainer().Invoke(func( - modelSvc service.ModelService, - statsSvc interfaces.TaskStatsService, - cfgSvc interfaces.NodeConfigService, - ) { - svr.modelSvc = modelSvc - svr.statsSvc = statsSvc - svr.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } - - return svr, nil -} diff --git a/core/grpc/server/task_server_v2.go b/core/grpc/server/task_server_v2.go index f1c5585e..050a5cd1 100644 --- a/core/grpc/server/task_server_v2.go +++ b/core/grpc/server/task_server_v2.go @@ -8,7 +8,7 @@ import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" @@ -72,7 +72,7 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo if nodeKey == "" { return nil, errors.New("invalid node key") } - n, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return nil, trace.TraceError(err) } @@ -109,53 +109,103 @@ func (svr TaskServerV2) Fetch(ctx context.Context, request *grpc.Request) (respo return HandleSuccessWithData(tid) } -func (svr TaskServerV2) SendNotification(ctx context.Context, request *grpc.Request) (response *grpc.Response, err error) { - svc := notification.GetServiceV2() - var t = new(models.TaskV2) - if err := json.Unmarshal(request.Data, t); err != nil { +func (svr TaskServerV2) SendNotification(_ context.Context, request *grpc.TaskServiceSendNotificationRequest) (response *grpc.Response, err error) { + if !utils.IsPro() { + return nil, nil + } + + // task id + taskId, err := primitive.ObjectIDFromHex(request.TaskId) + if err != nil { + log.Errorf("invalid task id: %s", request.TaskId) return nil, trace.TraceError(err) } - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(t.Id) + + // arguments + var args []any + + // task + task, err := service.NewModelServiceV2[models2.TaskV2]().GetById(taskId) + if err != nil { + log.Errorf("task not found: %s", request.TaskId) + return nil, trace.TraceError(err) + } + args = append(args, task) + + // task stat + taskStat, err := service.NewModelServiceV2[models2.TaskStatV2]().GetById(task.Id) + if err != nil { + log.Errorf("task stat not found for task: %s", request.TaskId) + return nil, trace.TraceError(err) + } + args = append(args, taskStat) + + // spider + spider, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(task.SpiderId) + if err != nil { + log.Errorf("spider not found for task: %s", request.TaskId) + return nil, trace.TraceError(err) + } + args = append(args, spider) + + // node + node, err := service.NewModelServiceV2[models2.NodeV2]().GetById(task.NodeId) if err != nil { return nil, trace.TraceError(err) } - td, err := json.Marshal(t) - if err != nil { - return nil, trace.TraceError(err) + args = append(args, node) + + // schedule + var schedule *models2.ScheduleV2 + if !task.ScheduleId.IsZero() { + schedule, err = service.NewModelServiceV2[models2.ScheduleV2]().GetById(task.ScheduleId) + if err != nil { + log.Errorf("schedule not found for task: %s", request.TaskId) + return nil, trace.TraceError(err) + } + args = append(args, schedule) } - var e bson.M - if err := json.Unmarshal(td, &e); err != nil { - return nil, trace.TraceError(err) - } - ts, err := service.NewModelServiceV2[models.TaskStatV2]().GetById(t.Id) - if err != nil { - return nil, trace.TraceError(err) - } - settings, _, err := svc.GetSettingList(bson.M{ + + // settings + settings, err := service.NewModelServiceV2[models2.NotificationSettingV2]().GetMany(bson.M{ "enabled": true, - }, nil, nil) + "trigger": bson.M{ + "$regex": constants.NotificationTriggerPatternTask, + }, + }, nil) if err != nil { return nil, trace.TraceError(err) } + + // notification service + svc := notification.GetNotificationServiceV2() + for _, s := range settings { - switch s.TaskTrigger { + // compatible with old settings + trigger := s.Trigger + if trigger == "" { + trigger = s.TaskTrigger + } + + // send notification + switch trigger { case constants.NotificationTriggerTaskFinish: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - _ = svc.Send(&s, e) + if task.Status != constants.TaskStatusPending && task.Status != constants.TaskStatusRunning { + go svc.Send(&s, args...) } case constants.NotificationTriggerTaskError: - if t.Status == constants.TaskStatusError || t.Status == constants.TaskStatusAbnormal { - _ = svc.Send(&s, e) + if task.Status == constants.TaskStatusError || task.Status == constants.TaskStatusAbnormal { + go svc.Send(&s, args...) } case constants.NotificationTriggerTaskEmptyResults: - if t.Status != constants.TaskStatusPending && t.Status != constants.TaskStatusRunning { - if ts.ResultCount == 0 { - _ = svc.Send(&s, e) + if task.Status != constants.TaskStatusPending && task.Status != constants.TaskStatusRunning { + if taskStat.ResultCount == 0 { + go svc.Send(&s, args...) } } - case constants.NotificationTriggerTaskNever: } } + return nil, nil } @@ -164,7 +214,7 @@ func (svr TaskServerV2) handleInsertData(msg *grpc.StreamMessage) (err error) { if err != nil { return err } - var records []interface{} + var records []map[string]interface{} for _, d := range data.Records { res, ok := d[constants.TaskKey] if ok { @@ -190,22 +240,22 @@ func (svr TaskServerV2) handleInsertLogs(msg *grpc.StreamMessage) (err error) { } func (svr TaskServerV2) getTaskQueueItemIdAndDequeue(query bson.M, opts *mongo.FindOptions, nid primitive.ObjectID) (tid primitive.ObjectID, err error) { - tq, err := service.NewModelServiceV2[models.TaskQueueItemV2]().GetOne(query, opts) + tq, err := service.NewModelServiceV2[models2.TaskQueueItemV2]().GetOne(query, opts) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return tid, nil } return tid, trace.TraceError(err) } - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(tq.Id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(tq.Id) if err == nil { t.NodeId = nid - err = service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + err = service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) if err != nil { return tid, trace.TraceError(err) } } - err = service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(tq.Id) + err = service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(tq.Id) if err != nil { return tid, trace.TraceError(err) } diff --git a/core/grpc/test/auth_token_test.go b/core/grpc/test/auth_token_test.go deleted file mode 100644 index fa09e976..00000000 --- a/core/grpc/test/auth_token_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package test - -import ( - "context" - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/node/config" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/stretchr/testify/require" - "io/ioutil" - "os" - "path" - "testing" -) - -func TestAuthToken(t *testing.T) { - var err error - - // auth key - authKey := "test-auth-key" - - // auth key (invalid) - authKeyInvalid := "test-auth-key-invalid" - - // tmp dir - tmpDir := os.TempDir() - - // master config - masterConfigPath := path.Join(tmpDir, "config-master.json") - masterConfig := config.Config{ - Key: "master", - IsMaster: true, - AuthKey: authKey, - } - masterConfigData, err := json.Marshal(&masterConfig) - require.Nil(t, err) - err = ioutil.WriteFile(masterConfigPath, masterConfigData, os.FileMode(0777)) - - // worker config - workerConfigPath := path.Join(tmpDir, "config-worker.json") - workerConfig := config.Config{ - Key: "worker", - IsMaster: false, - AuthKey: authKey, - } - workerConfigData, err := json.Marshal(&workerConfig) - require.Nil(t, err) - err = ioutil.WriteFile(workerConfigPath, workerConfigData, os.FileMode(0777)) - - // worker config (invalid) - workerInvalidConfigPath := path.Join(tmpDir, "worker-invalid") - workerInvalidConfig := config.Config{ - Key: "worker", - IsMaster: false, - AuthKey: authKeyInvalid, - } - workerInvalidConfigData, err := json.Marshal(&workerInvalidConfig) - require.Nil(t, err) - err = ioutil.WriteFile(workerInvalidConfigPath, workerInvalidConfigData, os.FileMode(0777)) - - // server - svr, err := server.NewServer( - server.WithConfigPath(masterConfigPath), - server.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "0.0.0.0", - Port: "9999", - })), - ) - require.Nil(t, err) - err = svr.Start() - require.Nil(t, err) - - // client - c, err := client.GetClient(workerConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "localhost", - Port: "9999", - }))) - require.Nil(t, err) - err = c.Start() - require.Nil(t, err) - _, err = c.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerConfig.Key}) - require.Nil(t, err) - - // client (invalid) - ci, err := client.GetClient(workerInvalidConfigPath, client.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: "localhost", - Port: "9999", - }))) - require.Nil(t, err) - err = ci.Start() - require.Nil(t, err) - _, err = ci.GetNodeClient().Ping(context.Background(), &grpc.Request{NodeKey: workerInvalidConfig.Key}) - require.NotNil(t, err) -} diff --git a/core/grpc/test/base.go b/core/grpc/test/base.go deleted file mode 100644 index e8e3923e..00000000 --- a/core/grpc/test/base.go +++ /dev/null @@ -1,77 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/node/test" - "testing" - "time" -) - -type Test struct { - Server interfaces.GrpcServer - Client interfaces.GrpcClient - - MasterNodeInfo *entity.NodeInfo - WorkerNodeInfo *entity.NodeInfo -} - -func (t *Test) Setup(t2 *testing.T) { - test.T.Cleanup() - t2.Cleanup(t.Cleanup) - if !T.Client.IsStarted() { - _ = T.Client.Start() - } else if T.Client.IsClosed() { - _ = T.Client.Restart() - } -} - -func (t *Test) Cleanup() { - _ = t.Client.Stop() - _ = t.Server.Stop() - test.T.Cleanup() - - // wait to avoid caching - time.Sleep(200 * time.Millisecond) -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // server - t.Server, err = server.NewServer( - server.WithConfigPath(test.T.MasterSvc.GetConfigPath()), - server.WithAddress(test.T.MasterSvc.GetAddress()), - ) - if err != nil { - return nil, err - } - if err := t.Server.Start(); err != nil { - return nil, err - } - - // client - t.Client, err = client.GetClient(test.T.WorkerSvc.GetConfigPath()) - if err != nil { - return nil, err - } - - // master node info - t.MasterNodeInfo = &entity.NodeInfo{ - Key: "master", - IsMaster: true, - } - - // worker node info - t.WorkerNodeInfo = &entity.NodeInfo{ - Key: "worker", - IsMaster: false, - } - - return t, nil -} diff --git a/core/grpc/test/main_test.go b/core/grpc/test/main_test.go deleted file mode 100644 index 232a8c6e..00000000 --- a/core/grpc/test/main_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package test - -import "testing" - -func TestMain(m *testing.M) { - m.Run() -} diff --git a/core/grpc/test/model_base_service_server_test.go b/core/grpc/test/model_base_service_server_test.go deleted file mode 100644 index ad9b5372..00000000 --- a/core/grpc/test/model_base_service_server_test.go +++ /dev/null @@ -1,312 +0,0 @@ -package test - -import ( - "encoding/json" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestModelBaseService_GetById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().GetById(ctx, req) - require.Nil(t, err) - var p2 models.Project - err = json.Unmarshal(res.Data, &p2) - require.Nil(t, err) - - // validate - require.Equal(t, p.Id, p2.Id) - require.Equal(t, p.Name, p2.Name) - require.Equal(t, p.Description, p2.Description) -} - -func TestModelBaseService_Get(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().Get(ctx, req) - require.Nil(t, err) - var p2 models.Project - err = json.Unmarshal(res.Data, &p2) - require.Nil(t, err) - - // validate - require.Equal(t, p.Id, p2.Id) - require.Equal(t, p.Name, p2.Name) - require.Equal(t, p.Description, p2.Description) -} - -func TestModelBaseService_GetList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // get list - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().GetList(ctx, req) - require.Nil(t, err) - var list []models.Project - err = json.Unmarshal(res.Data, &list) - require.Nil(t, err) - - // validate - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_DeleteById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().DeleteById(ctx, req) - require.Nil(t, err) - - // validate - p, err = test.T.ModelSvc.GetProjectById(p.Id) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelBaseService_Delete(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Delete(ctx, req) - require.Nil(t, err) - - // validate - p, err = test.T.ModelSvc.GetProjectById(p.Id) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelBaseService_DeleteList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().DeleteList(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) -} - -func TestModelBaseService_ForceDeleteList(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // delete by id - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().ForceDeleteList(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) -} - -func TestModelBaseService_UpdateById(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - p, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // update by id - ctx, cancel := T.Client.Context() - defer cancel() - update := bson.M{ - "name": "test-new-project", - } - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Id: p.Id, Update: update}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().UpdateById(ctx, req) - require.Nil(t, err) - - // validate - p2, err := test.T.ModelSvc.GetProjectById(p.Id) - require.Nil(t, err) - require.Equal(t, "test-new-project", p2.Name) -} - -func TestModelBaseService_Update(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // update - ctx, cancel := T.Client.Context() - defer cancel() - update := bson.M{ - "name": "test-new-project", - } - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}, Update: update}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Update(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, 0, len(list)) - list, err = test.T.ModelSvc.GetProjectList(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_Insert(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // insert - var docs []interface{} - n := 10 - for i := 0; i < n; i++ { - docs = append(docs, models.Project{ - Id: primitive.NewObjectID(), - Name: "test-project", - }) - } - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Docs: docs}) - require.Nil(t, err) - _, err = T.Client.GetModelBaseServiceClient().Insert(ctx, req) - require.Nil(t, err) - - // validate - list, err := test.T.ModelSvc.GetProjectList(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - require.Equal(t, n, len(list)) -} - -func TestModelBaseService_Count(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - n := 10 - for i := 0; i < n; i++ { - modelDelegateAdd(t) - } - - // count - ctx, cancel := T.Client.Context() - defer cancel() - req, err := T.Client.NewModelBaseServiceRequest(interfaces.ModelIdProject, &entity.GrpcBaseServiceParams{Query: bson.M{"name": "test-project"}}) - require.Nil(t, err) - res, err := T.Client.GetModelBaseServiceClient().Count(ctx, req) - require.Nil(t, err) - - // validate - var total int - err = json.Unmarshal(res.Data, &total) - require.Nil(t, err) - require.Equal(t, n, total) -} diff --git a/core/grpc/test/model_delegate_server_test.go b/core/grpc/test/model_delegate_server_test.go deleted file mode 100644 index c761b9b6..00000000 --- a/core/grpc/test/model_delegate_server_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "testing" -) - -func TestModelDelegate_Do(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAdd(t) - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - // get artifact - a := modelDelegateGetArtifact(t) - require.Equal(t, project.GetId(), a.GetId()) - - // save - modelDelegateSave(t) - project, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - require.Equal(t, "test-new-project", project.Name) - require.Equal(t, "test-new-description", project.Description) - - // delete - modelDelegateDelete(t) - _, err = test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Equal(t, mongo2.ErrNoDocuments, err) -} - -func TestModelDelegate_Do_All(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - - // add - modelDelegateAddAll(t) - modelDelegateValidateAddAll(t) -} - -func modelDelegateAdd(t *testing.T) { - // modelDelegateAdd - project := &models.Project{ - Name: "test-project", - Description: "test-description", - } - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err := projectD.Add() - require.Nil(t, err) -} - -func modelDelegateGetArtifact(t *testing.T) interfaces.ModelArtifact { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - a, err := projectD.GetArtifact() - require.Nil(t, err) - return a -} - -func modelDelegateSave(t *testing.T) { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-project"}, nil) - require.Nil(t, err) - - project.Name = "test-new-project" - project.Description = "test-new-description" - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err = projectD.Save() - require.Nil(t, err) -} - -func modelDelegateDelete(t *testing.T) { - project, err := test.T.ModelSvc.GetProject(bson.M{"name": "test-new-project"}, nil) - require.Nil(t, err) - - projectD := client.NewModelDelegate(project, client.WithDelegateConfigPath(T.Client.GetConfigPath())) - err = projectD.Delete() - require.Nil(t, err) -} - -func modelDelegateAddAll(t *testing.T) { - var err error - cfgOpt := client.WithDelegateConfigPath(T.Client.GetConfigPath()) - m := models.NewModelMap() - err = client.NewModelDelegate(&m.Tag, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Node, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Project, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Spider, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Task, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Job, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Schedule, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.User, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Setting, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Token, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.Variable, cfgOpt).Add() - require.Nil(t, err) - err = client.NewModelDelegate(&m.User, cfgOpt).Add() - require.Nil(t, err) -} - -func modelDelegateValidateAddAll(t *testing.T) { - var err error - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) - _, err = test.T.ModelSvc.GetBaseService(interfaces.ModelIdNode).Get(bson.M{}, nil) - require.Nil(t, err) -} diff --git a/core/grpc/test/node_server_test.go b/core/grpc/test/node_server_test.go deleted file mode 100644 index de858bed..00000000 --- a/core/grpc/test/node_server_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package test - -import ( - "context" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/node/test" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestGrpcServer_Register(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) -} - -func TestGrpcServer_Register_Existing(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // add to db - node := &models.Node{ - Key: T.WorkerNodeInfo.Key, - IsMaster: false, - Status: constants.NodeStatusUnregistered, - } - nodeD := delegate.NewModelDelegate(node) - err = nodeD.Add() - require.Nil(t, err) - - // register - register(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusRegistered, workerNode.Status) -} - -func TestGrpcServer_SendHeartbeat(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // send heartbeat - sendHeartbeat(t) - - // validate - workerNodeKey := T.WorkerNodeInfo.Key - workerNode, err := test.T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, workerNodeKey, workerNode.Key) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) -} - -func TestGrpcServer_Subscribe(t *testing.T) { - var err error - - T, _ = NewTest() - T.Setup(t) - - // register - register(t) - - // handle client message - go handleClientMessage(t) - - time.Sleep(1 * time.Second) - - // server PING client - sub, err := T.Server.GetSubscribe("node:" + T.WorkerNodeInfo.Key) - require.Nil(t, err) - require.NotNil(t, sub) - err = sub.GetStream().Send(&grpc.StreamMessage{ - Code: grpc.StreamMessageCode_PING, - NodeKey: T.MasterNodeInfo.Key, - }) - require.Nil(t, err) - - // wait - time.Sleep(1 * time.Second) - - // validate - workerNode, err := test.T.ModelSvc.GetNodeByKey(T.WorkerNodeInfo.Key, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) -} - -func register(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - res, err := T.Client.GetNodeClient().Register(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.Equal(t, grpc.ResponseCode_OK, res.Code) -} - -func sendHeartbeat(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - res, err := T.Client.GetNodeClient().SendHeartbeat(ctx, T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.Equal(t, grpc.ResponseCode_OK, res.Code) -} - -func handleClientMessage(t *testing.T) { - msgCh := T.Client.GetMessageChannel() - for { - msg := <-msgCh - switch msg.Code { - case grpc.StreamMessageCode_PING: - require.Equal(t, T.MasterNodeInfo.Key, msg.NodeKey) - res, err := T.Client.GetNodeClient().SendHeartbeat(context.Background(), T.Client.NewRequest(T.WorkerNodeInfo)) - require.Nil(t, err) - require.NotNil(t, res) - } - } -} diff --git a/core/interfaces/fs_service_v2.go b/core/interfaces/fs_service_v2.go index 4bc3d7df..533a9d6b 100644 --- a/core/interfaces/fs_service_v2.go +++ b/core/interfaces/fs_service_v2.go @@ -9,4 +9,5 @@ type FsServiceV2 interface { Rename(path, newPath string) (err error) Delete(path string) (err error) Copy(path, newPath string) (err error) + Export() (resultPath string, err error) } diff --git a/core/models/client/model_service_v2.go b/core/models/client/model_service_v2.go index 049f584e..8ad827d8 100644 --- a/core/models/client/model_service_v2.go +++ b/core/models/client/model_service_v2.go @@ -331,17 +331,14 @@ func NewModelServiceV2[T any]() *ModelServiceV2[T] { typeName := t.Name() if _, exists := onceMap[typeName]; !exists { - onceMap[typeName] = &sync.Once{} + onceMap[typeName] = new(sync.Once) } var instance *ModelServiceV2[T] - c, err := client.GetGrpcClientV2() - if err != nil { - panic(err) - } + c := client.GetGrpcClientV2() if !c.IsStarted() { - err = c.Start() + err := c.Start() if err != nil { panic(err) } diff --git a/core/models/client/model_service_v2_test.go b/core/models/client/model_service_v2_test.go index c4053bb1..6738af4a 100644 --- a/core/models/client/model_service_v2_test.go +++ b/core/models/client/model_service_v2_test.go @@ -2,9 +2,10 @@ package client_test import ( "context" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" @@ -17,7 +18,7 @@ import ( "time" ) -type TestModel models.TestModel +type TestModel models.TestModelV2 func setupTestDB() { viper.Set("mongo.db", "testdb") @@ -28,13 +29,27 @@ func teardownTestDB() { db.Drop(context.Background()) } +func startSvr(svr *server.GrpcServerV2) { + err := svr.Start() + if err != nil { + log.Errorf("failed to start grpc server: %v", err) + } +} + +func stopSvr(svr *server.GrpcServerV2) { + err := svr.Stop() + if err != nil { + log.Errorf("failed to stop grpc server: %v", err) + } +} + func TestModelServiceV2_GetById(t *testing.T) { setupTestDB() defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -45,7 +60,7 @@ func TestModelServiceV2_GetById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -61,8 +76,8 @@ func TestModelServiceV2_GetOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -73,7 +88,7 @@ func TestModelServiceV2_GetOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -89,8 +104,8 @@ func TestModelServiceV2_GetMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -101,7 +116,7 @@ func TestModelServiceV2_GetMany(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -118,8 +133,8 @@ func TestModelServiceV2_DeleteById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -130,7 +145,7 @@ func TestModelServiceV2_DeleteById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -148,8 +163,8 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -160,7 +175,7 @@ func TestModelServiceV2_DeleteOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -178,8 +193,8 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -190,7 +205,7 @@ func TestModelServiceV2_DeleteMany(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -208,8 +223,8 @@ func TestModelServiceV2_UpdateById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -220,7 +235,7 @@ func TestModelServiceV2_UpdateById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -238,8 +253,8 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -250,7 +265,7 @@ func TestModelServiceV2_UpdateOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -268,8 +283,8 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m1 := TestModel{ Name: "Test Name", @@ -284,7 +299,7 @@ func TestModelServiceV2_UpdateMany(t *testing.T) { require.Nil(t, err) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -302,8 +317,8 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -314,7 +329,7 @@ func TestModelServiceV2_ReplaceById(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -333,8 +348,8 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) m := TestModel{ Name: "Test Name", @@ -345,7 +360,7 @@ func TestModelServiceV2_ReplaceOne(t *testing.T) { m.SetId(id) time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -364,10 +379,10 @@ func TestModelServiceV2_InsertOne(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() @@ -388,25 +403,25 @@ func TestModelServiceV2_InsertMany(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() clientSvc := client.NewModelServiceV2[TestModel]() - models := []TestModel{ + testModels := []TestModel{ {Name: "Test Name 1"}, {Name: "Test Name 2"}, } - ids, err := clientSvc.InsertMany(models) + ids, err := clientSvc.InsertMany(testModels) require.Nil(t, err) for i, id := range ids { res, err := clientSvc.GetById(id) require.Nil(t, err) - assert.Equal(t, res.Name, models[i].Name) + assert.Equal(t, res.Name, testModels[i].Name) } } @@ -415,8 +430,8 @@ func TestModelServiceV2_Count(t *testing.T) { defer teardownTestDB() svr, err := server.NewGrpcServerV2() require.Nil(t, err) - go svr.Start() - defer svr.Stop() + go startSvr(svr) + defer stopSvr(svr) modelSvc := service.NewModelServiceV2[TestModel]() for i := 0; i < 5; i++ { @@ -427,7 +442,7 @@ func TestModelServiceV2_Count(t *testing.T) { } time.Sleep(100 * time.Millisecond) - c, err := grpc.Dial("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) + c, err := grpc.NewClient("localhost:9666", grpc.WithTransportCredentials(insecure.NewCredentials())) require.Nil(t, err) c.Connect() diff --git a/core/models/common/index_service.go b/core/models/common/index_service.go deleted file mode 100644 index ff75b92e..00000000 --- a/core/models/common/index_service.go +++ /dev/null @@ -1,145 +0,0 @@ -package common - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/db/mongo" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -func CreateIndexes() { - // artifacts - mongo.GetMongoCol(interfaces.ModelColNameArtifact).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"_col": 1}}, - {Keys: bson.M{"_del": 1}}, - {Keys: bson.M{"_tid": 1}}, - }) - - // tags - mongo.GetMongoCol(interfaces.ModelColNameTag).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"col": 1}}, - {Keys: bson.M{"name": 1}}, - }) - - // nodes - mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, // key - {Keys: bson.M{"name": 1}}, // name - {Keys: bson.M{"is_master": 1}}, // is_master - {Keys: bson.M{"status": 1}}, // status - {Keys: bson.M{"enabled": 1}}, // enabled - {Keys: bson.M{"active": 1}}, // active - }) - - // projects - mongo.GetMongoCol(interfaces.ModelColNameNode).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // spiders - mongo.GetMongoCol(interfaces.ModelColNameSpider).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - {Keys: bson.M{"type": 1}}, - {Keys: bson.M{"col_id": 1}}, - {Keys: bson.M{"project_id": 1}}, - }) - - // tasks - mongo.GetMongoCol(interfaces.ModelColNameTask).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"spider_id": 1}}, - {Keys: bson.M{"status": 1}}, - {Keys: bson.M{"node_id": 1}}, - {Keys: bson.M{"schedule_id": 1}}, - {Keys: bson.M{"type": 1}}, - {Keys: bson.M{"mode": 1}}, - {Keys: bson.M{"priority": 1}}, - {Keys: bson.M{"parent_id": 1}}, - {Keys: bson.M{"has_sub": 1}}, - {Keys: bson.M{"create_ts": -1}}, - }) - - // task stats - mongo.GetMongoCol(interfaces.ModelColNameTaskStat).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"create_ts": 1}}, - }) - - // schedules - mongo.GetMongoCol(interfaces.ModelColNameSchedule).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - {Keys: bson.M{"spider_id": 1}}, - {Keys: bson.M{"enabled": 1}}, - }) - - // users - mongo.GetMongoCol(interfaces.ModelColNameUser).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"username": 1}}, - {Keys: bson.M{"role": 1}}, - {Keys: bson.M{"email": 1}}, - }) - - // settings - mongo.GetMongoCol(interfaces.ModelColNameSetting).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, - }) - - // tokens - mongo.GetMongoCol(interfaces.ModelColNameToken).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // variables - mongo.GetMongoCol(interfaces.ModelColNameVariable).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"key": 1}}, - }) - - // data sources - mongo.GetMongoCol(interfaces.ModelColNameDataSource).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // data collections - mongo.GetMongoCol(interfaces.ModelColNameDataCollection).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"name": 1}}, - }) - - // extra values - mongo.GetMongoCol(interfaces.ModelColNameExtraValues).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.M{"oid": 1}}, - {Keys: bson.M{"m": 1}}, - {Keys: bson.M{"t": 1}}, - {Keys: bson.M{"m": 1, "t": 1}}, - {Keys: bson.M{"oid": 1, "m": 1, "t": 1}}, - }) - - // roles - mongo.GetMongoCol(interfaces.ModelColNameRole).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // user role relations - mongo.GetMongoCol(interfaces.ModelColNameUserRole).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, - {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // permissions - mongo.GetMongoCol(interfaces.ModelColNamePermission).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // role permission relations - mongo.GetMongoCol(interfaces.ModelColNameRolePermission).MustCreateIndexes([]mongo2.IndexModel{ - {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, - {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, - }) - - // cache - mongo.GetMongoCol(constants.CacheColName).MustCreateIndexes([]mongo2.IndexModel{ - { - Keys: bson.M{constants.CacheColTime: 1}, - Options: options.Index().SetExpireAfterSeconds(3600 * 24), - }, - }) -} diff --git a/core/models/common/index_service_v2.go b/core/models/common/index_service_v2.go new file mode 100644 index 00000000..2247649f --- /dev/null +++ b/core/models/common/index_service_v2.go @@ -0,0 +1,234 @@ +package common + +import ( + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/db/mongo" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +func CreateIndexesV2() { + // nodes + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, // key + {Keys: bson.M{"name": 1}}, // name + {Keys: bson.M{"is_master": 1}}, // is_master + {Keys: bson.M{"status": 1}}, // status + {Keys: bson.M{"enabled": 1}}, // enabled + {Keys: bson.M{"active": 1}}, // active + }) + + // projects + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // spiders + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"col_id": 1}}, + {Keys: bson.M{"project_id": 1}}, + }) + + // tasks + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"status": 1}}, + {Keys: bson.M{"node_id": 1}}, + {Keys: bson.M{"schedule_id": 1}}, + {Keys: bson.M{"type": 1}}, + {Keys: bson.M{"mode": 1}}, + {Keys: bson.M{"priority": 1}}, + {Keys: bson.M{"parent_id": 1}}, + {Keys: bson.M{"has_sub": 1}}, + {Keys: bson.M{"create_ts": -1}}, + }) + + // task stats + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"create_ts": 1}}, + }) + + // schedules + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + {Keys: bson.M{"spider_id": 1}}, + {Keys: bson.M{"enabled": 1}}, + }) + + // users + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"username": 1}}, + {Keys: bson.M{"role": 1}}, + {Keys: bson.M{"email": 1}}, + }) + + // settings + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // tokens + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // variables + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.VariableV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"key": 1}}, + }) + + // data sources + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // data collections + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.M{"name": 1}}, + }) + + // roles + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // user role relations + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // permissions + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // role permission relations + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{ + {Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)}, + {Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)}, + }) + + // dependencies + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"type", 1}, + {"node_id", 1}, + {"name", 1}, + }, + Options: (&options.IndexOptions{}).SetUnique(true), + }, + }) + + // dependency settings + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"type", 1}, + {"node_id", 1}, + {"name", 1}, + }, + Options: (&options.IndexOptions{}).SetUnique(true), + }, + }) + + // dependency logs + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{{"task_id", 1}}, + }, + { + Keys: bson.D{{"update_ts", 1}}, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24), + }, + }) + + // dependency tasks + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"update_ts", 1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24), + }, + }) + + // metrics + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30), + }, + { + Keys: bson.D{ + {"node_id", 1}, + }, + }, + { + Keys: bson.D{ + {"type", 1}, + }, + }, + }) + + // notification requests + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 7), + }, + { + Keys: bson.D{ + {"channel_id", 1}, + }, + }, + { + Keys: bson.D{ + {"setting_id", 1}, + }, + }, + { + Keys: bson.D{ + {"status", 1}, + }, + }, + }) + + // databases + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"data_source_id", 1}, + }, + }, + }) + + // database metrics + mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})).MustCreateIndexes([]mongo2.IndexModel{ + { + Keys: bson.D{ + {"created_ts", -1}, + }, + Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30), + }, + { + Keys: bson.D{ + {"database_id", 1}, + }, + }, + { + Keys: bson.D{ + {"type", 1}, + }, + }, + }) +} diff --git a/core/models/delegate/model_user_role_test.go b/core/models/delegate/model_user_role_test.go deleted file mode 100644 index d55b90c6..00000000 --- a/core/models/delegate/model_user_role_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package delegate_test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/delegate" - models2 "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - "go.mongodb.org/mongo-driver/bson/primitive" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") - common.CreateIndexes() -} - -func TestUserRole_Add(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - require.NotNil(t, p.Id) - - a, err := delegate.NewModelDelegate(p).GetArtifact() - require.Nil(t, err) - require.Equal(t, p.Id, a.GetId()) - require.NotNil(t, a.GetSys().GetCreateTs()) - require.NotNil(t, a.GetSys().GetUpdateTs()) -} - -func TestUserRole_Save(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{ - UserId: primitive.NewObjectID(), - RoleId: primitive.NewObjectID(), - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - uid := primitive.NewObjectID() - rid := primitive.NewObjectID() - p.UserId = uid - p.RoleId = rid - err = delegate.NewModelDelegate(p).Save() - require.Nil(t, err) - - err = mongo.GetMongoCol(interfaces.ModelColNameUserRole).FindId(p.Id).One(&p) - require.Nil(t, err) - require.Equal(t, uid, p.UserId) - require.Equal(t, rid, p.RoleId) -} - -func TestUserRole_Delete(t *testing.T) { - SetupTest(t) - - p := &models2.UserRole{} - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - - err = delegate.NewModelDelegate(p).Delete() - require.Nil(t, err) - - var a models2.Artifact - col := mongo.GetMongoCol(interfaces.ModelColNameArtifact) - err = col.FindId(p.Id).One(&a) - require.Nil(t, err) - require.NotNil(t, a.Obj) - require.True(t, a.Del) -} - -func TestUserRole_AddDuplicates(t *testing.T) { - SetupTest(t) - - uid := primitive.NewObjectID() - rid := primitive.NewObjectID() - p := &models2.UserRole{ - UserId: uid, - RoleId: rid, - } - p2 := &models2.UserRole{ - UserId: uid, - RoleId: rid, - } - - err := delegate.NewModelDelegate(p).Add() - require.Nil(t, err) - err = delegate.NewModelDelegate(p2).Add() - require.NotNil(t, err) -} diff --git a/core/models/models/data_collection_v2.go b/core/models/models/data_collection_v2.go deleted file mode 100644 index 1a3c724e..00000000 --- a/core/models/models/data_collection_v2.go +++ /dev/null @@ -1,17 +0,0 @@ -package models - -import ( - "github.com/crawlab-team/crawlab/core/entity" -) - -type DataCollectionV2 struct { - any `collection:"data_collections"` - BaseModelV2[DataCollection] `bson:",inline"` - Name string `json:"name" bson:"name"` - Fields []entity.DataField `json:"fields" bson:"fields"` - Dedup struct { - Enabled bool `json:"enabled" bson:"enabled"` - Keys []string `json:"keys" bson:"keys"` - Type string `json:"type" bson:"type"` - } `json:"dedup" bson:"dedup"` -} diff --git a/core/models/models/data_source.go b/core/models/models/data_source.go index f444266f..5693d9eb 100644 --- a/core/models/models/data_source.go +++ b/core/models/models/data_source.go @@ -11,7 +11,7 @@ type DataSource struct { Type string `json:"type" bson:"type"` Description string `json:"description" bson:"description"` Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` + Port int `json:"port" bson:"port"` Url string `json:"url" bson:"url"` Hosts []string `json:"hosts" bson:"hosts"` Database string `json:"database" bson:"database"` diff --git a/core/models/models/data_source_v2.go b/core/models/models/data_source_v2.go deleted file mode 100644 index eb81e199..00000000 --- a/core/models/models/data_source_v2.go +++ /dev/null @@ -1,20 +0,0 @@ -package models - -type DataSourceV2 struct { - any `collection:"data_sources"` - BaseModelV2[DataSource] `bson:",inline"` - Name string `json:"name" bson:"name"` - Type string `json:"type" bson:"type"` - Description string `json:"description" bson:"description"` - Host string `json:"host" bson:"host"` - Port string `json:"port" bson:"port"` - Url string `json:"url" bson:"url"` - Hosts []string `json:"hosts" bson:"hosts"` - Database string `json:"database" bson:"database"` - Username string `json:"username" bson:"username"` - Password string `json:"password,omitempty" bson:"-"` - ConnectType string `json:"connect_type" bson:"connect_type"` - Status string `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Extra map[string]string `json:"extra,omitempty" bson:"extra,omitempty"` -} diff --git a/core/models/models/dependency_setting_v2.go b/core/models/models/dependency_setting_v2.go deleted file mode 100644 index d4c241e6..00000000 --- a/core/models/models/dependency_setting_v2.go +++ /dev/null @@ -1,17 +0,0 @@ -package models - -import ( - "time" -) - -type DependencySettingV2 struct { - any `collection:"dependency_settings"` - BaseModelV2[DependencySetting] `bson:",inline"` - Key string `json:"key" bson:"key"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Cmd string `json:"cmd" bson:"cmd"` - Proxy string `json:"proxy" bson:"proxy"` - LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` -} diff --git a/core/models/models/git_v2.go b/core/models/models/git_v2.go deleted file mode 100644 index b72c130a..00000000 --- a/core/models/models/git_v2.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -type GitV2 struct { - any `collection:"gits"` - BaseModelV2[GitV2] `bson:",inline"` - Url string `json:"url" bson:"url"` - AuthType string `json:"auth_type" bson:"auth_type"` - Username string `json:"username" bson:"username"` - Password string `json:"password" bson:"password"` - CurrentBranch string `json:"current_branch" bson:"current_branch"` - AutoPull bool `json:"auto_pull" bson:"auto_pull"` -} diff --git a/core/models/models/notification_setting_v2.go b/core/models/models/notification_setting_v2.go deleted file mode 100644 index 6b565410..00000000 --- a/core/models/models/notification_setting_v2.go +++ /dev/null @@ -1,32 +0,0 @@ -package models - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type NotificationSettingV2 struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` -} - -type NotificationSettingMail struct { - Server string `json:"server" bson:"server"` - Port string `json:"port,omitempty" bson:"port,omitempty"` - User string `json:"user,omitempty" bson:"user,omitempty"` - Password string `json:"password,omitempty" bson:"password,omitempty"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` - To string `json:"to,omitempty" bson:"to,omitempty"` - Cc string `json:"cc,omitempty" bson:"cc,omitempty"` -} - -type NotificationSettingMobile struct { - Webhook string `json:"webhook" bson:"webhook"` -} diff --git a/core/models/models/test.go b/core/models/models/test.go deleted file mode 100644 index 67027d19..00000000 --- a/core/models/models/test.go +++ /dev/null @@ -1,7 +0,0 @@ -package models - -type TestModel struct { - any `collection:"testmodels"` - BaseModelV2[TestModel] `bson:",inline"` - Name string `json:"name" bson:"name"` -} diff --git a/core/models/models/base_v2.go b/core/models/models/v2/base_v2.go similarity index 76% rename from core/models/models/base_v2.go rename to core/models/models/v2/base_v2.go index cc1468e6..13a33abe 100644 --- a/core/models/models/base_v2.go +++ b/core/models/models/v2/base_v2.go @@ -7,10 +7,10 @@ import ( type BaseModelV2[T any] struct { Id primitive.ObjectID `json:"_id" bson:"_id"` - CreatedAt time.Time `json:"created_ts" bson:"created_ts"` - CreatedBy primitive.ObjectID `json:"created_by" bson:"created_by"` - UpdatedAt time.Time `json:"updated_ts" bson:"updated_ts"` - UpdatedBy primitive.ObjectID `json:"updated_by" bson:"updated_by"` + CreatedAt time.Time `json:"created_ts,omitempty" bson:"created_ts,omitempty"` + CreatedBy primitive.ObjectID `json:"created_by,omitempty" bson:"created_by,omitempty"` + UpdatedAt time.Time `json:"updated_ts,omitempty" bson:"updated_ts,omitempty"` + UpdatedBy primitive.ObjectID `json:"updated_by,omitempty" bson:"updated_by,omitempty"` } func (m *BaseModelV2[T]) GetId() primitive.ObjectID { diff --git a/core/models/models/v2/data_collection_v2.go b/core/models/models/v2/data_collection_v2.go new file mode 100644 index 00000000..8ec5e8cb --- /dev/null +++ b/core/models/models/v2/data_collection_v2.go @@ -0,0 +1,17 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/entity" +) + +type DataCollectionV2 struct { + any `collection:"data_collections"` + BaseModelV2[DataCollectionV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Fields []entity.DataField `json:"fields" bson:"fields"` + Dedup struct { + Enabled bool `json:"enabled" bson:"enabled"` + Keys []string `json:"keys" bson:"keys"` + Type string `json:"type" bson:"type"` + } `json:"dedup" bson:"dedup"` +} diff --git a/core/models/models/v2/database_metric_v2.go b/core/models/models/v2/database_metric_v2.go new file mode 100644 index 00000000..0d5b0081 --- /dev/null +++ b/core/models/models/v2/database_metric_v2.go @@ -0,0 +1,24 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DatabaseMetricV2 struct { + any `collection:"database_metrics"` + BaseModelV2[DatabaseMetricV2] `bson:",inline"` + DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + Connections int `json:"connections" bson:"connections"` + QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"` + TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"` + CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"` + ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"` + LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"` +} diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go new file mode 100644 index 00000000..e09a6fb3 --- /dev/null +++ b/core/models/models/v2/database_v2.go @@ -0,0 +1,48 @@ +package models + +import ( + "time" +) + +type DatabaseV2 struct { + any `collection:"databases"` + BaseModelV2[DatabaseV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + DataSource string `json:"data_source" bson:"data_source"` + Host string `json:"host" bson:"host"` + Port int `json:"port" bson:"port"` + URI string `json:"uri,omitempty" bson:"uri,omitempty"` + Database string `json:"database,omitempty" bson:"database,omitempty"` + Username string `json:"username,omitempty" bson:"username,omitempty"` + Password string `json:"password,omitempty" bson:"-"` + EncryptedPassword string `json:"-,omitempty" bson:"encrypted_password,omitempty"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Active bool `json:"active" bson:"active"` + ActiveAt time.Time `json:"active_ts" bson:"active_ts"` + IsDefault bool `json:"is_default" bson:"-"` + + MongoParams *struct { + AuthSource string `json:"auth_source,omitempty" bson:"auth_source,omitempty"` + AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"` + } `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"` + PostgresParams *struct { + SSLMode string `json:"ssl_mode,omitempty" bson:"ssl_mode,omitempty"` + } `json:"postgres_params,omitempty" bson:"postgres_params,omitempty"` + SnowflakeParams *struct { + Account string `json:"account,omitempty" bson:"account,omitempty"` + Schema string `json:"schema,omitempty" bson:"schema,omitempty"` + Warehouse string `json:"warehouse,omitempty" bson:"warehouse,omitempty"` + Role string `json:"role,omitempty" bson:"role,omitempty"` + } `json:"snowflake_params,omitempty" bson:"snowflake_params,omitempty"` + CassandraParams *struct { + Keyspace string `json:"keyspace,omitempty" bson:"keyspace,omitempty"` + } `json:"cassandra_params,omitempty" bson:"cassandra_params,omitempty"` + HiveParams *struct { + Auth string `json:"auth,omitempty" bson:"auth,omitempty"` + } `json:"hive_params,omitempty" bson:"hive_params,omitempty"` + RedisParams *struct { + DB int `json:"db,omitempty" bson:"db,omitempty"` + } `json:"redis_params,omitempty" bson:"redis_params,omitempty"` +} diff --git a/core/models/models/v2/dependency_log_v2.go b/core/models/models/v2/dependency_log_v2.go new file mode 100644 index 00000000..f0178c2e --- /dev/null +++ b/core/models/models/v2/dependency_log_v2.go @@ -0,0 +1,10 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyLogV2 struct { + any `collection:"dependency_logs"` + BaseModelV2[DependencyLogV2] `bson:",inline"` + TaskId primitive.ObjectID `json:"task_id" bson:"task_id"` + Content string `json:"content" bson:"content"` +} diff --git a/core/models/models/v2/dependency_setting_v2.go b/core/models/models/v2/dependency_setting_v2.go new file mode 100644 index 00000000..528c44d0 --- /dev/null +++ b/core/models/models/v2/dependency_setting_v2.go @@ -0,0 +1,17 @@ +package models + +import ( + "time" +) + +type DependencySettingV2 struct { + any `collection:"dependency_settings"` + BaseModelV2[DependencySettingV2] `bson:",inline"` + Key string `json:"key" bson:"key"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Cmd string `json:"cmd" bson:"cmd"` + Proxy string `json:"proxy" bson:"proxy"` + LastUpdateTs time.Time `json:"last_update_ts" bson:"last_update_ts"` +} diff --git a/core/models/models/v2/dependency_task_v2.go b/core/models/models/v2/dependency_task_v2.go new file mode 100644 index 00000000..c9fd40d6 --- /dev/null +++ b/core/models/models/v2/dependency_task_v2.go @@ -0,0 +1,15 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type DependencyTaskV2 struct { + any `collection:"dependency_tasks"` + BaseModelV2[DependencyTaskV2] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Action string `json:"action" bson:"action"` + DepNames []string `json:"dep_names" bson:"dep_names"` +} diff --git a/core/models/models/v2/dependency_v2.go b/core/models/models/v2/dependency_v2.go new file mode 100644 index 00000000..6d8d6039 --- /dev/null +++ b/core/models/models/v2/dependency_v2.go @@ -0,0 +1,18 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type DependencyV2 struct { + any `collection:"dependencies"` + BaseModelV2[DependencyV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + Type string `json:"type" bson:"type"` + LatestVersion string `json:"latest_version" bson:"latest_version"` + Version string `json:"version" bson:"version"` + Result entity.DependencyResult `json:"result" bson:"-"` +} diff --git a/core/models/models/environment_v2.go b/core/models/models/v2/environment_v2.go similarity index 100% rename from core/models/models/environment_v2.go rename to core/models/models/v2/environment_v2.go diff --git a/core/models/models/v2/git_v2.go b/core/models/models/v2/git_v2.go new file mode 100644 index 00000000..730c2c89 --- /dev/null +++ b/core/models/models/v2/git_v2.go @@ -0,0 +1,26 @@ +package models + +import ( + "github.com/crawlab-team/crawlab/vcs" + "time" +) + +type GitV2 struct { + any `collection:"gits"` + BaseModelV2[GitV2] `bson:",inline"` + Url string `json:"url" bson:"url"` + Name string `json:"name" bson:"name"` + AuthType string `json:"auth_type" bson:"auth_type"` + Username string `json:"username" bson:"username"` + Password string `json:"password" bson:"password"` + CurrentBranch string `json:"current_branch" bson:"current_branch"` + Status string `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Spiders []SpiderV2 `json:"spiders,omitempty" bson:"-"` + Refs []vcs.GitRef `json:"refs" bson:"refs"` + RefsUpdatedAt time.Time `json:"refs_updated_at" bson:"refs_updated_at"` + CloneLogs []string `json:"clone_logs,omitempty" bson:"clone_logs"` + + // settings + AutoPull bool `json:"auto_pull" bson:"auto_pull"` +} diff --git a/core/models/models/v2/metric_v2.go b/core/models/models/v2/metric_v2.go new file mode 100644 index 00000000..8d23dd91 --- /dev/null +++ b/core/models/models/v2/metric_v2.go @@ -0,0 +1,25 @@ +package models + +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type MetricV2 struct { + any `collection:"metrics"` + BaseModelV2[MetricV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + NodeId primitive.ObjectID `json:"node_id" bson:"node_id"` + CpuUsagePercent float32 `json:"cpu_usage_percent" bson:"cpu_usage_percent"` + TotalMemory uint64 `json:"total_memory" bson:"total_memory"` + AvailableMemory uint64 `json:"available_memory" bson:"available_memory"` + UsedMemory uint64 `json:"used_memory" bson:"used_memory"` + UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"` + TotalDisk uint64 `json:"total_disk" bson:"total_disk"` + AvailableDisk uint64 `json:"available_disk" bson:"available_disk"` + UsedDisk uint64 `json:"used_disk" bson:"used_disk"` + UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"` + DiskReadBytesRate float32 `json:"disk_read_bytes_rate" bson:"disk_read_bytes_rate"` + DiskWriteBytesRate float32 `json:"disk_write_bytes_rate" bson:"disk_write_bytes_rate"` + NetworkBytesSentRate float32 `json:"network_bytes_sent_rate" bson:"network_bytes_sent_rate"` + NetworkBytesRecvRate float32 `json:"network_bytes_recv_rate" bson:"network_bytes_recv_rate"` +} diff --git a/core/models/models/node_v2.go b/core/models/models/v2/node_v2.go similarity index 94% rename from core/models/models/node_v2.go rename to core/models/models/v2/node_v2.go index 6d5a8442..968b9bc9 100644 --- a/core/models/models/node_v2.go +++ b/core/models/models/v2/node_v2.go @@ -10,7 +10,6 @@ type NodeV2 struct { Key string `json:"key" bson:"key"` Name string `json:"name" bson:"name"` Ip string `json:"ip" bson:"ip"` - Port string `json:"port" bson:"port"` Mac string `json:"mac" bson:"mac"` Hostname string `json:"hostname" bson:"hostname"` Description string `json:"description" bson:"description"` diff --git a/core/models/models/v2/notification_alert_v2.go b/core/models/models/v2/notification_alert_v2.go new file mode 100644 index 00000000..d91e4436 --- /dev/null +++ b/core/models/models/v2/notification_alert_v2.go @@ -0,0 +1,19 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationAlertV2 struct { + any `collection:"notification_alerts"` + BaseModelV2[NotificationAlertV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + HasMetricTarget bool `json:"has_metric_target" bson:"has_metric_target"` + MetricTargetId primitive.ObjectID `json:"metric_target_id,omitempty" bson:"metric_target_id,omitempty"` + MetricName string `json:"metric_name" bson:"metric_name"` + Operator string `json:"operator" bson:"operator"` + LastingSeconds int `json:"lasting_seconds" bson:"lasting_seconds"` + TargetValue float32 `json:"target_value" bson:"target_value"` + Level string `json:"level" bson:"level"` + TemplateKey string `json:"template_key,omitempty" bson:"template_key,omitempty"` +} diff --git a/core/models/models/v2/notification_channel_v2.go b/core/models/models/v2/notification_channel_v2.go new file mode 100644 index 00000000..2e9ebed3 --- /dev/null +++ b/core/models/models/v2/notification_channel_v2.go @@ -0,0 +1,18 @@ +package models + +type NotificationChannelV2 struct { + any `collection:"notification_channels"` + BaseModelV2[NotificationChannelV2] `bson:",inline"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Provider string `json:"provider" bson:"provider"` + SMTPServer string `json:"smtp_server,omitempty" bson:"smtp_server,omitempty"` + SMTPPort int `json:"smtp_port,omitempty" bson:"smtp_port,omitempty"` + SMTPUsername string `json:"smtp_username,omitempty" bson:"smtp_username,omitempty"` + SMTPPassword string `json:"smtp_password,omitempty" bson:"smtp_password,omitempty"` + WebhookUrl string `json:"webhook_url,omitempty" bson:"webhook_url,omitempty"` + TelegramBotToken string `json:"telegram_bot_token,omitempty" bson:"telegram_bot_token,omitempty"` + TelegramChatId string `json:"telegram_chat_id,omitempty" bson:"telegram_chat_id,omitempty"` + GoogleOAuth2Json string `json:"google_oauth2_json,omitempty" bson:"google_oauth2_json,omitempty"` +} diff --git a/core/models/models/v2/notification_request_v2.go b/core/models/models/v2/notification_request_v2.go new file mode 100644 index 00000000..bbfe6023 --- /dev/null +++ b/core/models/models/v2/notification_request_v2.go @@ -0,0 +1,21 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationRequestV2 struct { + any `collection:"notification_requests"` + BaseModelV2[NotificationRequestV2] `bson:",inline"` + Status string `json:"status" bson:"status"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + Title string `json:"title" bson:"title"` + Content string `json:"content" bson:"content"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + SettingId primitive.ObjectID `json:"setting_id" bson:"setting_id"` + ChannelId primitive.ObjectID `json:"channel_id" bson:"channel_id"` + Setting *NotificationSettingV2 `json:"setting,omitempty" bson:"-"` + Channel *NotificationChannelV2 `json:"channel,omitempty" bson:"-"` +} diff --git a/core/models/models/v2/notification_setting_v2.go b/core/models/models/v2/notification_setting_v2.go new file mode 100644 index 00000000..7a932b3b --- /dev/null +++ b/core/models/models/v2/notification_setting_v2.go @@ -0,0 +1,34 @@ +package models + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationSettingV2 struct { + any `collection:"notification_settings"` + BaseModelV2[NotificationSettingV2] `bson:",inline"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template" bson:"template"` + TemplateMode string `json:"template_mode" bson:"template_mode"` + TemplateMarkdown string `json:"template_markdown,omitempty" bson:"template_markdown,omitempty"` + TemplateRichText string `json:"template_rich_text,omitempty" bson:"template_rich_text,omitempty"` + TemplateRichTextJson string `json:"template_rich_text_json,omitempty" bson:"template_rich_text_json,omitempty"` + TemplateTheme string `json:"template_theme,omitempty" bson:"template_theme,omitempty"` + + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Trigger string `json:"trigger" bson:"trigger"` + + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + UseCustomSenderEmail bool `json:"use_custom_sender_email,omitempty" bson:"use_custom_sender_email,omitempty"` + SenderName string `json:"sender_name,omitempty" bson:"sender_name,omitempty"` + MailTo []string `json:"mail_to,omitempty" bson:"mail_to,omitempty"` + MailCc []string `json:"mail_cc,omitempty" bson:"mail_cc,omitempty"` + MailBcc []string `json:"mail_bcc,omitempty" bson:"mail_bcc,omitempty"` + + ChannelIds []primitive.ObjectID `json:"channel_ids,omitempty" bson:"channel_ids,omitempty"` + Channels []NotificationChannelV2 `json:"channels,omitempty" bson:"-"` + + AlertId primitive.ObjectID `json:"alert_id,omitempty" bson:"alert_id,omitempty"` +} diff --git a/core/models/models/permission_v2.go b/core/models/models/v2/permission_v2.go similarity index 100% rename from core/models/models/permission_v2.go rename to core/models/models/v2/permission_v2.go diff --git a/core/models/models/project_v2.go b/core/models/models/v2/project_v2.go similarity index 100% rename from core/models/models/project_v2.go rename to core/models/models/v2/project_v2.go diff --git a/core/models/models/role_permission_v2.go b/core/models/models/v2/role_permission_v2.go similarity index 100% rename from core/models/models/role_permission_v2.go rename to core/models/models/v2/role_permission_v2.go diff --git a/core/models/models/role_v2.go b/core/models/models/v2/role_v2.go similarity index 100% rename from core/models/models/role_v2.go rename to core/models/models/v2/role_v2.go diff --git a/core/models/models/schedule_v2.go b/core/models/models/v2/schedule_v2.go similarity index 93% rename from core/models/models/schedule_v2.go rename to core/models/models/v2/schedule_v2.go index a52f1b92..4d8cf42c 100644 --- a/core/models/models/schedule_v2.go +++ b/core/models/models/v2/schedule_v2.go @@ -19,5 +19,4 @@ type ScheduleV2 struct { NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` Priority int `json:"priority" bson:"priority"` Enabled bool `json:"enabled" bson:"enabled"` - UserId primitive.ObjectID `json:"user_id" bson:"user_id"` } diff --git a/core/models/models/setting_v2.go b/core/models/models/v2/setting_v2.go similarity index 100% rename from core/models/models/setting_v2.go rename to core/models/models/v2/setting_v2.go diff --git a/core/models/models/spider_stat_v2.go b/core/models/models/v2/spider_stat_v2.go similarity index 100% rename from core/models/models/spider_stat_v2.go rename to core/models/models/v2/spider_stat_v2.go diff --git a/core/models/models/spider_v2.go b/core/models/models/v2/spider_v2.go similarity index 67% rename from core/models/models/spider_v2.go rename to core/models/models/v2/spider_v2.go index 4f98f6e7..b743ede2 100644 --- a/core/models/models/spider_v2.go +++ b/core/models/models/v2/spider_v2.go @@ -5,26 +5,27 @@ import ( ) type SpiderV2 struct { - any `collection:"spiders"` // spider id + any `collection:"spiders"` BaseModelV2[SpiderV2] `bson:",inline"` Name string `json:"name" bson:"name"` // spider name - Type string `json:"type" bson:"type"` // spider type - ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id - ColName string `json:"col_name,omitempty" bson:"-"` // data collection name + ColId primitive.ObjectID `json:"col_id" bson:"col_id"` // data collection id (deprecated) # TODO: remove this field in the future + ColName string `json:"col_name,omitempty" bson:"col_name"` // data collection name DataSourceId primitive.ObjectID `json:"data_source_id" bson:"data_source_id"` // data source id - DataSource *DataSourceV2 `json:"data_source,omitempty" bson:"-"` // data source + DataSource *DatabaseV2 `json:"data_source,omitempty" bson:"-"` // data source Description string `json:"description" bson:"description"` // description ProjectId primitive.ObjectID `json:"project_id" bson:"project_id"` // Project.Id Mode string `json:"mode" bson:"mode"` // default Task.Mode NodeIds []primitive.ObjectID `json:"node_ids" bson:"node_ids"` // default Task.NodeIds - Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` + GitId primitive.ObjectID `json:"git_id" bson:"git_id"` // related Git.Id + GitRootPath string `json:"git_root_path" bson:"git_root_path"` + Git *GitV2 `json:"git,omitempty" bson:"-"` + + // stats + Stat *SpiderStatV2 `json:"stat,omitempty" bson:"-"` // execution Cmd string `json:"cmd" bson:"cmd"` // execute command Param string `json:"param" bson:"param"` // default task param Priority int `json:"priority" bson:"priority"` AutoInstall bool `json:"auto_install" bson:"auto_install"` - - // settings - IncrementalSync bool `json:"incremental_sync" bson:"incremental_sync"` // whether to incrementally sync files } diff --git a/core/models/models/task_queue_item_v2.go b/core/models/models/v2/task_queue_item_v2.go similarity index 100% rename from core/models/models/task_queue_item_v2.go rename to core/models/models/v2/task_queue_item_v2.go diff --git a/core/models/models/task_stat_v2.go b/core/models/models/v2/task_stat_v2.go similarity index 81% rename from core/models/models/task_stat_v2.go rename to core/models/models/v2/task_stat_v2.go index 5456946d..65459244 100644 --- a/core/models/models/task_stat_v2.go +++ b/core/models/models/v2/task_stat_v2.go @@ -7,12 +7,10 @@ import ( type TaskStatV2 struct { any `collection:"task_stats"` BaseModelV2[TaskStatV2] `bson:",inline"` - CreateTs time.Time `json:"create_ts" bson:"create_ts,omitempty"` StartTs time.Time `json:"start_ts" bson:"start_ts,omitempty"` EndTs time.Time `json:"end_ts" bson:"end_ts,omitempty"` WaitDuration int64 `json:"wait_duration" bson:"wait_duration,omitempty"` // in millisecond RuntimeDuration int64 `json:"runtime_duration" bson:"runtime_duration,omitempty"` // in millisecond TotalDuration int64 `json:"total_duration" bson:"total_duration,omitempty"` // in millisecond ResultCount int64 `json:"result_count" bson:"result_count"` - ErrorLogCount int64 `json:"error_log_count" bson:"error_log_count"` } diff --git a/core/models/models/task_v2.go b/core/models/models/v2/task_v2.go similarity index 94% rename from core/models/models/task_v2.go rename to core/models/models/v2/task_v2.go index 3473b6db..d3f2b178 100644 --- a/core/models/models/task_v2.go +++ b/core/models/models/v2/task_v2.go @@ -2,7 +2,6 @@ package models import ( "go.mongodb.org/mongo-driver/bson/primitive" - "time" ) type TaskV2 struct { @@ -26,5 +25,4 @@ type TaskV2 struct { SubTasks []TaskV2 `json:"sub_tasks,omitempty" bson:"-"` Spider *SpiderV2 `json:"spider,omitempty" bson:"-"` UserId primitive.ObjectID `json:"-" bson:"-"` - CreateTs time.Time `json:"create_ts" bson:"create_ts"` } diff --git a/core/models/models/v2/test_v2.go b/core/models/models/v2/test_v2.go new file mode 100644 index 00000000..1468abc1 --- /dev/null +++ b/core/models/models/v2/test_v2.go @@ -0,0 +1,7 @@ +package models + +type TestModelV2 struct { + any `collection:"testmodels"` + BaseModelV2[TestModelV2] `bson:",inline"` + Name string `json:"name" bson:"name"` +} diff --git a/core/models/models/token_v2.go b/core/models/models/v2/token_v2.go similarity index 100% rename from core/models/models/token_v2.go rename to core/models/models/v2/token_v2.go diff --git a/core/models/models/user_role_v2.go b/core/models/models/v2/user_role_v2.go similarity index 100% rename from core/models/models/user_role_v2.go rename to core/models/models/v2/user_role_v2.go diff --git a/core/models/models/user_v2.go b/core/models/models/v2/user_v2.go similarity index 100% rename from core/models/models/user_v2.go rename to core/models/models/v2/user_v2.go diff --git a/core/models/models/variable_v2.go b/core/models/models/v2/variable_v2.go similarity index 100% rename from core/models/models/variable_v2.go rename to core/models/models/v2/variable_v2.go diff --git a/core/models/service/base_service_v2.go b/core/models/service/base_service_v2.go index 71dc0f15..3d03a28a 100644 --- a/core/models/service/base_service_v2.go +++ b/core/models/service/base_service_v2.go @@ -1,13 +1,16 @@ package service import ( + "context" "fmt" + "go.mongodb.org/mongo-driver/mongo/options" + "reflect" + "sync" + "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" - "sync" ) var ( @@ -30,6 +33,15 @@ func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err erro return &result, nil } +func (svc *ModelServiceV2[T]) GetByIdContext(ctx context.Context, id primitive.ObjectID) (model *T, err error) { + var result T + err = svc.col.GetCollection().FindOne(ctx, bson.M{"_id": id}).Decode(&result) + if err != nil { + return nil, err + } + return &result, nil +} + func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) { var result T err = svc.col.Find(query, options).One(&result) @@ -39,6 +51,25 @@ func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) ( return &result, nil } +func (svc *ModelServiceV2[T]) GetOneContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (model *T, err error) { + var result T + _opts := &options.FindOneOptions{} + if opts != nil { + if opts.Skip != 0 { + skipInt64 := int64(opts.Skip) + _opts.Skip = &skipInt64 + } + if opts.Sort != nil { + _opts.Sort = opts.Sort + } + } + err = svc.col.GetCollection().FindOne(ctx, query, _opts).Decode(&result) + if err != nil { + return nil, err + } + return &result, nil +} + func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) { var result []T err = svc.col.Find(query, options).All(&result) @@ -48,44 +79,115 @@ func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) return result, nil } +func (svc *ModelServiceV2[T]) GetManyContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (models []T, err error) { + var result []T + _opts := &options.FindOptions{} + if opts != nil { + if opts.Skip != 0 { + skipInt64 := int64(opts.Skip) + _opts.Skip = &skipInt64 + } + if opts.Limit != 0 { + limitInt64 := int64(opts.Limit) + _opts.Limit = &limitInt64 + } + if opts.Sort != nil { + _opts.Sort = opts.Sort + } + } + cur, err := svc.col.GetCollection().Find(ctx, query, _opts) + if err != nil { + return nil, err + } + defer cur.Close(ctx) + for cur.Next(ctx) { + var model T + if err := cur.Decode(&model); err != nil { + return nil, err + } + result = append(result, model) + } + return result, nil +} + func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) { return svc.col.DeleteId(id) } +func (svc *ModelServiceV2[T]) DeleteByIdContext(ctx context.Context, id primitive.ObjectID) (err error) { + _, err = svc.col.GetCollection().DeleteOne(ctx, bson.M{"_id": id}) + return err +} + func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) { _, err = svc.col.GetCollection().DeleteOne(svc.col.GetContext(), query) return err } +func (svc *ModelServiceV2[T]) DeleteOneContext(ctx context.Context, query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteOne(ctx, query) + return err +} + func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) { _, err = svc.col.GetCollection().DeleteMany(svc.col.GetContext(), query, nil) return err } +func (svc *ModelServiceV2[T]) DeleteManyContext(ctx context.Context, query bson.M) (err error) { + _, err = svc.col.GetCollection().DeleteMany(ctx, query, nil) + return err +} + func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) { return svc.col.UpdateId(id, update) } +func (svc *ModelServiceV2[T]) UpdateByIdContext(ctx context.Context, id primitive.ObjectID, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateOne(ctx, bson.M{"_id": id}, update) + return err +} + func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) { _, err = svc.col.GetCollection().UpdateOne(svc.col.GetContext(), query, update) return err } +func (svc *ModelServiceV2[T]) UpdateOneContext(ctx context.Context, query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateOne(ctx, query, update) + return err +} + func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) { _, err = svc.col.GetCollection().UpdateMany(svc.col.GetContext(), query, update) return err } +func (svc *ModelServiceV2[T]) UpdateManyContext(ctx context.Context, query bson.M, update bson.M) (err error) { + _, err = svc.col.GetCollection().UpdateMany(ctx, query, update) + return err +} + func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) { _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), bson.M{"_id": id}, model) return err } +func (svc *ModelServiceV2[T]) ReplaceByIdContext(ctx context.Context, id primitive.ObjectID, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(ctx, bson.M{"_id": id}, model) + return err +} + func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) { _, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), query, model) return err } +func (svc *ModelServiceV2[T]) ReplaceOneContext(ctx context.Context, query bson.M, model T) (err error) { + _, err = svc.col.GetCollection().ReplaceOne(ctx, query, model) + return err +} + func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) { m := any(&model).(interfaces.Model) if m.GetId().IsZero() { @@ -98,6 +200,18 @@ func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err err return res.InsertedID.(primitive.ObjectID), nil } +func (svc *ModelServiceV2[T]) InsertOneContext(ctx context.Context, model T) (id primitive.ObjectID, err error) { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + res, err := svc.col.GetCollection().InsertOne(ctx, m) + if err != nil { + return primitive.NilObjectID, err + } + return res.InsertedID.(primitive.ObjectID), nil +} + func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) { var _models []any for _, model := range models { @@ -117,6 +231,25 @@ func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, return ids, nil } +func (svc *ModelServiceV2[T]) InsertManyContext(ctx context.Context, models []T) (ids []primitive.ObjectID, err error) { + var _models []any + for _, model := range models { + m := any(&model).(interfaces.Model) + if m.GetId().IsZero() { + m.SetId(primitive.NewObjectID()) + } + _models = append(_models, m) + } + res, err := svc.col.GetCollection().InsertMany(ctx, _models) + if err != nil { + return nil, err + } + for _, v := range res.InsertedIDs { + ids = append(ids, v.(primitive.ObjectID)) + } + return ids, nil +} + func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) { return svc.col.Count(query) } @@ -146,7 +279,7 @@ func NewModelServiceV2[T any]() *ModelServiceV2[T] { defer mu.Unlock() if _, exists := onceMap[typeName]; !exists { - onceMap[typeName] = &sync.Once{} + onceMap[typeName] = new(sync.Once) } var instance *ModelServiceV2[T] @@ -166,7 +299,7 @@ func NewModelServiceV2WithColName[T any](colName string) *ModelServiceV2[T] { defer mu.Unlock() if _, exists := onceColNameMap[colName]; !exists { - onceColNameMap[colName] = &sync.Once{} + onceColNameMap[colName] = new(sync.Once) } var instance *ModelServiceV2[T] diff --git a/core/models/service/base_service_v2_test.go b/core/models/service/base_service_v2_test.go index 18eec697..2d23cb47 100644 --- a/core/models/service/base_service_v2_test.go +++ b/core/models/service/base_service_v2_test.go @@ -2,7 +2,7 @@ package service_test import ( "context" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/db/mongo" "github.com/spf13/viper" diff --git a/core/node/config/config_service.go b/core/node/config/config_service.go index 7e05adde..9a4cee4f 100644 --- a/core/node/config/config_service.go +++ b/core/node/config/config_service.go @@ -8,7 +8,8 @@ import ( "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" "os" - "path" + "path/filepath" + "sync" ) type Service struct { @@ -18,7 +19,7 @@ type Service struct { func (svc *Service) Init() (err error) { // check config directory path - configDirPath := path.Dir(svc.path) + configDirPath := filepath.Dir(svc.path) if !utils.Exists(configDirPath) { if err := os.MkdirAll(configDirPath, os.FileMode(0766)); err != nil { return trace.TraceError(err) @@ -55,13 +56,14 @@ func (svc *Service) Reload() (err error) { } func (svc *Service) GetBasicNodeInfo() (res interfaces.Entity) { - return &entity.NodeInfo{ + res = &entity.NodeInfo{ Key: svc.GetNodeKey(), Name: svc.GetNodeName(), IsMaster: svc.IsMaster(), AuthKey: svc.GetAuthKey(), MaxRunners: svc.GetMaxRunners(), } + return res } func (svc *Service) GetNodeKey() (res string) { @@ -92,7 +94,7 @@ func (svc *Service) SetConfigPath(path string) { svc.path = path } -func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { +func newNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { // cfg cfg := NewConfig(nil) @@ -114,17 +116,15 @@ func NewNodeConfigService() (svc2 interfaces.NodeConfigService, err error) { } var _service interfaces.NodeConfigService +var _serviceOnce = new(sync.Once) func GetNodeConfigService() interfaces.NodeConfigService { - if _service != nil { - return _service - } - - var err error - _service, err = NewNodeConfigService() - if err != nil { - panic(err) - } - + _serviceOnce.Do(func() { + var err error + _service, err = newNodeConfigService() + if err != nil { + panic(err) + } + }) return _service } diff --git a/core/node/service/master_service.go b/core/node/service/master_service.go deleted file mode 100644 index 3ade5da1..00000000 --- a/core/node/service/master_service.go +++ /dev/null @@ -1,368 +0,0 @@ -package service - -import ( - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/grpc/server" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/notification" - "github.com/crawlab-team/crawlab/core/system" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type MasterService struct { - // dependencies - modelSvc service.ModelService - cfgSvc interfaces.NodeConfigService - server interfaces.GrpcServer - schedulerSvc interfaces.TaskSchedulerService - handlerSvc interfaces.TaskHandlerService - scheduleSvc interfaces.ScheduleService - notificationSvc *notification.Service - spiderAdminSvc interfaces.SpiderAdminService - systemSvc *system.Service - - // settings - cfgPath string - address interfaces.Address - monitorInterval time.Duration - stopOnError bool -} - -func (svc *MasterService) Init() (err error) { - // do nothing - return nil -} - -func (svc *MasterService) Start() { - // create indexes - common.CreateIndexes() - - // start grpc server - if err := svc.server.Start(); err != nil { - panic(err) - } - - // register to db - if err := svc.Register(); err != nil { - panic(err) - } - - // start monitoring worker nodes - go svc.Monitor() - - // start task handler - go svc.handlerSvc.Start() - - // start task scheduler - go svc.schedulerSvc.Start() - - // start schedule service - go svc.scheduleSvc.Start() - - // start notification service - go svc.notificationSvc.Start() - - // start spider admin service - go svc.spiderAdminSvc.Start() - - // wait for quit signal - svc.Wait() - - // stop - svc.Stop() -} - -func (svc *MasterService) Wait() { - utils.DefaultWait() -} - -func (svc *MasterService) Stop() { - _ = svc.server.Stop() - log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey()) -} - -func (svc *MasterService) Monitor() { - log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey()) - for { - if err := svc.monitor(); err != nil { - trace.PrintError(err) - if svc.stopOnError { - log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey()) - svc.Stop() - return - } - } - - time.Sleep(svc.monitorInterval) - } -} - -func (svc *MasterService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *MasterService) GetConfigPath() (path string) { - return svc.cfgPath -} - -func (svc *MasterService) SetConfigPath(path string) { - svc.cfgPath = path -} - -func (svc *MasterService) GetAddress() (address interfaces.Address) { - return svc.address -} - -func (svc *MasterService) SetAddress(address interfaces.Address) { - svc.address = address -} - -func (svc *MasterService) SetMonitorInterval(duration time.Duration) { - svc.monitorInterval = duration -} - -func (svc *MasterService) Register() (err error) { - nodeKey := svc.GetConfigService().GetNodeKey() - nodeName := svc.GetConfigService().GetNodeName() - node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { - // not exists - log.Infof("master[%s] does not exist in db", nodeKey) - node := &models.Node{ - Key: nodeKey, - Name: nodeName, - MaxRunners: config.DefaultConfigOptions.MaxRunners, - IsMaster: true, - Status: constants.NodeStatusOnline, - Enabled: true, - Active: true, - ActiveTs: time.Now(), - } - if viper.GetInt("task.handler.maxRunners") > 0 { - node.MaxRunners = viper.GetInt("task.handler.maxRunners") - } - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.Add(); err != nil { - return err - } - log.Infof("added master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - return nil - } else if err == nil { - // exists - log.Infof("master[%s] exists in db", nodeKey) - nodeD := delegate.NewModelNodeDelegate(node) - if err := nodeD.UpdateStatusOnline(); err != nil { - return err - } - log.Infof("updated master[%s] in db. id: %s", nodeKey, nodeD.GetModel().GetId().Hex()) - return nil - } else { - // error - return err - } -} - -func (svc *MasterService) StopOnError() { - svc.stopOnError = true -} - -func (svc *MasterService) GetServer() (svr interfaces.GrpcServer) { - return svc.server -} - -func (svc *MasterService) monitor() (err error) { - // update master node status in db - if err := svc.updateMasterNodeStatus(); err != nil { - if err.Error() == mongo2.ErrNoDocuments.Error() { - return nil - } - return err - } - - // all worker nodes - nodes, err := svc.getAllWorkerNodes() - if err != nil { - return err - } - - // error flag - isErr := false - - // iterate all nodes - for _, n := range nodes { - // subscribe - if err := svc.subscribeNode(&n); err != nil { - isErr = true - continue - } - - // ping client - if err := svc.pingNodeClient(&n); err != nil { - isErr = true - continue - } - - // update node available runners - if err := svc.updateNodeAvailableRunners(&n); err != nil { - isErr = true - continue - } - } - - if isErr { - return trace.TraceError(errors.ErrorNodeMonitorError) - } - - return nil -} - -func (svc *MasterService) getAllWorkerNodes() (nodes []models.Node, err error) { - query := bson.M{ - "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self - "active": true, // active - } - nodes, err = svc.modelSvc.GetNodeList(query, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - return nil, nil - } - return nil, trace.TraceError(err) - } - return nodes, nil -} - -func (svc *MasterService) updateMasterNodeStatus() (err error) { - nodeKey := svc.GetConfigService().GetNodeKey() - node, err := svc.modelSvc.GetNodeByKey(nodeKey, nil) - if err != nil { - return err - } - nodeD := delegate.NewModelNodeDelegate(node) - return nodeD.UpdateStatusOnline() -} - -func (svc *MasterService) setWorkerNodeOffline(n interfaces.Node) (err error) { - return delegate.NewModelNodeDelegate(n).UpdateStatusOffline() -} - -func (svc *MasterService) subscribeNode(n interfaces.Node) (err error) { - _, err = svc.server.GetSubscribe("node:" + n.GetKey()) - if err != nil { - log.Errorf("cannot subscribe worker node[%s]: %v", n.GetKey(), err) - if err := svc.setWorkerNodeOffline(n); err != nil { - return trace.TraceError(err) - } - return trace.TraceError(err) - } - return nil -} - -func (svc *MasterService) pingNodeClient(n interfaces.Node) (err error) { - if err := svc.server.SendStreamMessage("node:"+n.GetKey(), grpc.StreamMessageCode_PING); err != nil { - log.Errorf("cannot ping worker node client[%s]: %v", n.GetKey(), err) - if err := svc.setWorkerNodeOffline(n); err != nil { - return trace.TraceError(err) - } - return trace.TraceError(err) - } - return nil -} - -func (svc *MasterService) updateNodeAvailableRunners(n interfaces.Node) (err error) { - query := bson.M{ - "node_id": n.GetId(), - "status": constants.TaskStatusRunning, - } - runningTasksCount, err := mongo.GetMongoCol(interfaces.ModelColNameTask).Count(query) - if err != nil { - return trace.TraceError(err) - } - n.SetAvailableRunners(n.GetMaxRunners() - runningTasksCount) - return delegate.NewModelDelegate(n).Save() -} - -func NewMasterService(opts ...Option) (res interfaces.NodeMasterService, err error) { - // master service - svc := &MasterService{ - cfgPath: config2.GetConfigPath(), - monitorInterval: 15 * time.Second, - stopOnError: false, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // server options - var serverOpts []server.Option - if svc.address != nil { - serverOpts = append(serverOpts, server.WithAddress(svc.address)) - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - server interfaces.GrpcServer, - schedulerSvc interfaces.TaskSchedulerService, - handlerSvc interfaces.TaskHandlerService, - scheduleSvc interfaces.ScheduleService, - spiderAdminSvc interfaces.SpiderAdminService, - ) { - svc.cfgSvc = cfgSvc - svc.modelSvc = modelSvc - svc.server = server - svc.schedulerSvc = schedulerSvc - svc.handlerSvc = handlerSvc - svc.scheduleSvc = scheduleSvc - svc.spiderAdminSvc = spiderAdminSvc - }); err != nil { - return nil, err - } - - // notification service - svc.notificationSvc = notification.GetService() - - // system service - svc.systemSvc = system.GetService() - - // init - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil -} - -func ProvideMasterService(path string, opts ...Option) func() (interfaces.NodeMasterService, error) { - // path - if path == "" || path == config2.GetConfigPath() { - if viper.GetString("config.path") != "" { - path = viper.GetString("config.path") - } else { - path = config2.GetConfigPath() - } - } - opts = append(opts, WithConfigPath(path)) - - return func() (interfaces.NodeMasterService, error) { - return NewMasterService(opts...) - } -} diff --git a/core/node/service/master_service_v2.go b/core/node/service/master_service_v2.go index 1ca7a52a..580a4f57 100644 --- a/core/node/service/master_service_v2.go +++ b/core/node/service/master_service_v2.go @@ -6,21 +6,19 @@ import ( "github.com/cenkalti/backoff/v4" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/common" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/notification" "github.com/crawlab-team/crawlab/core/schedule" - "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/system" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/task/scheduler" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -31,14 +29,12 @@ import ( type MasterServiceV2 struct { // dependencies - cfgSvc interfaces.NodeConfigService - server *server.GrpcServerV2 - schedulerSvc *scheduler.ServiceV2 - handlerSvc *handler.ServiceV2 - scheduleSvc *schedule.ServiceV2 - notificationSvc *notification.Service - spiderAdminSvc *admin.ServiceV2 - systemSvc *system.Service + cfgSvc interfaces.NodeConfigService + server *server.GrpcServerV2 + schedulerSvc *scheduler.ServiceV2 + handlerSvc *handler.ServiceV2 + scheduleSvc *schedule.ServiceV2 + systemSvc *system.ServiceV2 // settings cfgPath string @@ -54,7 +50,7 @@ func (svc *MasterServiceV2) Init() (err error) { func (svc *MasterServiceV2) Start() { // create indexes - common.CreateIndexes() + common.CreateIndexesV2() // start grpc server if err := svc.server.Start(); err != nil { @@ -78,12 +74,6 @@ func (svc *MasterServiceV2) Start() { // start schedule service go svc.scheduleSvc.Start() - // start notification service - go svc.notificationSvc.Start() - - // start spider admin service - go svc.spiderAdminSvc.Start() - // wait for quit signal svc.Wait() @@ -102,8 +92,14 @@ func (svc *MasterServiceV2) Stop() { func (svc *MasterServiceV2) Monitor() { log.Infof("master[%s] monitoring started", svc.GetConfigService().GetNodeKey()) + + // ticker + ticker := time.NewTicker(svc.monitorInterval) + for { - if err := svc.monitor(); err != nil { + // monitor + err := svc.monitor() + if err != nil { trace.PrintError(err) if svc.stopOnError { log.Errorf("master[%s] monitor error, now stopping...", svc.GetConfigService().GetNodeKey()) @@ -112,7 +108,8 @@ func (svc *MasterServiceV2) Monitor() { } } - time.Sleep(svc.monitorInterval) + // wait + <-ticker.C } } @@ -143,11 +140,11 @@ func (svc *MasterServiceV2) SetMonitorInterval(duration time.Duration) { func (svc *MasterServiceV2) Register() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() nodeName := svc.GetConfigService().GetNodeName() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil && err.Error() == mongo2.ErrNoDocuments.Error() { // not exists log.Infof("master[%s] does not exist in db", nodeKey) - node := models.NodeV2{ + node := models2.NodeV2{ Key: nodeKey, Name: nodeName, MaxRunners: config.DefaultConfigOptions.MaxRunners, @@ -159,7 +156,7 @@ func (svc *MasterServiceV2) Register() (err error) { } node.SetCreated(primitive.NilObjectID) node.SetUpdated(primitive.NilObjectID) - id, err := service.NewModelServiceV2[models.NodeV2]().InsertOne(node) + id, err := service.NewModelServiceV2[models2.NodeV2]().InsertOne(node) if err != nil { return err } @@ -171,7 +168,7 @@ func (svc *MasterServiceV2) Register() (err error) { node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } @@ -210,12 +207,13 @@ func (svc *MasterServiceV2) monitor() (err error) { wg := sync.WaitGroup{} wg.Add(len(workerNodes)) for _, n := range workerNodes { - go func(n *models.NodeV2) { + go func(n *models2.NodeV2) { + defer wg.Done() + // subscribe ok := svc.subscribeNode(n) if !ok { go svc.setWorkerNodeOffline(n) - wg.Done() return } @@ -223,19 +221,14 @@ func (svc *MasterServiceV2) monitor() (err error) { ok = svc.pingNodeClient(n) if !ok { go svc.setWorkerNodeOffline(n) - wg.Done() return } // update node available runners if err := svc.updateNodeAvailableRunners(n); err != nil { trace.PrintError(err) - wg.Done() return } - - // done - wg.Done() }(&n) } @@ -244,12 +237,12 @@ func (svc *MasterServiceV2) monitor() (err error) { return nil } -func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err error) { +func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models2.NodeV2, err error) { query := bson.M{ "key": bson.M{"$ne": svc.cfgSvc.GetNodeKey()}, // not self "active": true, // active } - nodes, err = service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err = service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { if errors.Is(err, mongo2.ErrNoDocuments) { return nil, nil @@ -261,32 +254,44 @@ func (svc *MasterServiceV2) getAllWorkerNodes() (nodes []models.NodeV2, err erro func (svc *MasterServiceV2) updateMasterNodeStatus() (err error) { nodeKey := svc.GetConfigService().GetNodeKey() - node, err := service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + node, err := service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) if err != nil { return err } + oldStatus := node.Status + node.Status = constants.NodeStatusOnline node.Active = true node.ActiveAt = time.Now() - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + newStatus := node.Status + + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } + + if utils.IsPro() { + if oldStatus != newStatus { + go svc.sendNotification(node) + } + } + return nil } -func (svc *MasterServiceV2) setWorkerNodeOffline(node *models.NodeV2) { +func (svc *MasterServiceV2) setWorkerNodeOffline(node *models2.NodeV2) { node.Status = constants.NodeStatusOffline node.Active = false err := backoff.Retry(func() error { - return service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + return service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) }, backoff.WithMaxRetries(backoff.NewConstantBackOff(1*time.Second), 3)) if err != nil { trace.PrintError(err) } + svc.sendNotification(node) } -func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) subscribeNode(n *models2.NodeV2) (ok bool) { _, err := svc.server.GetSubscribe("node:" + n.Key) if err != nil { log.Errorf("cannot subscribe worker node[%s]: %v", n.Key, err) @@ -295,7 +300,7 @@ func (svc *MasterServiceV2) subscribeNode(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { +func (svc *MasterServiceV2) pingNodeClient(n *models2.NodeV2) (ok bool) { if err := svc.server.SendStreamMessage("node:"+n.Key, grpc.StreamMessageCode_PING); err != nil { log.Errorf("cannot ping worker node client[%s]: %v", n.Key, err) return false @@ -303,24 +308,31 @@ func (svc *MasterServiceV2) pingNodeClient(n *models.NodeV2) (ok bool) { return true } -func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models.NodeV2) (err error) { +func (svc *MasterServiceV2) updateNodeAvailableRunners(node *models2.NodeV2) (err error) { query := bson.M{ "node_id": node.Id, "status": constants.TaskStatusRunning, } - runningTasksCount, err := service.NewModelServiceV2[models.TaskV2]().Count(query) + runningTasksCount, err := service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { return trace.TraceError(err) } node.AvailableRunners = node.MaxRunners - runningTasksCount - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(node.Id, *node) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(node.Id, *node) if err != nil { return err } return nil } -func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { +func (svc *MasterServiceV2) sendNotification(node *models2.NodeV2) { + if !utils.IsPro() { + return + } + go notification.GetNotificationServiceV2().SendNodeNotification(node) +} + +func newMasterServiceV2() (res *MasterServiceV2, err error) { // master service svc := &MasterServiceV2{ cfgPath: config2.GetConfigPath(), @@ -328,20 +340,8 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { stopOnError: false, } - // server options - var serverOpts []server.Option - if svc.address != nil { - serverOpts = append(serverOpts, server.WithAddress(svc.address)) - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - ) { - svc.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } + // node config service + svc.cfgSvc = config.GetNodeConfigService() // grpc server svc.server, err = server.GetGrpcServerV2() @@ -367,17 +367,8 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { return nil, err } - // notification service - svc.notificationSvc = notification.GetService() - - // spider admin service - svc.spiderAdminSvc, err = admin.GetSpiderAdminServiceV2() - if err != nil { - return nil, err - } - // system service - svc.systemSvc = system.GetService() + svc.systemSvc = system.GetSystemServiceV2() // init if err := svc.Init(); err != nil { @@ -386,3 +377,17 @@ func NewMasterServiceV2() (res interfaces.NodeMasterService, err error) { return svc, nil } + +var masterServiceV2 *MasterServiceV2 +var masterServiceV2Once = new(sync.Once) + +func GetMasterServiceV2() (res *MasterServiceV2, err error) { + masterServiceV2Once.Do(func() { + masterServiceV2, err = newMasterServiceV2() + if err != nil { + log.Errorf("failed to get master service: %v", err) + } + }) + return masterServiceV2, err + +} diff --git a/core/node/service/worker_service.go b/core/node/service/worker_service.go deleted file mode 100644 index cf91625a..00000000 --- a/core/node/service/worker_service.go +++ /dev/null @@ -1,238 +0,0 @@ -package service - -import ( - "context" - "encoding/json" - "github.com/apex/log" - config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/grpc/client" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/spf13/viper" - "time" -) - -type WorkerService struct { - // dependencies - cfgSvc interfaces.NodeConfigService - client interfaces.GrpcClient - handlerSvc interfaces.TaskHandlerService - - // settings - cfgPath string - address interfaces.Address - heartbeatInterval time.Duration - - // internals - n interfaces.Node - s grpc.NodeService_SubscribeClient -} - -func (svc *WorkerService) Init() (err error) { - // do nothing - return nil -} - -func (svc *WorkerService) Start() { - // start grpc client - if err := svc.client.Start(); err != nil { - panic(err) - } - - // register to master - svc.Register() - - // start receiving stream messages - go svc.Recv() - - // start sending heartbeat to master - go svc.ReportStatus() - - // start handler - go svc.handlerSvc.Start() - - // wait for quit signal - svc.Wait() - - // stop - svc.Stop() -} - -func (svc *WorkerService) Wait() { - utils.DefaultWait() -} - -func (svc *WorkerService) Stop() { - _ = svc.client.Stop() - log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey()) -} - -func (svc *WorkerService) Register() { - ctx, cancel := svc.client.Context() - defer cancel() - req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) - res, err := svc.client.GetNodeClient().Register(ctx, req) - if err != nil { - panic(err) - } - if err := json.Unmarshal(res.Data, svc.n); err != nil { - panic(err) - } - log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) - return -} - -func (svc *WorkerService) Recv() { - msgCh := svc.client.GetMessageChannel() - for { - // return if client is closed - if svc.client.IsClosed() { - return - } - - // receive message from channel - msg := <-msgCh - - // handle message - if err := svc.handleStreamMessage(msg); err != nil { - continue - } - } -} - -func (svc *WorkerService) handleStreamMessage(msg *grpc.StreamMessage) (err error) { - log.Debugf("[WorkerService] handle msg: %v", msg) - switch msg.Code { - case grpc.StreamMessageCode_PING: - if _, err := svc.client.GetNodeClient().SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { - return trace.TraceError(err) - } - case grpc.StreamMessageCode_RUN_TASK: - var t models.Task - if err := json.Unmarshal(msg.Data, &t); err != nil { - return trace.TraceError(err) - } - if err := svc.handlerSvc.Run(t.Id); err != nil { - return trace.TraceError(err) - } - case grpc.StreamMessageCode_CANCEL_TASK: - var t models.Task - if err := json.Unmarshal(msg.Data, &t); err != nil { - return trace.TraceError(err) - } - if err := svc.handlerSvc.Cancel(t.Id); err != nil { - return trace.TraceError(err) - } - } - - return nil -} - -func (svc *WorkerService) ReportStatus() { - for { - // return if client is closed - if svc.client.IsClosed() { - return - } - - // report status - svc.reportStatus() - - // sleep - time.Sleep(svc.heartbeatInterval) - } -} - -func (svc *WorkerService) GetConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *WorkerService) GetConfigPath() (path string) { - return svc.cfgPath -} - -func (svc *WorkerService) SetConfigPath(path string) { - svc.cfgPath = path -} - -func (svc *WorkerService) GetAddress() (address interfaces.Address) { - return svc.address -} - -func (svc *WorkerService) SetAddress(address interfaces.Address) { - svc.address = address -} - -func (svc *WorkerService) SetHeartbeatInterval(duration time.Duration) { - svc.heartbeatInterval = duration -} - -func (svc *WorkerService) reportStatus() { - ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) - defer cancel() - _, err := svc.client.GetNodeClient().SendHeartbeat(ctx, &grpc.Request{ - NodeKey: svc.cfgSvc.GetNodeKey(), - }) - if err != nil { - trace.PrintError(err) - } -} - -func NewWorkerService(opts ...Option) (res *WorkerService, err error) { - svc := &WorkerService{ - cfgPath: config2.GetConfigPath(), - heartbeatInterval: 15 * time.Second, - n: &models.Node{}, - } - - // apply options - for _, opt := range opts { - opt(svc) - } - - // dependency options - var clientOpts []client.Option - if svc.address != nil { - clientOpts = append(clientOpts, client.WithAddress(svc.address)) - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - client interfaces.GrpcClient, - taskHandlerSvc interfaces.TaskHandlerService, - ) { - svc.cfgSvc = cfgSvc - svc.client = client - svc.handlerSvc = taskHandlerSvc - }); err != nil { - return nil, err - } - - // init - if err := svc.Init(); err != nil { - return nil, err - } - - return svc, nil -} - -func ProvideWorkerService(path string, opts ...Option) func() (interfaces.NodeWorkerService, error) { - // path - if path == "" || path == config2.GetConfigPath() { - if viper.GetString("config.path") != "" { - path = viper.GetString("config.path") - } else { - path = config2.GetConfigPath() - } - } - opts = append(opts, WithConfigPath(path)) - - return func() (interfaces.NodeWorkerService, error) { - return NewWorkerService(opts...) - } -} diff --git a/core/node/service/worker_service_v2.go b/core/node/service/worker_service_v2.go index 0134c54d..a041877c 100644 --- a/core/node/service/worker_service_v2.go +++ b/core/node/service/worker_service_v2.go @@ -5,14 +5,18 @@ import ( "encoding/json" "github.com/apex/log" config2 "github.com/crawlab-team/crawlab/core/config" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" + client2 "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" + "go.mongodb.org/mongo-driver/bson" + "sync" "time" ) @@ -28,7 +32,7 @@ type WorkerServiceV2 struct { heartbeatInterval time.Duration // internals - n interfaces.Node + n *models2.NodeV2 s grpc.NodeService_SubscribeClient } @@ -74,15 +78,21 @@ func (svc *WorkerServiceV2) Stop() { func (svc *WorkerServiceV2) Register() { ctx, cancel := svc.client.Context() defer cancel() - req := svc.client.NewRequest(svc.GetConfigService().GetBasicNodeInfo()) - res, err := svc.client.NodeClient.Register(ctx, req) + _, err := svc.client.NodeClient.Register(ctx, &grpc.NodeServiceRegisterRequest{ + Key: svc.cfgSvc.GetNodeKey(), + Name: svc.cfgSvc.GetNodeName(), + IsMaster: svc.cfgSvc.IsMaster(), + AuthKey: svc.cfgSvc.GetAuthKey(), + MaxRunners: int32(svc.cfgSvc.GetMaxRunners()), + }) if err != nil { panic(err) } - if err := json.Unmarshal(res.Data, svc.n); err != nil { + svc.n, err = client2.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": svc.GetConfigService().GetNodeKey()}, nil) + if err != nil { panic(err) } - log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.GetId().Hex()) + log.Infof("worker[%s] registered to master. id: %s", svc.GetConfigService().GetNodeKey(), svc.n.Id.Hex()) return } @@ -108,7 +118,10 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er log.Debugf("[WorkerServiceV2] handle msg: %v", msg) switch msg.Code { case grpc.StreamMessageCode_PING: - if _, err := svc.client.NodeClient.SendHeartbeat(context.Background(), svc.client.NewRequest(svc.cfgSvc.GetBasicNodeInfo())); err != nil { + _, err := svc.client.NodeClient.SendHeartbeat(context.Background(), &grpc.NodeServiceSendHeartbeatRequest{ + Key: svc.cfgSvc.GetNodeKey(), + }) + if err != nil { return trace.TraceError(err) } case grpc.StreamMessageCode_RUN_TASK: @@ -133,9 +146,11 @@ func (svc *WorkerServiceV2) handleStreamMessage(msg *grpc.StreamMessage) (err er } func (svc *WorkerServiceV2) ReportStatus() { + ticker := time.NewTicker(svc.heartbeatInterval) for { // return if client is closed if svc.client.IsClosed() { + ticker.Stop() return } @@ -143,7 +158,7 @@ func (svc *WorkerServiceV2) ReportStatus() { svc.reportStatus() // sleep - time.Sleep(svc.heartbeatInterval) + <-ticker.C } } @@ -174,19 +189,21 @@ func (svc *WorkerServiceV2) SetHeartbeatInterval(duration time.Duration) { func (svc *WorkerServiceV2) reportStatus() { ctx, cancel := context.WithTimeout(context.Background(), svc.heartbeatInterval) defer cancel() - _, err := svc.client.NodeClient.SendHeartbeat(ctx, &grpc.Request{ - NodeKey: svc.cfgSvc.GetNodeKey(), + _, err := svc.client.NodeClient.SendHeartbeat(ctx, &grpc.NodeServiceSendHeartbeatRequest{ + Key: svc.cfgSvc.GetNodeKey(), }) if err != nil { trace.PrintError(err) } } -func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { +var workerServiceV2 *WorkerServiceV2 +var workerServiceV2Once = new(sync.Once) + +func newWorkerServiceV2() (res *WorkerServiceV2, err error) { svc := &WorkerServiceV2{ cfgPath: config2.GetConfigPath(), heartbeatInterval: 15 * time.Second, - n: &models.Node{}, } // dependency options @@ -195,20 +212,11 @@ func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { clientOpts = append(clientOpts, client.WithAddress(svc.address)) } - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - ) { - svc.cfgSvc = cfgSvc - }); err != nil { - return nil, err - } + // node config service + svc.cfgSvc = nodeconfig.GetNodeConfigService() // grpc client - svc.client, err = client.NewGrpcClientV2() - if err != nil { - return nil, err - } + svc.client = client.GetGrpcClientV2() // handler service svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() @@ -217,9 +225,20 @@ func NewWorkerServiceV2() (res *WorkerServiceV2, err error) { } // init - if err := svc.Init(); err != nil { + err = svc.Init() + if err != nil { return nil, err } return svc, nil } + +func GetWorkerServiceV2() (res *WorkerServiceV2, err error) { + workerServiceV2Once.Do(func() { + workerServiceV2, err = newWorkerServiceV2() + if err != nil { + log.Errorf("failed to get worker service: %v", err) + } + }) + return workerServiceV2, err +} diff --git a/core/node/test/base.go b/core/node/test/base.go deleted file mode 100644 index 223c6a92..00000000 --- a/core/node/test/base.go +++ /dev/null @@ -1,206 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/interfaces" - service2 "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/node/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/spf13/viper" - "go.uber.org/dig" - "io/ioutil" - "os" - "path" - "testing" - "time" -) - -func init() { - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -var T *Test - -type Test struct { - DefaultSvc interfaces.NodeMasterService - MasterSvc interfaces.NodeMasterService - WorkerSvc interfaces.NodeWorkerService - MasterSvcMonitor interfaces.NodeMasterService - WorkerSvcMonitor interfaces.NodeWorkerService - ModelSvc service2.ModelService -} - -func NewTest() (res *Test, err error) { - // test - t := &Test{} - - // recreate config directory path - _ = os.RemoveAll(viper.GetString("metadata")) - _ = os.MkdirAll(viper.GetString("metadata"), os.FileMode(0766)) - - // master config and settings - masterNodeConfigName := "config-master.json" - masterNodeConfigPath := path.Join(viper.GetString("metadata"), masterNodeConfigName) - if err := ioutil.WriteFile(masterNodeConfigPath, []byte("{\"key\":\"master\",\"is_master\":true}"), os.FileMode(0766)); err != nil { - return nil, err - } - masterHost := "0.0.0.0" - masterPort := "9667" - - // worker config and settings - workerNodeConfigName := "config-worker.json" - workerNodeConfigPath := path.Join(viper.GetString("metadata"), workerNodeConfigName) - if err = ioutil.WriteFile(workerNodeConfigPath, []byte("{\"key\":\"worker\",\"is_master\":false}"), os.FileMode(0766)); err != nil { - return nil, err - } - workerHost := "localhost" - workerPort := masterPort - - // master for monitor config and settings - masterNodeMonitorConfigName := "config-master-monitor.json" - masterNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), masterNodeMonitorConfigName) - if err := ioutil.WriteFile(masterNodeMonitorConfigPath, []byte("{\"key\":\"master-monitor\",\"is_master\":true}"), os.FileMode(0766)); err != nil { - return nil, err - } - masterMonitorHost := masterHost - masterMonitorPort := "9668" - - // worker for monitor config and settings - workerNodeMonitorConfigName := "config-worker-monitor.json" - workerNodeMonitorConfigPath := path.Join(viper.GetString("metadata"), workerNodeMonitorConfigName) - if err := ioutil.WriteFile(workerNodeMonitorConfigPath, []byte("{\"key\":\"worker-monitor\",\"is_master\":false}"), os.FileMode(0766)); err != nil { - return nil, err - } - workerMonitorHost := workerHost - workerMonitorPort := masterMonitorPort - - // dependency injection - c := dig.New() - if err := c.Provide(service.ProvideMasterService( - masterNodeConfigPath, - service.WithMonitorInterval(3*time.Second), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: masterHost, - Port: masterPort, - })), - )); err != nil { - return nil, err - } - if err := c.Provide(service.ProvideWorkerService( - workerNodeConfigPath, - service.WithHeartbeatInterval(1*time.Second), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: workerHost, - Port: workerPort, - })), - )); err != nil { - return nil, err - } - if err := c.Provide(service2.NewService); err != nil { - return nil, err - } - if err := c.Invoke(func(masterSvc interfaces.NodeMasterService, workerSvc interfaces.NodeWorkerService, modelSvc service2.ModelService) { - t.MasterSvc = masterSvc - t.WorkerSvc = workerSvc - t.ModelSvc = modelSvc - }); err != nil { - return nil, err - } - - // default service - t.DefaultSvc, err = service.NewMasterService() - if err != nil { - return nil, err - } - - // master and worker for monitor - t.MasterSvcMonitor, err = service.NewMasterService( - service.WithConfigPath(masterNodeMonitorConfigPath), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: masterMonitorHost, - Port: masterMonitorPort, - })), - service.WithMonitorInterval(3*time.Second), - service.WithStopOnError(), - ) - if err != nil { - return nil, err - } - t.WorkerSvcMonitor, err = service.NewWorkerService( - service.WithConfigPath(workerNodeMonitorConfigPath), - service.WithAddress(entity.NewAddress(&entity.AddressOptions{ - Host: workerMonitorHost, - Port: workerMonitorPort, - })), - service.WithHeartbeatInterval(1*time.Second), - service.WithStopOnError(), - ) - if err != nil { - return nil, err - } - - // removed all data in db - _ = t.ModelSvc.DropAll() - - // visualize dependencies - if err := utils.VisualizeContainer(c); err != nil { - return nil, err - } - - return t, nil -} - -func (t *Test) Setup(t2 *testing.T) { - if err := t.ModelSvc.DropAll(); err != nil { - panic(err) - } - _ = os.RemoveAll(viper.GetString("metadata")) - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - if err := t.ModelSvc.DropAll(); err != nil { - panic(err) - } - _ = os.RemoveAll(viper.GetString("metadata")) -} - -func (t *Test) StartMasterWorker() { - startMasterWorker() -} - -func (t *Test) StopMasterWorker() { - stopMasterWorker() -} - -func startMasterWorker() { - go T.MasterSvc.Start() - time.Sleep(1 * time.Second) - go T.WorkerSvc.Start() - time.Sleep(1 * time.Second) -} - -func stopMasterWorker() { - go T.MasterSvc.Stop() - time.Sleep(1 * time.Second) - go T.WorkerSvc.Stop() - time.Sleep(1 * time.Second) -} - -func startMasterWorkerMonitor() { - go T.MasterSvcMonitor.Start() - time.Sleep(1 * time.Second) - go T.WorkerSvcMonitor.Start() - time.Sleep(1 * time.Second) -} - -func stopMasterWorkerMonitor() { - go T.MasterSvcMonitor.Stop() - time.Sleep(1 * time.Second) - go T.WorkerSvcMonitor.Stop() - time.Sleep(1 * time.Second) -} diff --git a/core/node/test/base_test.go b/core/node/test/base_test.go deleted file mode 100644 index 70832586..00000000 --- a/core/node/test/base_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestNodeServices_Master_Worker(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - startMasterWorker() - - // validate master - masterNodeKey := T.MasterSvc.GetConfigService().GetNodeKey() - masterNode, err := T.ModelSvc.GetNodeByKey(masterNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, masterNode.Status) - require.Equal(t, masterNodeKey, masterNode.Key) - require.True(t, masterNode.IsMaster) - - // validate worker - workerNodeKey := T.WorkerSvc.GetConfigService().GetNodeKey() - workerNode, err := T.ModelSvc.GetNodeByKey(workerNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, workerNode.Status) - require.Equal(t, workerNodeKey, workerNode.Key) - require.False(t, workerNode.IsMaster) - - stopMasterWorker() -} - -func TestNodeServices_Default(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - - go T.DefaultSvc.Start() - time.Sleep(1 * time.Second) - - // validate default - defaultNodeKey := T.DefaultSvc.GetConfigService().GetNodeKey() - defaultNode, err := T.ModelSvc.GetNodeByKey(defaultNodeKey, nil) - require.Nil(t, err) - require.Equal(t, constants.NodeStatusOnline, defaultNode.Status) - require.Equal(t, defaultNodeKey, defaultNode.Key) - require.True(t, defaultNode.IsMaster) - - T.DefaultSvc.Stop() - time.Sleep(1 * time.Second) -} - -func TestNodeServices_Monitor(t *testing.T) { - T, _ = NewTest() - T.Setup(t) - startMasterWorkerMonitor() - time.Sleep(3 * time.Second) - - // stop worker - T.WorkerSvcMonitor.Stop() - time.Sleep(5 * time.Second) - - // validate - require.True(t, T.MasterSvcMonitor.GetServer().IsStopped()) - - stopMasterWorkerMonitor() -} diff --git a/core/notification/base_test.go b/core/notification/base_test.go deleted file mode 100644 index 5b29810d..00000000 --- a/core/notification/base_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package notification - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/gavv/httpexpect/v2" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson/primitive" - "net/http/httptest" - "testing" -) - -func init() { - viper.Set("mongo.db", "crawlab_test") - var err error - T, err = NewTest() - if err != nil { - panic(err) - } -} - -type Test struct { - svc *Service - svr *httptest.Server - - // test data - TestNode interfaces.Node - TestSpider interfaces.Spider - TestTask interfaces.Task - TestTaskStat interfaces.TaskStat -} - -func (t *Test) Setup(t2 *testing.T) { - _ = t.svc.Start() - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.svc.Stop() -} - -func (t *Test) NewExpect(t2 *testing.T) (e *httpexpect.Expect) { - e = httpexpect.New(t2, t.svr.URL) - return e -} - -var T *Test - -func NewTest() (res *Test, err error) { - // test - t := &Test{ - svc: NewService(), - } - - // test node - t.TestNode = &models.Node{Id: primitive.NewObjectID(), Name: "test-node"} - _ = delegate.NewModelDelegate(t.TestNode).Add() - - // test spider - t.TestSpider = &models.Spider{Id: primitive.NewObjectID(), Name: "test-spider"} - _ = delegate.NewModelDelegate(t.TestSpider).Add() - - // test task - t.TestTask = &models.Task{Id: primitive.NewObjectID(), SpiderId: t.TestSpider.GetId(), NodeId: t.TestNode.GetId()} - _ = delegate.NewModelDelegate(t.TestTask).Add() - - // test task stat - t.TestTaskStat = &models.TaskStat{Id: t.TestTask.GetId()} - _ = delegate.NewModelDelegate(t.TestTaskStat).Add() - - return t, nil -} diff --git a/core/notification/constants.go b/core/notification/constants.go index d3f56b72..817f476f 100644 --- a/core/notification/constants.go +++ b/core/notification/constants.go @@ -1,10 +1,32 @@ package notification const ( - TypeMail = "mail" - TypeMobile = "mobile" + TypeMail = "mail" + TypeIM = "im" ) const ( - SettingsColName = "notification_settings" + ChannelMailProviderGmail = "gmail" + ChannelMailProviderOutlook = "outlook" + ChannelMailProviderYahoo = "yahoo" + ChannelMailProviderICloud = "icloud" + ChannelMailProviderAol = "aol" + ChannelMailProviderZoho = "zoho" + ChannelMailProviderQQ = "qq" + ChannelMailProvider163 = "163" + ChannelMailProviderExmail = "exmail" + + ChannelIMProviderSlack = "slack" // https://api.slack.com/messaging/webhooks + ChannelIMProviderTelegram = "telegram" // https://core.telegram.org/bots/api + ChannelIMProviderDiscord = "discord" // https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks + ChannelIMProviderMSTeams = "ms_teams" // https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cjavascript + ChannelIMProviderWechatWork = "wechat_work" // https://developer.work.weixin.qq.com/document/path/91770 + ChannelIMProviderDingtalk = "dingtalk" // https://open.dingtalk.com/document/orgapp/custom-robot-access + ChannelIMProviderLark = "lark" // https://www.larksuite.com/hc/en-US/articles/099698615114-use-webhook-triggers +) + +const ( + StatusSending = "sending" + StatusSuccess = "success" + StatusError = "error" ) diff --git a/core/notification/entity.go b/core/notification/entity.go new file mode 100644 index 00000000..cf7933ee --- /dev/null +++ b/core/notification/entity.go @@ -0,0 +1,13 @@ +package notification + +import "github.com/crawlab-team/crawlab/core/models/models/v2" + +type VariableData struct { + Task *models.TaskV2 `json:"task"` + TaskStat *models.TaskStatV2 `json:"task_stat"` + Spider *models.SpiderV2 `json:"spider"` + Node *models.NodeV2 `json:"node"` + Schedule *models.ScheduleV2 `json:"schedule"` + Alert *models.NotificationAlertV2 `json:"alert"` + Metric *models.MetricV2 `json:"metric"` +} diff --git a/core/notification/im.go b/core/notification/im.go new file mode 100644 index 00000000..181b8b89 --- /dev/null +++ b/core/notification/im.go @@ -0,0 +1,353 @@ +package notification + +import ( + "errors" + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/trace" + "github.com/imroc/req" + "regexp" + "strings" +) + +type ResBody struct { + ErrCode int `json:"errcode"` + ErrMsg string `json:"errmsg"` +} + +func SendIMNotification(ch *models.NotificationChannelV2, title, content string) error { + switch ch.Provider { + case ChannelIMProviderLark: + return sendIMLark(ch, title, content) + case ChannelIMProviderDingtalk: + return sendIMDingTalk(ch, title, content) + case ChannelIMProviderWechatWork: + return sendIMWechatWork(ch, title, content) + case ChannelIMProviderSlack: + return sendIMSlack(ch, title, content) + case ChannelIMProviderTelegram: + return sendIMTelegram(ch, title, content) + case ChannelIMProviderDiscord: + return sendIMDiscord(ch, title, content) + case ChannelIMProviderMSTeams: + return sendIMMSTeams(ch, title, content) + } + + // request header + header := req.Header{ + "Content-Type": "application/json; charset=utf-8", + } + + // request data + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": content, + "content": content, + }, + "at": req.Param{ + "atMobiles": []string{}, + "isAtAll": false, + }, + "text": content, + } + if strings.Contains(strings.ToLower(ch.WebhookUrl), "feishu") { + data = req.Param{ + "msg_type": "text", + "content": req.Param{ + "text": content, + }, + } + } + + // perform request + res, err := req.Post(ch.WebhookUrl, header, req.BodyJSON(&data)) + if err != nil { + return trace.TraceError(err) + } + + // parse response + var resBody ResBody + if err := res.ToJSON(&resBody); err != nil { + return trace.TraceError(err) + } + + // validate response code + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + + return nil +} + +func getIMRequestHeader() req.Header { + return req.Header{ + "Content-Type": "application/json; charset=utf-8", + } +} + +func performIMRequest(webhookUrl string, data req.Param) (res *req.Resp, err error) { + // perform request + res, err = req.Post(webhookUrl, getIMRequestHeader(), req.BodyJSON(&data)) + if err != nil { + log.Errorf("IM request error: %v", err) + return nil, err + } + + // get response + response := res.Response() + + // check status code + if response.StatusCode >= 400 { + log.Errorf("IM response status code: %d", res.Response().StatusCode) + return nil, errors.New(fmt.Sprintf("IM error response %d: %s", response.StatusCode, res.String())) + } + + return res, nil +} + +func performIMRequestWithJson[T any](webhookUrl string, data req.Param) (resBody T, err error) { + res, err := performIMRequest(webhookUrl, data) + if err != nil { + return resBody, err + } + + // parse response + if err := res.ToJSON(&resBody); err != nil { + log.Warnf("Parsing IM response error: %v", err) + resText, err := res.ToString() + if err != nil { + log.Warnf("Converting response to string error: %v", err) + return resBody, err + } + log.Infof("IM response: %s", resText) + return resBody, nil + } + + return resBody, nil +} + +func convertMarkdownToSlack(markdown string) string { + // Convert bold text + reBold := regexp.MustCompile(`\*\*(.*?)\*\*`) + slack := reBold.ReplaceAllString(markdown, `*$1*`) + + // Convert italic text + reItalic := regexp.MustCompile(`\*(.*?)\*`) + slack = reItalic.ReplaceAllString(slack, `_$1_`) + + // Convert links + reLink := regexp.MustCompile(`\[(.*?)\]\((.*?)\)`) + slack = reLink.ReplaceAllString(slack, `<$2|$1>`) + + // Convert inline code + reInlineCode := regexp.MustCompile("`(.*?)`") + slack = reInlineCode.ReplaceAllString(slack, "`$1`") + + // Convert unordered list + slack = strings.ReplaceAll(slack, "- ", "• ") + + // Convert ordered list + reOrderedList := regexp.MustCompile(`^\d+\. `) + slack = reOrderedList.ReplaceAllStringFunc(slack, func(s string) string { + return strings.Replace(s, ". ", ". ", 1) + }) + + // Convert blockquote + reBlockquote := regexp.MustCompile(`^> (.*)`) + slack = reBlockquote.ReplaceAllString(slack, `> $1`) + + return slack +} + +func convertMarkdownToTelegram(markdownText string) string { + // Combined regex to handle bold and italic + re := regexp.MustCompile(`(?m)(\*\*)(.*)(\*\*)|(__)(.*)(__)|(\*)(.*)(\*)|(_)(.*)(_)`) + markdownText = re.ReplaceAllStringFunc(markdownText, func(match string) string { + groups := re.FindStringSubmatch(match) + if groups[1] != "" || groups[4] != "" { + // Handle bold + return "*" + match[2:len(match)-2] + "*" + } else if groups[6] != "" || groups[9] != "" { + // Handle italic + return "_" + match[1:len(match)-1] + "_" + } else { + // No match + return match + } + }) + + // Convert unordered list + re = regexp.MustCompile(`(?m)^- (.*)`) + markdownText = re.ReplaceAllString(markdownText, "• $1") + + // Escape characters + escapeChars := []string{"#", "-", "."} + for _, c := range escapeChars { + markdownText = strings.ReplaceAll(markdownText, c, "\\"+c) + } + + return markdownText +} + +func sendIMLark(ch *models.NotificationChannelV2, title, content string) error { + data := req.Param{ + "msg_type": "interactive", + "card": req.Param{ + "header": req.Param{ + "title": req.Param{ + "tag": "plain_text", + "content": title, + }, + }, + "elements": []req.Param{ + { + "tag": "markdown", + "content": content, + }, + }, + }, + } + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} + +func sendIMDingTalk(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": fmt.Sprintf("# %s\n\n%s", title, content), + }, + } + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} + +func sendIMWechatWork(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "content": fmt.Sprintf("# %s\n\n%s", title, content), + }, + } + resBody, err := performIMRequestWithJson[ResBody](ch.WebhookUrl, data) + if err != nil { + return err + } + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + return nil +} + +func sendIMSlack(ch *models.NotificationChannelV2, title, content string) error { + data := req.Param{ + "blocks": []req.Param{ + {"type": "header", "text": req.Param{"type": "plain_text", "text": title}}, + {"type": "section", "text": req.Param{"type": "mrkdwn", "text": convertMarkdownToSlack(content)}}, + }, + } + _, err := performIMRequest(ch.WebhookUrl, data) + if err != nil { + return err + } + return nil +} + +func sendIMTelegram(ch *models.NotificationChannelV2, title string, content string) error { + type ResBody struct { + Ok bool `json:"ok"` + Description string `json:"description"` + } + + // chat id + chatId := ch.TelegramChatId + if !strings.HasPrefix("@", ch.TelegramChatId) { + chatId = fmt.Sprintf("@%s", ch.TelegramChatId) + } + + // webhook url + webhookUrl := fmt.Sprintf("https://api.telegram.org/bot%s/sendMessage", ch.TelegramBotToken) + + // original Markdown text + text := fmt.Sprintf("**%s**\n\n%s", title, content) + + // convert to Telegram MarkdownV2 + text = convertMarkdownToTelegram(text) + + // request data + data := req.Param{ + "chat_id": chatId, + "text": text, + "parse_mode": "MarkdownV2", + } + + // perform request + _, err := performIMRequest(webhookUrl, data) + if err != nil { + return err + } + return nil +} + +func sendIMDiscord(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "embeds": []req.Param{ + { + "title": title, + "description": content, + }, + }, + } + _, err := performIMRequest(ch.WebhookUrl, data) + if err != nil { + return err + } + return nil +} + +func sendIMMSTeams(ch *models.NotificationChannelV2, title string, content string) error { + data := req.Param{ + "type": "message", + "attachments": []req.Param{{ + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": nil, + "content": req.Param{ + "$schema": "https://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.2", + "body": []req.Param{ + { + "type": "TextBlock", + "text": fmt.Sprintf("**%s**", title), + "size": "Large", + }, + { + "type": "TextBlock", + "text": content, + }, + }, + }, + }}, + } + _, err := performIMRequest(ch.WebhookUrl, data) + if err != nil { + return err + } + return nil +} diff --git a/core/notification/mail.go b/core/notification/mail.go index 905a5446..49cd5bd0 100644 --- a/core/notification/mail.go +++ b/core/notification/mail.go @@ -2,86 +2,72 @@ package notification import ( "errors" + "github.com/PuerkitoBio/goquery" "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/matcornic/hermes/v2" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/trace" "gopkg.in/gomail.v2" "net/mail" - "runtime/debug" - "strconv" + "regexp" "strings" ) -func SendMail(m *models.NotificationSettingMail, to, cc, title, content string) error { - // theme - theme := new(MailThemeFlat) - - // hermes instance - h := hermes.Hermes{ - Theme: theme, - Product: hermes.Product{ - Logo: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMzAwIiBoZWlnaHQ9IjMwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KICAgIDxnIGZpbGw9Im5vbmUiPgogICAgICAgIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjE1MCIgcj0iMTMwIiBmaWxsPSJub25lIiBzdHJva2Utd2lkdGg9IjQwIiBzdHJva2U9IiM0MDllZmYiPgogICAgICAgIDwvY2lyY2xlPgogICAgICAgIDxjaXJjbGUgY3g9IjE1MCIgY3k9IjE1MCIgcj0iMTEwIiBmaWxsPSJ3aGl0ZSI+CiAgICAgICAgPC9jaXJjbGU+CiAgICAgICAgPGNpcmNsZSBjeD0iMTUwIiBjeT0iMTUwIiByPSI3MCIgZmlsbD0iIzQwOWVmZiI+CiAgICAgICAgPC9jaXJjbGU+CiAgICAgICAgPHBhdGggZD0iCiAgICAgICAgICAgIE0gMTUwLDE1MAogICAgICAgICAgICBMIDI4MCwyMjUKICAgICAgICAgICAgQSAxNTAsMTUwIDkwIDAgMCAyODAsNzUKICAgICAgICAgICAgIiBmaWxsPSIjNDA5ZWZmIj4KICAgICAgICA8L3BhdGg+CiAgICA8L2c+Cjwvc3ZnPgo=", - Name: "Crawlab", - Copyright: "© 2024 Crawlab-Team", - }, +func SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, to, cc, bcc []string, title, content string) error { + // sender email + senderEmail := ch.SMTPUsername + if s.UseCustomSenderEmail { + senderEmail = s.SenderEmail } // config - port, _ := strconv.Atoi(m.Port) - password := m.Password // test password: ALWVDPRHBEXOENXD - SMTPUser := m.User smtpConfig := smtpAuthentication{ - Server: m.Server, - Port: port, - SenderEmail: m.SenderEmail, - SenderIdentity: m.SenderIdentity, - SMTPPassword: password, - SMTPUser: SMTPUser, + Server: ch.SMTPServer, + Port: ch.SMTPPort, + SenderIdentity: s.SenderName, + SenderEmail: senderEmail, + SMTPUser: ch.SMTPUsername, + SMTPPassword: ch.SMTPPassword, } + options := sendOptions{ + Subject: title, To: to, Cc: cc, - Subject: title, + Bcc: bcc, } - // add style - content += theme.GetStyle() - - // markdown - markdown := hermes.Markdown(content + GetFooter()) - - // email instance - email := hermes.Email{ - Body: hermes.Body{ - Signature: "Happy Crawling ☺", - FreeMarkdown: markdown, - }, + // convert html to text + text := content + if isHtml(text) { + text = convertHtmlToText(text) } - // generate html - html, err := h.GenerateHTML(email) + // apply theme + if isHtml(content) { + content = GetTheme() + content + } + + switch ch.Provider { + case ChannelMailProviderGmail: + return sendMailGmail(ch, smtpConfig, options, content, text) + default: + return sendMail(smtpConfig, options, content, text) + } +} + +func isHtml(content string) bool { + regex := regexp.MustCompile(`(?i)<\s*(html|head|body|div|span|p|a|img|table|tr|td|th|tbody|thead|tfoot|ul|ol|li|dl|dt|dd|form|input|textarea|button|select|option|optgroup|fieldset|legend|label|iframe|embed|object|param|video|audio|source|canvas|svg|math|style|link|script|meta|base|title|br|hr|b|strong|i|em|u|s|strike|del|ins|mark|small|sub|sup|big|pre|code|q|blockquote|abbr|address|bdo|cite|dfn|kbd|var|samp|ruby|rt|rp|time|progress|meter|output|area|map)`) + return regex.MatchString(content) +} + +func convertHtmlToText(content string) string { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return err + log.Errorf("failed to convert html to text: %v", err) + trace.PrintError(err) + return "" } - - // generate text - text, err := h.GeneratePlainText(email) - if err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return err - } - - // send the email - if err := send(smtpConfig, options, html, text); err != nil { - log.Errorf(err.Error()) - debug.PrintStack() - return err - } - - return nil + return doc.Text() } type smtpAuthentication struct { @@ -95,13 +81,42 @@ type smtpAuthentication struct { // sendOptions are options for sending an email type sendOptions struct { - To string Subject string - Cc string + To []string + Cc []string + Bcc []string +} + +func getMailMessage(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) (m *gomail.Message, err error) { + if len(options.To) == 0 { + return nil, errors.New("no receiver emails configured") + } + + // from + from := mail.Address{ + Name: smtpConfig.SenderIdentity, + Address: smtpConfig.SenderEmail, + } + + // message + m = gomail.NewMessage() + m.SetHeader("From", from.String()) + m.SetHeader("To", options.To...) + m.SetHeader("Subject", options.Subject) + if len(options.Cc) > 0 { + m.SetHeader("Cc", options.Cc...) + } + if len(options.Bcc) > 0 { + m.SetHeader("Bcc", options.Bcc...) + } + m.SetBody("text/plain", txtBody) + m.AddAlternative("text/html", htmlBody) + + return m, nil } // send email -func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { +func sendMail(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, txtBody string) error { if smtpConfig.Server == "" { return errors.New("SMTP server config is empty") } @@ -114,65 +129,13 @@ func send(smtpConfig smtpAuthentication, options sendOptions, htmlBody string, t return errors.New("SMTP user is empty") } - if smtpConfig.SenderIdentity == "" { - return errors.New("SMTP sender identity is empty") + m, err := getMailMessage(smtpConfig, options, htmlBody, txtBody) + if err != nil { + return err } - if smtpConfig.SenderEmail == "" { - return errors.New("SMTP sender email is empty") - } - - if options.To == "" { - return errors.New("no receiver emails configured") - } - - from := mail.Address{ - Name: smtpConfig.SenderIdentity, - Address: smtpConfig.SenderEmail, - } - - var toList []string - if strings.Contains(options.To, ";") { - toList = strings.Split(options.To, ";") - // trim space - for i, to := range toList { - toList[i] = strings.TrimSpace(to) - } - } else { - toList = []string{options.To} - } - - m := gomail.NewMessage() - m.SetHeader("From", from.String()) - m.SetHeader("To", getRecipientList(options.To)...) - m.SetHeader("Subject", options.Subject) - if options.Cc != "" { - m.SetHeader("Cc", getRecipientList(options.Cc)...) - } - - m.SetBody("text/plain", txtBody) - m.AddAlternative("text/html", htmlBody) - + // dialer d := gomail.NewDialer(smtpConfig.Server, smtpConfig.Port, smtpConfig.SMTPUser, smtpConfig.SMTPPassword) return d.DialAndSend(m) } - -func getRecipientList(value string) (values []string) { - if strings.Contains(value, ";") { - values = strings.Split(value, ";") - // trim space - for i, v := range values { - values[i] = strings.TrimSpace(v) - } - } else { - values = []string{value} - } - return values -} - -func GetFooter() string { - return ` -[Github](https://github.com/crawlab-team/crawlab) | [Documentation](http://docs.crawlab.cn) | [Docker](https://hub.docker.com/r/tikazyq/crawlab) -` -} diff --git a/core/notification/mail_gmail.go b/core/notification/mail_gmail.go new file mode 100644 index 00000000..aebf4e5f --- /dev/null +++ b/core/notification/mail_gmail.go @@ -0,0 +1,61 @@ +package notification + +import ( + "context" + "encoding/base64" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/trace" + "golang.org/x/oauth2/google" + "google.golang.org/api/gmail/v1" + "strings" +) + +func sendMailGmail(ch *models.NotificationChannelV2, smtpConfig smtpAuthentication, options sendOptions, htmlBody, txtBody string) error { + // 读取服务账户 JSON 密钥 + b := []byte(ch.GoogleOAuth2Json) + + // 使用服务账户 JSON 密钥文件创建 JWT 配置 + config, err := google.JWTConfigFromJSON(b, gmail.GmailSendScope) + if err != nil { + log.Errorf("Unable to parse service account key file to config: %v", err) + return trace.TraceError(err) + } + + // 使用服务账户的电子邮件地址来模拟用户 + config.Subject = ch.SMTPUsername + + // 创建 Gmail 服务 + client := config.Client(context.Background()) + srv, err := gmail.New(client) + if err != nil { + log.Errorf("Unable to create Gmail client: %v", err) + return trace.TraceError(err) + } + + // 创建 MIME 邮件 + m, err := getMailMessage(smtpConfig, options, htmlBody, txtBody) + if err != nil { + return err + } + + var buf strings.Builder + if _, err := m.WriteTo(&buf); err != nil { + log.Errorf("Unable to write message: %v", err) + return trace.TraceError(err) + } + + // 将邮件内容进行 base64 编码 + gmsg := &gmail.Message{ + Raw: base64.URLEncoding.EncodeToString([]byte(buf.String())), + } + + // 发送邮件 + _, err = srv.Users.Messages.Send("me", gmsg).Do() + if err != nil { + log.Errorf("Unable to send email: %v", err) + return trace.TraceError(err) + } + + return nil +} diff --git a/core/notification/mail_theme.go b/core/notification/mail_theme.go deleted file mode 100644 index 9266ddd8..00000000 --- a/core/notification/mail_theme.go +++ /dev/null @@ -1,8 +0,0 @@ -package notification - -import "github.com/matcornic/hermes/v2" - -type MailTheme interface { - hermes.Theme - GetStyle() string -} diff --git a/core/notification/mail_theme_flat.go b/core/notification/mail_theme_flat.go deleted file mode 100644 index 6edb480f..00000000 --- a/core/notification/mail_theme_flat.go +++ /dev/null @@ -1,287 +0,0 @@ -package notification - -// MailThemeFlat is a theme -type MailThemeFlat struct{} - -// Name returns the name of the flat theme -func (dt *MailThemeFlat) Name() string { - return "flat" -} - -// HTMLTemplate returns a Golang template that will generate an HTML email. -func (dt *MailThemeFlat) HTMLTemplate() string { - return ` - - - - - - - - - - - - -
- - - - - - - - - - - - - - -
- - -` -} - -// PlainTextTemplate returns a Golang template that will generate an plain text email. -func (dt *MailThemeFlat) PlainTextTemplate() string { - return `{{ with .Email.Body.Intros }} - {{ range $line := . }} -

{{ $line }}

- {{ end }} -{{ end }} -{{ if (ne .Email.Body.FreeMarkdown "") }} - {{ .Email.Body.FreeMarkdown.ToHTML }} -{{ else }} - {{ with .Email.Body.Dictionary }} - - {{ end }} - {{ with .Email.Body.Table }} - {{ $data := .Data }} - {{ $columns := .Columns }} - {{ if gt (len $data) 0 }} - - - {{ $col := index $data 0 }} - {{ range $entry := $col }} - - {{ end }} - - {{ range $row := $data }} - - {{ range $cell := $row }} - - {{ end }} - - {{ end }} -
{{ $entry.Key }}
- {{ $cell.Value }} -
- {{ end }} - {{ end }} - {{ with .Email.Body.Actions }} - {{ range $action := . }} -

{{ $action.Instructions }} {{ $action.Button.Link }}

- {{ end }} - {{ end }} -{{ end }} -{{ with .Email.Body.Outros }} - {{ range $line := . }} -

{{ $line }}

- {{ end }} -{{ end }} -

{{.Email.Body.Signature}},
{{.Hermes.Product.Name}} - {{.Hermes.Product.Link}}

- -

{{.Hermes.Product.Copyright}}

-` -} - -func (dt *MailThemeFlat) GetStyle() string { - return ` - -` -} diff --git a/core/notification/mobile.go b/core/notification/mobile.go deleted file mode 100644 index 90a23870..00000000 --- a/core/notification/mobile.go +++ /dev/null @@ -1,62 +0,0 @@ -package notification - -import ( - "errors" - "github.com/crawlab-team/crawlab/trace" - "github.com/imroc/req" - "strings" -) - -type ResBody struct { - ErrCode int `json:"errcode"` - ErrMsg string `json:"errmsg"` -} - -func SendMobileNotification(webhook string, title string, content string) error { - // request header - header := req.Header{ - "Content-Type": "application/json; charset=utf-8", - } - - // request data - data := req.Param{ - "msgtype": "markdown", - "markdown": req.Param{ - "title": title, - "text": content, - "content": content, - }, - "at": req.Param{ - "atMobiles": []string{}, - "isAtAll": false, - }, - "text": content, - } - if strings.Contains(strings.ToLower(webhook), "feishu") { - data = req.Param{ - "msg_type": "text", - "content": req.Param{ - "text": content, - }, - } - } - - // perform request - res, err := req.Post(webhook, header, req.BodyJSON(&data)) - if err != nil { - return trace.TraceError(err) - } - - // parse response - var resBody ResBody - if err := res.ToJSON(&resBody); err != nil { - return trace.TraceError(err) - } - - // validate response code - if resBody.ErrCode != 0 { - return errors.New(resBody.ErrMsg) - } - - return nil -} diff --git a/core/notification/models.go b/core/notification/models.go deleted file mode 100644 index efb55448..00000000 --- a/core/notification/models.go +++ /dev/null @@ -1,32 +0,0 @@ -package notification - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type NotificationSetting struct { - Id primitive.ObjectID `json:"_id" bson:"_id"` - Type string `json:"type" bson:"type"` - Name string `json:"name" bson:"name"` - Description string `json:"description" bson:"description"` - Enabled bool `json:"enabled" bson:"enabled"` - Global bool `json:"global" bson:"global"` - Title string `json:"title,omitempty" bson:"title,omitempty"` - Template string `json:"template,omitempty" bson:"template,omitempty"` - TaskTrigger string `json:"task_trigger" bson:"task_trigger"` - Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` - Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` -} - -type NotificationSettingMail struct { - Server string `json:"server" bson:"server"` - Port string `json:"port,omitempty" bson:"port,omitempty"` - User string `json:"user,omitempty" bson:"user,omitempty"` - Password string `json:"password,omitempty" bson:"password,omitempty"` - SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` - SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` - To string `json:"to,omitempty" bson:"to,omitempty"` - Cc string `json:"cc,omitempty" bson:"cc,omitempty"` -} - -type NotificationSettingMobile struct { - Webhook string `json:"webhook" bson:"webhook"` -} diff --git a/core/notification/oauth2_gmail.go b/core/notification/oauth2_gmail.go new file mode 100644 index 00000000..b6c1bbed --- /dev/null +++ b/core/notification/oauth2_gmail.go @@ -0,0 +1,47 @@ +package notification + +import ( + "context" + "github.com/apex/log" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "net/smtp" + "time" +) + +// 获取服务账户的OAuth2配置 +func getGmailOAuth2Token(oauth2Json string) (token *oauth2.Token, err error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // 读取服务账户 JSON 密钥 + b := []byte(oauth2Json) + + // 使用服务账户 JSON 密钥文件创建 JWT 配置 + config, err := google.JWTConfigFromJSON(b, "https://mail.google.com/") + if err != nil { + log.Errorf("Unable to parse service account key file to config: %v", err) + return nil, err + } + + // 使用服务账户的电子邮件和访问令牌 + token, err = config.TokenSource(ctx).Token() + if err != nil { + log.Errorf("Unable to generate token: %v", err) + return nil, err + } + return token, nil +} + +// GmailOAuth2Auth 自定义OAuth2认证 +type GmailOAuth2Auth struct { + username, accessToken string +} + +func (a *GmailOAuth2Auth) Start(_ *smtp.ServerInfo) (string, []byte, error) { + return "XOAUTH2", []byte("user=" + a.username + "\x01auth=Bearer " + a.accessToken + "\x01\x01"), nil +} + +func (a *GmailOAuth2Auth) Next(_ []byte, _ bool) ([]byte, error) { + return nil, nil +} diff --git a/core/notification/payload.go b/core/notification/payload.go deleted file mode 100644 index 7337ebe0..00000000 --- a/core/notification/payload.go +++ /dev/null @@ -1,8 +0,0 @@ -package notification - -import "go.mongodb.org/mongo-driver/bson/primitive" - -type SendPayload struct { - TaskId primitive.ObjectID `json:"task_id"` - Data string `json:"data"` -} diff --git a/core/notification/service.go b/core/notification/service.go deleted file mode 100644 index 37a7f93d..00000000 --- a/core/notification/service.go +++ /dev/null @@ -1,395 +0,0 @@ -package notification - -import ( - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" - parser "github.com/crawlab-team/crawlab/template-parser" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" -) - -type Service struct { - col *mongo2.Col // notification settings - modelSvc service.ModelService -} - -func (svc *Service) Init() (err error) { - if !utils.IsPro() { - return nil - } - - return nil -} - -func (svc *Service) Start() (err error) { - // initialize data - if err := svc.initData(); err != nil { - return err - } - - return nil -} - -func (svc *Service) Stop() (err error) { - return nil -} - -func (svc *Service) initData() (err error) { - total, err := svc.col.Count(nil) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []NotificationSetting{ - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "任务通知（邮件）", - Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -|键|值| -|:-:|:--| -|任务状态|{{$.status}}| -|任务优先级|{{$.priority}}| -|任务模式|{{$.mode}}| -|执行命令|{{$.cmd}}| -|执行参数|{{$.param}}| -|错误信息|{{$.error}}| -|节点|{{$.node.name}}| -|爬虫|{{$.spider.name}}| -|项目|{{$.spider.project.name}}| -|定时任务|{{$.schedule.name}}| -|结果数|{{$.:task_stat.result_count}}| -|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| -|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| -|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| -|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "Task Change (Mail)", - Description: "This is the default mail notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -|Key|Value| -|:-:|:--| -|Task Status|{{$.status}}| -|Task Priority|{{$.priority}}| -|Task Mode|{{$.mode}}| -|Task Command|{{$.cmd}}| -|Task Params|{{$.param}}| -|Error Message|{{$.error}}| -|Node|{{$.node.name}}| -|Spider|{{$.spider.name}}| -|Project|{{$.spider.project.name}}| -|Schedule|{{$.schedule.name}}| -|Result Count|{{$.:task_stat.result_count}}| -|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| -|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| -|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| -|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "任务通知（移动端）", - Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -- **任务状态**: {{$.status}} -- **任务优先级**: {{$.priority}} -- **任务模式**: {{$.mode}} -- **执行命令**: {{$.cmd}} -- **执行参数**: {{$.param}} -- **错误信息**: {{$.error}} -- **节点**: {{$.node.name}} -- **爬虫**: {{$.spider.name}} -- **项目**: {{$.spider.project.name}} -- **定时任务**: {{$.schedule.name}} -- **结果数**: {{$.:task_stat.result_count}} -- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} -- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} -- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: NotificationSettingMobile{}, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "Task Change (Mobile)", - Description: "This is the default mobile notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskError, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -- **Task Status**: {{$.status}} -- **Task Priority**: {{$.priority}} -- **Task Mode**: {{$.mode}} -- **Task Command**: {{$.cmd}} -- **Task Params**: {{$.param}} -- **Error Message**: {{$.error}} -- **Node**: {{$.node.name}} -- **Spider**: {{$.spider.name}} -- **Project**: {{$.spider.project.name}} -- **Schedule**: {{$.schedule.name}} -- **Result Count**: {{$.:task_stat.result_count}} -- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} -- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} -- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: NotificationSettingMobile{}, - }, - } - var data []interface{} - for _, s := range settings { - data = append(data, s) - } - _, err = svc.col.InsertMany(data) - if err != nil { - return err - } - return nil -} - -func (svc *Service) Send(s NotificationSetting, entity bson.M) (err error) { - switch s.Type { - case TypeMail: - return svc.SendMail(s, entity) - case TypeMobile: - return svc.SendMobile(s, entity) - } - return nil -} - -func (svc *Service) SendMail(s NotificationSetting, entity bson.M) (err error) { - // to - to, err := parser.Parse(s.Mail.To, entity) - if err != nil { - log.Warnf("parsing 'to' error: %v", err) - } - if to == "" { - return nil - } - - // cc - cc, err := parser.Parse(s.Mail.Cc, entity) - if err != nil { - log.Warnf("parsing 'cc' error: %v", err) - } - - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } - - // send mail - if err := SendMail(&models.NotificationSettingMail{ - Server: s.Mail.Server, - Port: s.Mail.Port, - User: s.Mail.User, - Password: s.Mail.Password, - SenderEmail: s.Mail.SenderEmail, - SenderIdentity: s.Mail.SenderIdentity, - To: s.Mail.To, - Cc: s.Mail.Cc, - }, to, cc, title, content); err != nil { - return err - } - - return nil -} - -func (svc *Service) SendMobile(s NotificationSetting, entity bson.M) (err error) { - // webhook - webhook, err := parser.Parse(s.Mobile.Webhook, entity) - if err != nil { - log.Warnf("parsing 'webhook' error: %v", err) - } - if webhook == "" { - return nil - } - - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } - - // send - if err := SendMobileNotification(webhook, title, content); err != nil { - return err - } - - return nil -} - -func (svc *Service) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []NotificationSetting, total int, err error) { - // options - var options *mongo2.FindOptions - if pagination != nil || sort != nil { - options = new(mongo2.FindOptions) - if pagination != nil { - options.Skip = pagination.Size * (pagination.Page - 1) - options.Limit = pagination.Size - } - if sort != nil { - options.Sort = sort - } - } - - // get list - var list []NotificationSetting - if err := svc.col.Find(query, options).All(&list); err != nil { - if err.Error() == mongo.ErrNoDocuments.Error() { - return nil, 0, nil - } else { - return nil, 0, err - } - } - - // total count - total, err = svc.col.Count(query) - if err != nil { - return nil, 0, err - } - - return list, total, nil -} - -func (svc *Service) GetSetting(id primitive.ObjectID) (res *NotificationSetting, err error) { - var s NotificationSetting - if err := svc.col.FindId(id).One(&s); err != nil { - return nil, err - } - return &s, nil -} - -func (svc *Service) PosSetting(s *NotificationSetting) (err error) { - s.Id = primitive.NewObjectID() - if _, err := svc.col.Insert(s); err != nil { - return err - } - return nil -} - -func (svc *Service) PutSetting(id primitive.ObjectID, s NotificationSetting) (err error) { - if err := svc.col.ReplaceId(id, s); err != nil { - return err - } - - return nil -} - -func (svc *Service) DeleteSetting(id primitive.ObjectID) (err error) { - if err := svc.col.DeleteId(id); err != nil { - return err - } - - return nil -} - -func (svc *Service) EnableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(true)(id) -} - -func (svc *Service) DisableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(false)(id) -} - -func (svc *Service) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { - return func(id primitive.ObjectID) (err error) { - var s NotificationSetting - if err := svc.col.FindId(id).One(&s); err != nil { - return err - } - s.Enabled = value - if err := svc.col.ReplaceId(id, s); err != nil { - return err - } - return nil - } -} - -func NewService() *Service { - // service - svc := &Service{ - col: mongo2.GetMongoCol(SettingsColName), - } - - // model service - modelSvc, err := service.GetService() - if err != nil { - panic(err) - } - svc.modelSvc = modelSvc - - if err := svc.Init(); err != nil { - panic(err) - } - - return svc -} - -var _service *Service - -func GetService() *Service { - if _service == nil { - _service = NewService() - } - return _service -} diff --git a/core/notification/service_test.go b/core/notification/service_test.go deleted file mode 100644 index 20e09478..00000000 --- a/core/notification/service_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package notification - -import ( - "net/http" - "testing" - "time" -) - -func TestService_sendMobile(t *testing.T) { - T.Setup(t) - e := T.NewExpect(t) - time.Sleep(1 * time.Second) - - data := map[string]interface{}{ - "task_id": T.TestTask.GetId().Hex(), - } - e.POST("/send/mobile").WithJSON(data). - Expect().Status(http.StatusOK) -} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go index 6db6a01a..377e2e68 100644 --- a/core/notification/service_v2.go +++ b/core/notification/service_v2.go @@ -1,362 +1,523 @@ package notification import ( + "fmt" "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" - mongo2 "github.com/crawlab-team/crawlab/db/mongo" - parser "github.com/crawlab-team/crawlab/template-parser" + "github.com/crawlab-team/crawlab/trace" + "github.com/gomarkdown/markdown" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" + "regexp" + "strings" + "sync" + "time" ) type ServiceV2 struct { } -func (svc *ServiceV2) Start() (err error) { - // initialize data - if err := svc.initData(); err != nil { - return err - } +func (svc *ServiceV2) Send(s *models.NotificationSettingV2, args ...any) { + title := s.Title - return nil + wg := sync.WaitGroup{} + wg.Add(len(s.ChannelIds)) + for _, chId := range s.ChannelIds { + go func(chId primitive.ObjectID) { + defer wg.Done() + ch, err := service.NewModelServiceV2[models.NotificationChannelV2]().GetById(chId) + if err != nil { + log.Errorf("[NotificationServiceV2] get channel error: %v", err) + return + } + content := svc.getContent(s, ch, args...) + switch ch.Type { + case TypeMail: + svc.SendMail(s, ch, title, content) + case TypeIM: + svc.SendIM(s, ch, title, content) + } + }(chId) + } + wg.Wait() } -func (svc *ServiceV2) Stop() (err error) { - return nil -} +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { + mailTo := s.MailTo + mailCc := s.MailCc + mailBcc := s.MailBcc -func (svc *ServiceV2) initData() (err error) { - total, err := service.NewModelServiceV2[models.NotificationSettingV2]().Count(nil) - if err != nil { - return err - } - if total > 0 { - return nil - } - - // data to initialize - settings := []models.NotificationSettingV2{ - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "任务通知（邮件）", - Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 爬虫任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -|键|值| -|:-:|:--| -|任务状态|{{$.status}}| -|任务优先级|{{$.priority}}| -|任务模式|{{$.mode}}| -|执行命令|{{$.cmd}}| -|执行参数|{{$.param}}| -|错误信息|{{$.error}}| -|节点|{{$.node.name}}| -|爬虫|{{$.spider.name}}| -|项目|{{$.spider.project.name}}| -|定时任务|{{$.schedule.name}}| -|结果数|{{$.:task_stat.result_count}}| -|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| -|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| -|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| -|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: models.NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMail, - Enabled: true, - Name: "Task Change (Mail)", - Description: "This is the default mail notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -|Key|Value| -|:-:|:--| -|Task Status|{{$.status}}| -|Task Priority|{{$.priority}}| -|Task Mode|{{$.mode}}| -|Task Command|{{$.cmd}}| -|Task Params|{{$.param}}| -|Error Message|{{$.error}}| -|Node|{{$.node.name}}| -|Spider|{{$.spider.name}}| -|Project|{{$.spider.project.name}}| -|Schedule|{{$.schedule.name}}| -|Result Count|{{$.:task_stat.result_count}}| -|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| -|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| -|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| -|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| -`, - Mail: models.NotificationSettingMail{ - Server: "smtp.163.com", - Port: "465", - To: "{{$.user[create].email}}", - }, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "任务通知（移动端）", - Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", - TaskTrigger: constants.NotificationTriggerTaskFinish, - Title: "[Crawlab] 任务更新: {{$.status}}", - Template: `尊敬的 {{$.user.username}}, - -请查看下面的任务数据。 - -- **任务状态**: {{$.status}} -- **任务优先级**: {{$.priority}} -- **任务模式**: {{$.mode}} -- **执行命令**: {{$.cmd}} -- **执行参数**: {{$.param}} -- **错误信息**: {{$.error}} -- **节点**: {{$.node.name}} -- **爬虫**: {{$.spider.name}} -- **项目**: {{$.spider.project.name}} -- **定时任务**: {{$.schedule.name}} -- **结果数**: {{$.:task_stat.result_count}} -- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} -- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} -- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: models.NotificationSettingMobile{}, - }, - { - Id: primitive.NewObjectID(), - Type: TypeMobile, - Enabled: true, - Name: "Task Change (Mobile)", - Description: "This is the default mobile notification. You can edit it with your own settings", - TaskTrigger: constants.NotificationTriggerTaskError, - Title: "[Crawlab] Task Update: {{$.status}}", - Template: `Dear {{$.user.username}}, - -Please find the task data as below. - -- **Task Status**: {{$.status}} -- **Task Priority**: {{$.priority}} -- **Task Mode**: {{$.mode}} -- **Task Command**: {{$.cmd}} -- **Task Params**: {{$.param}} -- **Error Message**: {{$.error}} -- **Node**: {{$.node.name}} -- **Spider**: {{$.spider.name}} -- **Project**: {{$.spider.project.name}} -- **Schedule**: {{$.schedule.name}} -- **Result Count**: {{$.:task_stat.result_count}} -- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} -- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} -- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} -- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, - Mobile: models.NotificationSettingMobile{}, - }, - } - _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertMany(settings) - if err != nil { - return err - } - return nil -} - -func (svc *ServiceV2) Send(s *models.NotificationSettingV2, entity bson.M) (err error) { - switch s.Type { - case TypeMail: - return svc.SendMail(s, entity) - case TypeMobile: - return svc.SendMobile(s, entity) - } - return nil -} - -func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, entity bson.M) (err error) { - // to - to, err := parser.Parse(s.Mail.To, entity) - if err != nil { - log.Warnf("parsing 'to' error: %v", err) - } - if to == "" { - return nil - } - - // cc - cc, err := parser.Parse(s.Mail.Cc, entity) - if err != nil { - log.Warnf("parsing 'cc' error: %v", err) - } - - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } + // request + r, _ := svc.createRequest(s, ch, title, content) // send mail - if err := SendMail(&s.Mail, to, cc, title, content); err != nil { - return err + err := SendMail(s, ch, mailTo, mailCc, mailBcc, title, content) + if err != nil { + log.Errorf("[NotificationServiceV2] send mail error: %v", err) } - return nil + // save request + go svc.saveRequest(r, err) } -func (svc *ServiceV2) SendMobile(s *models.NotificationSettingV2, entity bson.M) (err error) { - // webhook - webhook, err := parser.Parse(s.Mobile.Webhook, entity) +func (svc *ServiceV2) SendIM(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) { + // request + r, _ := svc.createRequest(s, ch, title, content) + + // send mobile notification + err := SendIMNotification(ch, title, content) if err != nil { - log.Warnf("parsing 'webhook' error: %v", err) - } - if webhook == "" { - return nil + log.Errorf("[NotificationServiceV2] send mobile notification error: %v", err) } - // title - title, err := parser.Parse(s.Title, entity) - if err != nil { - log.Warnf("parsing 'title' error: %v", err) - } - - // content - content, err := parser.Parse(s.Template, entity) - if err != nil { - log.Warnf("parsing 'content' error: %v", err) - } - - // send - if err := SendMobileNotification(webhook, title, content); err != nil { - return err - } - - return nil + // save request + go svc.saveRequest(r, err) } -func (svc *ServiceV2) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []models.NotificationSettingV2, total int, err error) { - // options - var options *mongo2.FindOptions - if pagination != nil || sort != nil { - options = new(mongo2.FindOptions) - if pagination != nil { - options.Skip = pagination.Size * (pagination.Page - 1) - options.Limit = pagination.Size +func (svc *ServiceV2) getContent(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, args ...any) (content string) { + vd := svc.getVariableData(args...) + switch s.TemplateMode { + case constants.NotificationTemplateModeMarkdown: + variables := svc.parseTemplateVariables(s.TemplateMarkdown) + content = svc.geContentWithVariables(s.TemplateMarkdown, variables, vd) + if ch.Type == TypeMail { + content = svc.convertMarkdownToHtml(content) } - if sort != nil { - options.Sort = sort + return content + case constants.NotificationTemplateModeRichText: + template := s.TemplateRichText + if ch.Type == TypeIM { + template = s.TemplateMarkdown + } + variables := svc.parseTemplateVariables(template) + return svc.geContentWithVariables(template, variables, vd) + } + + return content +} + +func (svc *ServiceV2) geContentWithVariables(template string, variables []entity.NotificationVariable, vd VariableData) (content string) { + content = template + for _, v := range variables { + switch v.Category { + case "task": + if vd.Task == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Id.Hex()) + case "status": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Status) + case "cmd": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Cmd) + case "param": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Param) + case "error": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Error) + case "pid": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Pid)) + case "type": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Type) + case "mode": + content = strings.ReplaceAll(content, v.GetKey(), vd.Task.Mode) + case "priority": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Task.Priority)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Task.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Task.UpdatedBy)) + } + + case "task_stat": + if vd.TaskStat == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "start_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.StartTs)) + case "end_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.TaskStat.EndTs)) + case "wait_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.WaitDuration/1000)) + case "runtime_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.RuntimeDuration/1000)) + case "total_duration": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%ds", vd.TaskStat.TotalDuration/1000)) + case "result_count": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.TaskStat.ResultCount)) + } + + case "spider": + if vd.Spider == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Name) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Description) + case "mode": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Mode) + case "cmd": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Cmd) + case "param": + content = strings.ReplaceAll(content, v.GetKey(), vd.Spider.Param) + case "priority": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Spider.Priority)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Spider.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Spider.UpdatedBy)) + } + + case "node": + if vd.Node == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Id.Hex()) + case "key": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Key) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Name) + case "is_master": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.IsMaster)) + case "ip": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Ip) + case "mac": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Mac) + case "hostname": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Hostname) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Description) + case "status": + content = strings.ReplaceAll(content, v.GetKey(), vd.Node.Status) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Enabled)) + case "active": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Node.Active)) + case "active_at": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.ActiveAt)) + case "available_runners": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.AvailableRunners)) + case "max_runners": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Node.MaxRunners)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Node.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Node.UpdatedBy)) + } + + case "schedule": + if vd.Schedule == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Name) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Description) + case "cron": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Cron) + case "cmd": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Cmd) + case "param": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Param) + case "mode": + content = strings.ReplaceAll(content, v.GetKey(), vd.Schedule.Mode) + case "priority": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Schedule.Priority)) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Schedule.Enabled)) + case "created_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.CreatedAt)) + case "created_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.CreatedBy)) + case "updated_ts": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTime(vd.Schedule.UpdatedAt)) + case "updated_by": + content = strings.ReplaceAll(content, v.GetKey(), svc.getUsernameById(vd.Schedule.UpdatedBy)) + } + + case "alert": + switch v.Name { + case "id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Id.Hex()) + case "name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Name) + case "description": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Description) + case "enabled": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%t", vd.Alert.Enabled)) + case "metric_name": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.MetricName) + case "operator": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Operator) + case "lasting_seconds": + content = strings.ReplaceAll(content, v.GetKey(), fmt.Sprintf("%d", vd.Alert.LastingSeconds)) + case "target_value": + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedTargetValue(vd.Alert)) + case "level": + content = strings.ReplaceAll(content, v.GetKey(), vd.Alert.Level) + } + + case "metric": + if vd.Metric == nil { + content = strings.ReplaceAll(content, v.GetKey(), "N/A") + continue + } + switch v.Name { + case "type": + content = strings.ReplaceAll(content, v.GetKey(), vd.Metric.Type) + case "node_id": + content = strings.ReplaceAll(content, v.GetKey(), vd.Metric.NodeId.Hex()) + default: + content = strings.ReplaceAll(content, v.GetKey(), svc.getFormattedMetricValue(v.Name, vd.Metric)) + } + + } + } + return content +} + +func (svc *ServiceV2) getVariableData(args ...any) (vd VariableData) { + for _, arg := range args { + switch arg.(type) { + case *models.TaskV2: + vd.Task = arg.(*models.TaskV2) + case *models.TaskStatV2: + vd.TaskStat = arg.(*models.TaskStatV2) + case *models.SpiderV2: + vd.Spider = arg.(*models.SpiderV2) + case *models.NodeV2: + vd.Node = arg.(*models.NodeV2) + case *models.ScheduleV2: + vd.Schedule = arg.(*models.ScheduleV2) + case *models.NotificationAlertV2: + vd.Alert = arg.(*models.NotificationAlertV2) + case *models.MetricV2: + vd.Metric = arg.(*models.MetricV2) + } + } + return vd +} + +func (svc *ServiceV2) parseTemplateVariables(template string) (variables []entity.NotificationVariable) { + // regex pattern + regex := regexp.MustCompile("\\$\\{(\\w+):(\\w+)}") + + // find all matches + matches := regex.FindAllStringSubmatch(template, -1) + + // variables map + variablesMap := make(map[string]entity.NotificationVariable) + + // iterate over matches + for _, match := range matches { + variable := entity.NotificationVariable{ + Category: match[1], + Name: match[2], + } + key := fmt.Sprintf("%s:%s", variable.Category, variable.Name) + if _, ok := variablesMap[key]; !ok { + variablesMap[key] = variable } } - // get list - list, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(query, options) - if err != nil { - if err.Error() == mongo.ErrNoDocuments.Error() { - return nil, 0, nil - } else { - return nil, 0, err - } + // convert map to slice + for _, variable := range variablesMap { + variables = append(variables, variable) } - // total count - total, err = service.NewModelServiceV2[models.NotificationSettingV2]().Count(query) - if err != nil { - return nil, 0, err - } - - return list, total, nil + return variables } -func (svc *ServiceV2) GetSetting(id primitive.ObjectID) (res *models.NotificationSettingV2, err error) { - s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) +func (svc *ServiceV2) getUsernameById(id primitive.ObjectID) (username string) { + if id.IsZero() { + return "" + } + u, err := service.NewModelServiceV2[models.UserV2]().GetById(id) if err != nil { + log.Errorf("[NotificationServiceV2] get user error: %v", err) + return "" + } + return u.Username +} + +func (svc *ServiceV2) getFormattedTime(t time.Time) (res string) { + if t.IsZero() { + return "N/A" + } + return t.Local().Format(time.DateTime) +} + +func (svc *ServiceV2) getFormattedTargetValue(a *models.NotificationAlertV2) (res string) { + if strings.HasSuffix(a.MetricName, "_percent") { + return fmt.Sprintf("%.2f%%", a.TargetValue) + } else if strings.HasSuffix(a.MetricName, "_memory") { + return fmt.Sprintf("%dMB", int(a.TargetValue/(1024*1024))) + } else if strings.HasSuffix(a.MetricName, "_disk") { + return fmt.Sprintf("%dGB", int(a.TargetValue/(1024*1024*1024))) + } else if strings.HasSuffix(a.MetricName, "_rate") { + return fmt.Sprintf("%.2fMB/s", a.TargetValue/(1024*1024)) + } else { + return fmt.Sprintf("%f", a.TargetValue) + } +} + +func (svc *ServiceV2) getFormattedMetricValue(metricName string, m *models.MetricV2) (res string) { + switch metricName { + case "cpu_usage_percent": + return fmt.Sprintf("%.2f%%", m.CpuUsagePercent) + case "total_memory": + return fmt.Sprintf("%dMB", m.TotalMemory/(1024*1024)) + case "available_memory": + return fmt.Sprintf("%dMB", m.AvailableMemory/(1024*1024)) + case "used_memory": + return fmt.Sprintf("%dMB", m.UsedMemory/(1024*1024)) + case "used_memory_percent": + return fmt.Sprintf("%.2f%%", m.UsedMemoryPercent) + case "total_disk": + return fmt.Sprintf("%dGB", m.TotalDisk/(1024*1024*1024)) + case "available_disk": + return fmt.Sprintf("%dGB", m.AvailableDisk/(1024*1024*1024)) + case "used_disk": + return fmt.Sprintf("%dGB", m.UsedDisk/(1024*1024*1024)) + case "used_disk_percent": + return fmt.Sprintf("%.2f%%", m.UsedDiskPercent) + case "disk_read_bytes_rate": + return fmt.Sprintf("%.2fMB/s", m.DiskReadBytesRate/(1024*1024)) + case "disk_write_bytes_rate": + return fmt.Sprintf("%.2fMB/s", m.DiskWriteBytesRate/(1024*1024)) + case "network_bytes_sent_rate": + return fmt.Sprintf("%.2fMB/s", m.NetworkBytesSentRate/(1024*1024)) + case "network_bytes_recv_rate": + return fmt.Sprintf("%.2fMB/s", m.NetworkBytesRecvRate/(1024*1024)) + default: + return "N/A" + } +} + +func (svc *ServiceV2) convertMarkdownToHtml(content string) (html string) { + return string(markdown.ToHTML([]byte(content), nil, nil)) +} + +func (svc *ServiceV2) SendNodeNotification(node *models.NodeV2) { + // arguments + var args []any + args = append(args, node) + + // settings + settings, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(bson.M{ + "enabled": true, + "trigger": bson.M{ + "$regex": constants.NotificationTriggerPatternNode, + }, + }, nil) + if err != nil { + log.Errorf("get notification settings error: %v", err) + trace.PrintError(err) + return + } + + for _, s := range settings { + // send notification + switch s.Trigger { + case constants.NotificationTriggerNodeStatusChange: + go svc.Send(&s, args...) + case constants.NotificationTriggerNodeOnline: + if node.Status == constants.NodeStatusOnline { + go svc.Send(&s, args...) + } + case constants.NotificationTriggerNodeOffline: + if node.Status == constants.NodeStatusOffline { + go svc.Send(&s, args...) + } + } + } +} + +func (svc *ServiceV2) createRequest(s *models.NotificationSettingV2, ch *models.NotificationChannelV2, title, content string) (res *models.NotificationRequestV2, err error) { + senderEmail := ch.SMTPUsername + if s.UseCustomSenderEmail { + senderEmail = s.SenderEmail + } + r := models.NotificationRequestV2{ + Status: StatusSending, + SettingId: s.Id, + ChannelId: ch.Id, + Title: title, + Content: content, + SenderEmail: senderEmail, + SenderName: s.SenderName, + MailTo: s.MailTo, + MailCc: s.MailCc, + MailBcc: s.MailBcc, + } + r.SetCreatedAt(time.Now()) + r.SetUpdatedAt(time.Now()) + r.Id, err = service.NewModelServiceV2[models.NotificationRequestV2]().InsertOne(r) + if err != nil { + log.Errorf("[NotificationServiceV2] save request error: %v", err) return nil, err } - return s, nil + return &r, nil } -func (svc *ServiceV2) PosSetting(s *models.NotificationSettingV2) (err error) { - s.Id = primitive.NewObjectID() - _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertOne(*s) +func (svc *ServiceV2) saveRequest(r *models.NotificationRequestV2, err error) { + if r == nil { + return + } + if err != nil { - return err + r.Status = StatusError + r.Error = err.Error() + } else { + r.Status = StatusSuccess } - return nil -} - -func (svc *ServiceV2) PutSetting(id primitive.ObjectID, s models.NotificationSettingV2) (err error) { - err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, s) + r.SetUpdatedAt(time.Now()) + err = service.NewModelServiceV2[models.NotificationRequestV2]().ReplaceById(r.Id, *r) if err != nil { - return err - } - - return nil -} - -func (svc *ServiceV2) DeleteSetting(id primitive.ObjectID) (err error) { - err = service.NewModelServiceV2[models.NotificationSettingV2]().DeleteById(id) - if err != nil { - return err - } - - return nil -} - -func (svc *ServiceV2) EnableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(true)(id) -} - -func (svc *ServiceV2) DisableSetting(id primitive.ObjectID) (err error) { - return svc._toggleSettingFunc(false)(id) -} - -func (svc *ServiceV2) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { - return func(id primitive.ObjectID) (err error) { - s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) - if err != nil { - return err - } - s.Enabled = value - err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, *s) - if err != nil { - return err - } - return nil + log.Errorf("[NotificationServiceV2] save request error: %v", err) } } -func NewServiceV2() *ServiceV2 { - // service - svc := &ServiceV2{} - - return svc +func newNotificationServiceV2() *ServiceV2 { + return &ServiceV2{} } var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) -func GetServiceV2() *ServiceV2 { - if _serviceV2 == nil { - _serviceV2 = NewServiceV2() - } +func GetNotificationServiceV2() *ServiceV2 { + _serviceV2Once.Do(func() { + _serviceV2 = newNotificationServiceV2() + }) return _serviceV2 } diff --git a/core/notification/service_v2_test.go b/core/notification/service_v2_test.go new file mode 100644 index 00000000..a2ca0b28 --- /dev/null +++ b/core/notification/service_v2_test.go @@ -0,0 +1,38 @@ +package notification + +import ( + "github.com/crawlab-team/crawlab/core/entity" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseTemplateVariables_WithValidTemplate_ReturnsVariables(t *testing.T) { + svc := ServiceV2{} + template := "Dear ${user:name}, your task ${task:id} is ${task:status}." + expected := []entity.NotificationVariable{ + {Category: "user", Name: "name"}, + {Category: "task", Name: "id"}, + {Category: "task", Name: "status"}, + } + + variables := svc.parseTemplateVariables(template) + + // contains all expected variables + assert.ElementsMatch(t, expected, variables) +} + +func TestParseTemplateVariables_WithRepeatedVariables_ReturnsUniqueVariables(t *testing.T) { + svc := ServiceV2{} + template := "Dear ${user:name}, your task ${task:id} is ${task:status}. Again, ${user:name} and ${task:id}." + expected := []entity.NotificationVariable{ + {Category: "user", Name: "name"}, + {Category: "task", Name: "id"}, + {Category: "task", Name: "status"}, + } + + variables := svc.parseTemplateVariables(template) + + // contains all expected variables + assert.ElementsMatch(t, expected, variables) +} diff --git a/core/notification/theme.go b/core/notification/theme.go new file mode 100644 index 00000000..f93fb510 --- /dev/null +++ b/core/notification/theme.go @@ -0,0 +1,265 @@ +package notification + +const defaultTheme = `` + +func GetTheme() string { + return defaultTheme +} diff --git a/core/result/service.go b/core/result/service.go index d1e94685..5497f3a6 100644 --- a/core/result/service.go +++ b/core/result/service.go @@ -38,7 +38,7 @@ func NewResultService(registryKey string, s *models.Spider) (svc2 interfaces.Res var store = sync.Map{} -func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfaces.ResultService, err error) { +func GetResultService(spiderId primitive.ObjectID) (svc2 interfaces.ResultService, err error) { // model service modelSvc, err := service.GetService() if err != nil { @@ -51,12 +51,6 @@ func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfa return nil, trace.TraceError(err) } - // apply options - _opts := &Options{} - for _, opt := range opts { - opt(_opts) - } - // store key storeKey := s.ColId.Hex() + ":" + s.DataSourceId.Hex() diff --git a/core/result/test/base.go b/core/result/test/base.go deleted file mode 100644 index a224fa29..00000000 --- a/core/result/test/base.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/db/mongo" - "go.uber.org/dig" - "testing" -) - -func init() { - T = NewTest() -} - -var T *Test - -type Test struct { - // dependencies - modelSvc service.ModelService - resultSvc interfaces.ResultService - - // test data - TestColName string - TestCol *mongo.Col - TestDc *models.DataCollection -} - -func (t *Test) Setup(t2 *testing.T) { - t2.Cleanup(t.Cleanup) -} - -func (t *Test) Cleanup() { - _ = t.modelSvc.DropAll() -} - -func NewTest() *Test { - var err error - - // test - t := &Test{ - TestColName: "test_results", - } - - // dependency injection - c := dig.New() - if err := c.Provide(service.NewService); err != nil { - panic(err) - } - if err := c.Invoke(func( - modelSvc service.ModelService, - ) { - t.modelSvc = modelSvc - }); err != nil { - panic(err) - } - - // data collection - t.TestDc = &models.DataCollection{ - Name: t.TestColName, - } - if err := delegate.NewModelDelegate(t.TestDc).Add(); err != nil { - panic(err) - } - t.TestCol = mongo.GetMongoCol(t.TestColName) - - // result service - t.resultSvc, err = result.GetResultService(t.TestDc.GetId()) - if err != nil { - panic(err) - } - - return t -} diff --git a/core/result/test/service_test.go b/core/result/test/service_test.go deleted file mode 100644 index b44faf4e..00000000 --- a/core/result/test/service_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package test - -import ( - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/stretchr/testify/require" - "testing" -) - -func TestResultService_GetList(t *testing.T) { - var err error - T.Setup(t) - - n := 1000 - var docs []interface{} - for i := 0; i < n; i++ { - d := &models.Result{ - "i": i, - } - docs = append(docs, d) - } - _, err = T.TestCol.InsertMany(docs) - require.Nil(t, err) - - // get all - results, err := T.resultSvc.List(nil, nil) - require.Nil(t, err) - require.Equal(t, n, len(results)) - - //query := bson.M{ - // "i": bson.M{ - // "$lt": n / 2, - // }, - //} - //results, err = T.resultSvc.List(query, nil) - //require.Nil(t, err) - //require.Equal(t, n/2, len(results)) -} - -func TestResultService_Count(t *testing.T) { - var err error - T.Setup(t) - - n := 1000 - var docs []interface{} - for i := 0; i < n; i++ { - d := &models.Result{ - "i": i, - } - docs = append(docs, d) - } - _, err = T.TestCol.InsertMany(docs) - require.Nil(t, err) - - // get all - total, err := T.resultSvc.Count(nil) - require.Nil(t, err) - require.Equal(t, n, total) - - //query := bson.M{ - // "i": bson.M{ - // "$lt": n / 2, - // }, - //} - //total, err = T.resultSvc.Count(query) - //require.Nil(t, err) - //require.Equal(t, n/2, total) -} diff --git a/core/routes/group.go b/core/routes/group.go deleted file mode 100644 index cade1af4..00000000 --- a/core/routes/group.go +++ /dev/null @@ -1,20 +0,0 @@ -package routes - -import ( - "github.com/crawlab-team/crawlab/core/middlewares" - "github.com/gin-gonic/gin" -) - -type RouterGroups struct { - AuthGroup *gin.RouterGroup - AnonymousGroup *gin.RouterGroup - FilerGroup *gin.RouterGroup -} - -func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { - return &RouterGroups{ - AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), - AnonymousGroup: app.Group("/"), - FilerGroup: app.Group("/filer", middlewares.FilerAuthorizationMiddleware()), - } -} diff --git a/core/routes/router.go b/core/routes/router.go deleted file mode 100644 index c82603bc..00000000 --- a/core/routes/router.go +++ /dev/null @@ -1,178 +0,0 @@ -package routes - -import ( - "fmt" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/controllers" - "github.com/gin-gonic/gin" - "net/http" - "path" -) - -type RouterServiceInterface interface { - RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) - RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) -} - -type RouterService struct { - app *gin.Engine -} - -func NewRouterService(app *gin.Engine) (svc *RouterService) { - return &RouterService{ - app: app, - } -} - -func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) { - group.GET(basePath, ctr.Get) - group.POST(basePath, ctr.Post) - group.PUT(basePath, ctr.Put) - group.DELETE(basePath, ctr.Delete) -} - -func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) { - group.GET(basePath+"/:id", ctr.Get) - group.GET(basePath, ctr.GetList) - group.POST(basePath, ctr.Post) - group.POST(basePath+"/batch", ctr.PostList) - group.PUT(basePath+"/:id", ctr.Put) - group.PUT(basePath, ctr.PutList) - group.DELETE(basePath+"/:id", ctr.Delete) - group.DELETE(basePath, ctr.DeleteList) -} - -func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) { - for _, action := range ctr.Actions() { - routerPath := path.Join(basePath, action.Path) - switch action.Method { - case http.MethodGet: - group.GET(routerPath, action.HandlerFunc) - case http.MethodPost: - group.POST(routerPath, action.HandlerFunc) - case http.MethodPut: - group.PUT(routerPath, action.HandlerFunc) - case http.MethodDelete: - group.DELETE(routerPath, action.HandlerFunc) - } - } -} - -func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) { - svc.RegisterListControllerToGroup(group, basePath, ctr) - svc.RegisterActionControllerToGroup(group, basePath, ctr) -} - -func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) { - switch method { - case http.MethodGet: - group.GET(path, handler) - case http.MethodPost: - group.POST(path, handler) - case http.MethodPut: - group.PUT(path, handler) - case http.MethodDelete: - group.DELETE(path, handler) - default: - log.Warn(fmt.Sprintf("%s is not a valid http method", method)) - } -} - -func InitRoutes(app *gin.Engine) (err error) { - // routes groups - groups := NewRouterGroups(app) - - // router service - svc := NewRouterService(app) - - // register routes - registerRoutesAnonymousGroup(svc, groups) - registerRoutesAuthGroup(svc, groups) - registerRoutesFilterGroup(svc, groups) - - return nil -} - -func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) { - // login - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController) - - // version - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController) - - // system info - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController) - - // demo - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController) - - // sync - svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController) -} - -func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { - // node - svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController) - - // project - svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController) - - // user - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController) - - // spider - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController) - - // task - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController) - - // tag - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController) - - // setting - svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController) - - // data collection - svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController) - - // result - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController) - - // schedule - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController) - - // stats - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController) - - // token - svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController) - - // git - svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController) - - // role - svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController) - - // permission - svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController) - - // export - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController) - - // notification - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController) - - // filter - svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) - - // data sources - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) - - // environments - svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController) -} - -func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { - // filer - svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController) -} diff --git a/core/routes/router_test.go b/core/routes/router_test.go deleted file mode 100644 index 7b72c852..00000000 --- a/core/routes/router_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package routes - -import ( - "github.com/gin-gonic/gin" - "github.com/stretchr/testify/require" - "net/http" - "testing" - "time" -) - -func TestInitRoutes(t *testing.T) { - app := gin.New() - err := InitRoutes(app) - require.Nil(t, err) - - srv := &http.Server{ - Handler: app, - Addr: "localhost:8000", - } - go func() { - err = srv.ListenAndServe() - require.Nil(t, err) - }() - - time.Sleep(5 * time.Second) -} diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go index e7c4f51d..a1a244ac 100644 --- a/core/schedule/service_v2.go +++ b/core/schedule/service_v2.go @@ -1,9 +1,10 @@ package schedule import ( + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/spider/admin" "github.com/crawlab-team/crawlab/core/utils" @@ -18,7 +19,7 @@ import ( type ServiceV2 struct { // dependencies interfaces.WithConfigPath - modelSvc *service.ModelServiceV2[models.ScheduleV2] + modelSvc *service.ModelServiceV2[models2.ScheduleV2] adminSvc *admin.ServiceV2 // settings variables @@ -30,7 +31,7 @@ type ServiceV2 struct { // internals cron *cron.Cron logger cron.Logger - schedules []models.ScheduleV2 + schedules []models2.ScheduleV2 stopped bool mu sync.Mutex } @@ -86,7 +87,7 @@ func (svc *ServiceV2) Stop() { svc.cron.Stop() } -func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Enable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -100,7 +101,7 @@ func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err er return svc.modelSvc.ReplaceById(s.Id, s) } -func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) Disable(s models2.ScheduleV2, by primitive.ObjectID) (err error) { svc.mu.Lock() defer svc.mu.Unlock() @@ -190,7 +191,7 @@ func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { } // spider - spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId) + spider, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(s.SpiderId) if err != nil { trace.PrintError(err) return @@ -249,7 +250,7 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { if err != nil { return nil, err } - svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]() + svc.modelSvc = service.NewModelServiceV2[models2.ScheduleV2]() // logger svc.logger = NewLogger() @@ -270,12 +271,18 @@ func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { } var svcV2 *ServiceV2 +var svcV2Once = new(sync.Once) func GetScheduleServiceV2() (res *ServiceV2, err error) { if svcV2 != nil { return svcV2, nil } - svcV2, err = NewScheduleServiceV2() + svcV2Once.Do(func() { + svcV2, err = NewScheduleServiceV2() + if err != nil { + log.Errorf("failed to get schedule service: %v", err) + } + }) if err != nil { return nil, err } diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go index c62a5fee..8fcba6c9 100644 --- a/core/spider/admin/service_v2.go +++ b/core/spider/admin/service_v2.go @@ -1,29 +1,20 @@ package admin import ( - "context" - "github.com/apex/log" + log2 "github.com/apex/log" config2 "github.com/crawlab-team/crawlab/core/config" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/core/task/scheduler" - "github.com/crawlab-team/crawlab/core/utils" "github.com/crawlab-team/crawlab/trace" - "github.com/crawlab-team/crawlab/vcs" - "github.com/google/uuid" "github.com/robfig/cron/v3" - "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "os" - "path" - "path/filepath" "sync" - "time" ) type ServiceV2 struct { @@ -37,13 +28,9 @@ type ServiceV2 struct { cfgPath string } -func (svc *ServiceV2) Start() (err error) { - return svc.SyncGit() -} - func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // spider - s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(id) if err != nil { return nil, err } @@ -52,37 +39,9 @@ func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRun return svc.scheduleTasks(s, opts) } -func (svc *ServiceV2) SyncGit() (err error) { - if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { - return trace.TraceError(err) - } - svc.cron.Start() - return nil -} - -func (svc *ServiceV2) SyncGitOne(g *models.GitV2) (err error) { - svc.syncGitOne(g) - return nil -} - -func (svc *ServiceV2) Export(id primitive.ObjectID) (filePath string, err error) { - // spider fs - workspacePath := viper.GetString("workspace") - spiderFolderPath := filepath.Join(workspacePath, id.Hex()) - - // zip files in workspace - dirPath := spiderFolderPath - zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") - if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { - return "", trace.TraceError(err) - } - - return zipFilePath, nil -} - -func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { +func (svc *ServiceV2) scheduleTasks(s *models2.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { // main task - mainTask := &models.TaskV2{ + t := &models2.TaskV2{ SpiderId: s.Id, Mode: opts.Mode, NodeIds: opts.NodeIds, @@ -90,68 +49,38 @@ func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderR Param: opts.Param, ScheduleId: opts.ScheduleId, Priority: opts.Priority, - UserId: opts.UserId, - CreateTs: time.Now(), } - mainTask.SetId(primitive.NewObjectID()) + t.SetId(primitive.NewObjectID()) // normalize - if mainTask.Mode == "" { - mainTask.Mode = s.Mode + if t.Mode == "" { + t.Mode = s.Mode } - if mainTask.NodeIds == nil { - mainTask.NodeIds = s.NodeIds + if t.NodeIds == nil { + t.NodeIds = s.NodeIds } - if mainTask.Cmd == "" { - mainTask.Cmd = s.Cmd + if t.Cmd == "" { + t.Cmd = s.Cmd } - if mainTask.Param == "" { - mainTask.Param = s.Param + if t.Param == "" { + t.Param = s.Param } - if mainTask.Priority == 0 { - mainTask.Priority = s.Priority + if t.Priority == 0 { + t.Priority = s.Priority } - if svc.isMultiTask(opts) { - // multi tasks - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - for _, nodeId := range nodeIds { - t := &models.TaskV2{ - SpiderId: s.Id, - Mode: opts.Mode, - Cmd: opts.Cmd, - Param: opts.Param, - NodeId: nodeId, - ScheduleId: opts.ScheduleId, - Priority: opts.Priority, - UserId: opts.UserId, - CreateTs: time.Now(), - } - t.SetId(primitive.NewObjectID()) - t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) - } - } else { - // single task - nodeIds, err := svc.getNodeIds(opts) - if err != nil { - return nil, err - } - if len(nodeIds) > 0 { - mainTask.NodeId = nodeIds[0] - } - t2, err := svc.schedulerSvc.Enqueue(mainTask, opts.UserId) - if err != nil { - return nil, err - } - taskIds = append(taskIds, t2.Id) + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err } + if len(nodeIds) > 0 { + t.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) return taskIds, nil } @@ -163,7 +92,7 @@ func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []p "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { return nil, err } @@ -183,7 +112,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) "enabled": true, "status": constants.NodeStatusOnline, } - nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + nodes, err := service.NewModelServiceV2[models2.NodeV2]().GetMany(query, nil) if err != nil { trace.PrintError(err) return false @@ -198,100 +127,7 @@ func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) } } -func (svc *ServiceV2) syncGit() { - if svc.syncLock { - log.Infof("[SpiderAdminService] sync git is locked, skip") - return - } - log.Infof("[SpiderAdminService] start to sync git") - - svc.syncLock = true - defer func() { - svc.syncLock = false - }() - - // spiders - spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(nil, nil) - if err != nil { - trace.PrintError(err) - return - } - - // spider ids - var spiderIds []primitive.ObjectID - for _, s := range spiders { - spiderIds = append(spiderIds, s.Id) - } - - if len(spiderIds) > 0 { - // gits - gits, err := service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{ - "_id": bson.M{ - "$in": spiderIds, - }, - "auto_pull": true, - }, nil) - if err != nil { - trace.PrintError(err) - return - } - - wg := sync.WaitGroup{} - wg.Add(len(gits)) - for _, g := range gits { - go func(g models.GitV2) { - svc.syncGitOne(&g) - wg.Done() - }(g) - } - wg.Wait() - } - - log.Infof("[SpiderAdminService] finished sync git") -} - -func (svc *ServiceV2) syncGitOne(g *models.GitV2) { - log.Infof("[SpiderAdminService] sync git %s", g.Id) - - ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) - defer cancel() - - // git client - workspacePath := viper.GetString("workspace") - gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.Id.Hex()))) - if err != nil { - return - } - - // set auth - utils.InitGitClientAuthV2(g, gitClient) - - // check if remote has changes - ok, err := gitClient.IsRemoteChanged() - if err != nil { - trace.PrintError(err) - return - } - if !ok { - // no change - return - } - - // pull and sync to workspace - if err := gitClient.Reset(); err != nil { - trace.PrintError(err) - return - } - if err := gitClient.Pull(); err != nil { - trace.PrintError(err) - return - } - - // wait for context to end - <-ctx.Done() -} - -func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { +func newSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { svc := &ServiceV2{ nodeCfgSvc: config.GetNodeConfigService(), cfgPath: config2.GetConfigPath(), @@ -313,16 +149,20 @@ func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { } var svcV2 *ServiceV2 +var svcV2Once = new(sync.Once) func GetSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { if svcV2 != nil { return svcV2, nil } - - svcV2, err = NewSpiderAdminServiceV2() + svcV2Once.Do(func() { + svcV2, err = newSpiderAdminServiceV2() + if err != nil { + log2.Errorf("[GetSpiderAdminServiceV2] error: %v", err) + } + }) if err != nil { return nil, err } - return svcV2, nil } diff --git a/core/sys_exec/sys_exec_darwin.go b/core/sys_exec/sys_exec_darwin.go index 130b7cb8..b6db18c2 100644 --- a/core/sys_exec/sys_exec_darwin.go +++ b/core/sys_exec/sys_exec_darwin.go @@ -4,12 +4,18 @@ package sys_exec import ( + "errors" "os/exec" + "strings" "syscall" ) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("sh", "-c", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } func SetPgid(cmd *exec.Cmd) { diff --git a/core/sys_exec/sys_exec_linux.go b/core/sys_exec/sys_exec_linux.go index bf532a43..33af73e3 100644 --- a/core/sys_exec/sys_exec_linux.go +++ b/core/sys_exec/sys_exec_linux.go @@ -4,12 +4,18 @@ package sys_exec import ( + "errors" "os/exec" + "strings" "syscall" ) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("sh", "-c", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } func SetPgid(cmd *exec.Cmd) { diff --git a/core/sys_exec/sys_exec_windows.go b/core/sys_exec/sys_exec_windows.go index e2678ccd..1f1afd5a 100644 --- a/core/sys_exec/sys_exec_windows.go +++ b/core/sys_exec/sys_exec_windows.go @@ -3,8 +3,16 @@ package sys_exec -import "os/exec" +import ( + "errors" + "os/exec" + "strings" +) -func BuildCmd(cmdStr string) *exec.Cmd { - return exec.Command("cmd", "/C", cmdStr) +func BuildCmd(cmdStr string) (cmd *exec.Cmd, err error) { + if cmdStr == "" { + return nil, errors.New("command string is empty") + } + args := strings.Split(cmdStr, " ") + return exec.Command(args[0], args[1:]...), nil } diff --git a/core/system/service.go b/core/system/service.go index 891657bb..94cd3140 100644 --- a/core/system/service.go +++ b/core/system/service.go @@ -25,7 +25,7 @@ func (svc *Service) Init() (err error) { func (svc *Service) initData() (err error) { total, err := svc.col.Count(bson.M{ - "key": "site_title", + "key": "customize", }) if err != nil { return err @@ -38,10 +38,13 @@ func (svc *Service) initData() (err error) { settings := []models.Setting{ { Id: primitive.NewObjectID(), - Key: "site_title", + Key: "customize", Value: bson.M{ - "customize_site_title": false, - "site_title": "", + "show_custom_title": false, + "custom_title": "", + "show_custom_logo": false, + "custom_logo": "", + "hide_platform_version": false, }, }, } diff --git a/core/system/service_v2.go b/core/system/service_v2.go new file mode 100644 index 00000000..d82a6048 --- /dev/null +++ b/core/system/service_v2.go @@ -0,0 +1,72 @@ +package system + +import ( + "github.com/crawlab-team/crawlab/core/models/models/v2" + "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson" + "sync" +) + +type ServiceV2 struct { +} + +func (svc *ServiceV2) Init() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) initData() (err error) { + total, err := service.NewModelServiceV2[models.SettingV2]().Count(bson.M{ + "key": "site_title", + }) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.SettingV2{ + { + Key: "site_title", + Value: bson.M{ + "customize_site_title": false, + "site_title": "", + }, + }, + } + _, err = service.NewModelServiceV2[models.SettingV2]().InsertMany(settings) + if err != nil { + return err + } + return nil +} + +func newSystemServiceV2() *ServiceV2 { + // service + svc := &ServiceV2{} + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) + +func GetSystemServiceV2() *ServiceV2 { + if _serviceV2 == nil { + _serviceV2 = newSystemServiceV2() + } + _serviceV2Once.Do(func() { + _serviceV2 = newSystemServiceV2() + }) + return _serviceV2 +} diff --git a/core/task/base.go b/core/task/base.go deleted file mode 100644 index 90de0ac2..00000000 --- a/core/task/base.go +++ /dev/null @@ -1,97 +0,0 @@ -package task - -import ( - "fmt" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" -) - -type BaseService struct { - // dependencies - interfaces.WithConfigPath - modelSvc service.ModelService - - // internals - stopped bool -} - -func (svc *BaseService) Init() error { - // implement me - return nil -} - -func (svc *BaseService) Start() { - // implement me -} - -func (svc *BaseService) Wait() { - utils.DefaultWait() -} - -func (svc *BaseService) Stop() { - svc.stopped = true -} - -// SaveTask deprecated -func (svc *BaseService) SaveTask(t interfaces.Task, status string) (err error) { - // normalize status - if status == "" { - status = constants.TaskStatusPending - } - - // set task status - t.SetStatus(status) - - // attempt to get task from database - _, err = svc.modelSvc.GetTaskById(t.GetId()) - if err != nil { - // if task does not exist, add to database - if err == mongo.ErrNoDocuments { - if err := delegate.NewModelDelegate(t).Add(); err != nil { - return err - } - return nil - } else { - return err - } - } else { - // otherwise, update - if err := delegate.NewModelDelegate(t).Save(); err != nil { - return err - } - return nil - } -} - -func (svc *BaseService) IsStopped() (res bool) { - return svc.stopped -} - -func (svc *BaseService) GetQueue(nodeId primitive.ObjectID) (queue string) { - if nodeId.IsZero() { - return fmt.Sprintf("%s", constants.TaskListQueuePrefixPublic) - } else { - return fmt.Sprintf("%s:%s", constants.TaskListQueuePrefixNodes, nodeId.Hex()) - } -} - -func NewBaseService() (svc2 interfaces.TaskBaseService, err error) { - svc := &BaseService{} - - // dependency injection - if err := container.GetContainer().Invoke(func(cfgPath interfaces.WithConfigPath, modelSvc service.ModelService) { - svc.WithConfigPath = cfgPath - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - return svc, nil -} diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go deleted file mode 100644 index b39d9445..00000000 --- a/core/task/handler/runner.go +++ /dev/null @@ -1,691 +0,0 @@ -package handler - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "github.com/apex/log" - "github.com/cenkalti/backoff/v4" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" - fs2 "github.com/crawlab-team/crawlab/core/fs" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/sys_exec" - "github.com/crawlab-team/crawlab/core/utils" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "github.com/shirou/gopsutil/process" - "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "io" - "net/http" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" -) - -type Runner struct { - // dependencies - svc interfaces.TaskHandlerService // task handler service - fsSvc interfaces.FsServiceV2 // task fs service - hookSvc interfaces.TaskHookService // task hook service - - // settings - subscribeTimeout time.Duration - bufferSize int - - // internals - cmd *exec.Cmd // process command instance - pid int // process id - tid primitive.ObjectID // task id - t interfaces.Task // task model.Task - s interfaces.Spider // spider model.Spider - ch chan constants.TaskSignal // channel to communicate between Service and Runner - err error // standard process error - envs []models.Env // environment variables - cwd string // working directory - c interfaces.GrpcClient // grpc client - sub grpc.TaskService_SubscribeClient // grpc task service stream client - - // log internals - scannerStdout *bufio.Reader - scannerStderr *bufio.Reader - logBatchSize int -} - -func (r *Runner) Init() (err error) { - // update task - if err := r.updateTask("", nil); err != nil { - return err - } - - // start grpc client - if !r.c.IsStarted() { - r.c.Start() - } - - // working directory - workspacePath := viper.GetString("workspace") - r.cwd = filepath.Join(workspacePath, r.s.GetId().Hex()) - - // sync files from master - if !utils.IsMaster() { - if err := r.syncFiles(); err != nil { - return err - } - } - - // grpc task service stream client - if err := r.initSub(); err != nil { - return err - } - - // pre actions - if r.hookSvc != nil { - if err := r.hookSvc.PreActions(r.t, r.s, r.fsSvc, r.svc); err != nil { - return err - } - } - - return nil -} - -func (r *Runner) Run() (err error) { - // log task started - log.Infof("task[%s] started", r.tid.Hex()) - - // configure cmd - r.configureCmd() - - // configure environment variables - r.configureEnv() - - // configure logging - r.configureLogging() - - // start process - if err := r.cmd.Start(); err != nil { - return r.updateTask(constants.TaskStatusError, err) - } - - // start logging - go r.startLogging() - - // process id - if r.cmd.Process == nil { - return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) - } - r.pid = r.cmd.Process.Pid - r.t.SetPid(r.pid) - - // update task status (processing) - if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { - return err - } - - // wait for process to finish - go r.wait() - - // start health check - go r.startHealthCheck() - - // declare task status - status := "" - - // wait for signal - signal := <-r.ch - switch signal { - case constants.TaskSignalFinish: - err = nil - status = constants.TaskStatusFinished - case constants.TaskSignalCancel: - err = constants.ErrTaskCancelled - status = constants.TaskStatusCancelled - case constants.TaskSignalError: - err = r.err - status = constants.TaskStatusError - case constants.TaskSignalLost: - err = constants.ErrTaskLost - status = constants.TaskStatusError - default: - err = constants.ErrInvalidSignal - status = constants.TaskStatusError - } - - // update task status - if err := r.updateTask(status, err); err != nil { - return err - } - - // post actions - if r.hookSvc != nil { - if err := r.hookSvc.PostActions(r.t, r.s, r.fsSvc, r.svc); err != nil { - return err - } - } - - return err -} - -func (r *Runner) Cancel() (err error) { - // kill process - opts := &sys_exec.KillProcessOptions{ - Timeout: r.svc.GetCancelTimeout(), - Force: true, - } - if err := sys_exec.KillProcess(r.cmd, opts); err != nil { - return err - } - - // make sure the process does not exist - op := func() error { - if exists, _ := process.PidExists(int32(r.pid)); exists { - return errors.ErrorTaskProcessStillExists - } - return nil - } - ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) - defer cancel() - b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) - if err := backoff.Retry(op, b); err != nil { - return trace.TraceError(errors.ErrorTaskUnableToCancel) - } - - return nil -} - -// CleanUp clean up task runner -func (r *Runner) CleanUp() (err error) { - return nil -} - -func (r *Runner) SetSubscribeTimeout(timeout time.Duration) { - r.subscribeTimeout = timeout -} - -func (r *Runner) GetTaskId() (id primitive.ObjectID) { - return r.tid -} - -func (r *Runner) configureCmd() { - var cmdStr string - - // customized spider - if r.t.GetCmd() == "" { - cmdStr = r.s.GetCmd() - } else { - cmdStr = r.t.GetCmd() - } - - // parameters - if r.t.GetParam() != "" { - cmdStr += " " + r.t.GetParam() - } else if r.s.GetParam() != "" { - cmdStr += " " + r.s.GetParam() - } - - // get cmd instance - r.cmd = sys_exec.BuildCmd(cmdStr) - - // set working directory - r.cmd.Dir = r.cwd - - // configure pgid to allow killing sub processes - //sys_exec.SetPgid(r.cmd) -} - -func (r *Runner) configureLogging() { - // set stdout reader - stdout, _ := r.cmd.StdoutPipe() - r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) - - // set stderr reader - stderr, _ := r.cmd.StderrPipe() - r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) -} - -func (r *Runner) startLogging() { - // start reading stdout - go r.startLoggingReaderStdout() - - // start reading stderr - go r.startLoggingReaderStderr() -} - -func (r *Runner) startLoggingReaderStdout() { - for { - line, err := r.scannerStdout.ReadString(byte('\n')) - if err != nil { - break - } - line = strings.TrimSuffix(line, "\n") - r.writeLogLines([]string{line}) - } -} - -func (r *Runner) startLoggingReaderStderr() { - for { - line, err := r.scannerStderr.ReadString(byte('\n')) - if err != nil { - break - } - line = strings.TrimSuffix(line, "\n") - r.writeLogLines([]string{line}) - } -} - -func (r *Runner) startHealthCheck() { - if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { - return - } - for { - exists, _ := process.PidExists(int32(r.pid)) - if !exists { - // process lost - r.ch <- constants.TaskSignalLost - return - } - time.Sleep(1 * time.Second) - } -} - -func (r *Runner) configureEnv() { - // 默认把Node.js的全局node_modules加入环境变量 - envPath := os.Getenv("PATH") - nodePath := "/usr/lib/node_modules" - if !strings.Contains(envPath, nodePath) { - _ = os.Setenv("PATH", nodePath+":"+envPath) - } - _ = os.Setenv("NODE_PATH", nodePath) - - // default envs - r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) - if viper.GetString("grpc.address") != "" { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) - } - if viper.GetString("grpc.authKey") != "" { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) - } else { - r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) - } - - // global environment variables - envs, err := r.svc.GetModelEnvironmentService().GetEnvironmentList(nil, nil) - if err != nil { - trace.PrintError(err) - return - } - for _, env := range envs { - r.cmd.Env = append(r.cmd.Env, env.GetKey()+"="+env.GetValue()) - } -} - -func (r *Runner) syncFiles() (err error) { - masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.GetId().Hex()) - workspacePath := viper.GetString("workspace") - workerDir := filepath.Join(workspacePath, r.s.GetId().Hex()) - - // get file list from master - resp, err := http.Get(masterURL + "/scan") - if err != nil { - fmt.Println("Error getting file list from master:", err) - return trace.TraceError(err) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - fmt.Println("Error reading response body:", err) - return trace.TraceError(err) - } - var masterFiles map[string]entity.FsFileInfo - err = json.Unmarshal(body, &masterFiles) - if err != nil { - fmt.Println("Error unmarshaling JSON:", err) - return trace.TraceError(err) - } - - // create a map for master files - masterFilesMap := make(map[string]entity.FsFileInfo) - for _, file := range masterFiles { - masterFilesMap[file.Path] = file - } - - // create worker directory if not exists - if _, err := os.Stat(workerDir); os.IsNotExist(err) { - if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { - fmt.Println("Error creating worker directory:", err) - return trace.TraceError(err) - } - } - - // get file list from worker - workerFiles, err := utils.ScanDirectory(workerDir) - if err != nil { - fmt.Println("Error scanning worker directory:", err) - return trace.TraceError(err) - } - - // set up wait group and error channel - var wg sync.WaitGroup - errCh := make(chan error, 1) - - // delete files that are deleted on master node - for path, workerFile := range workerFiles { - if _, exists := masterFilesMap[path]; !exists { - fmt.Println("Deleting file:", path) - err := os.Remove(workerFile.FullPath) - if err != nil { - fmt.Println("Error deleting file:", err) - } - } - } - - // download files that are new or modified on master node - for path, masterFile := range masterFilesMap { - workerFile, exists := workerFiles[path] - if !exists || masterFile.Hash != workerFile.Hash { - wg.Add(1) - go func(path string, masterFile entity.FsFileInfo) { - defer wg.Done() - logrus.Infof("File needs to be synchronized: %s", path) - err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) - if err != nil { - logrus.Errorf("Error downloading file: %v", err) - select { - case errCh <- err: - default: - } - } - }(path, masterFile) - } - } - - wg.Wait() - close(errCh) - if err := <-errCh; err != nil { - return err - } - - return nil -} - -func (r *Runner) downloadFile(url string, filePath string) error { - resp, err := http.Get(url) - if err != nil { - return err - } - defer resp.Body.Close() - - out, err := os.Create(filePath) - if err != nil { - return err - } - defer out.Close() - - _, err = io.Copy(out, resp.Body) - return err -} - -// wait for process to finish and send task signal (constants.TaskSignal) -// to task runner's channel (Runner.ch) according to exit code -func (r *Runner) wait() { - // wait for process to finish - if err := r.cmd.Wait(); err != nil { - exitError, ok := err.(*exec.ExitError) - if !ok { - r.ch <- constants.TaskSignalError - return - } - exitCode := exitError.ExitCode() - if exitCode == -1 { - // cancel error - r.ch <- constants.TaskSignalCancel - return - } - - // standard error - r.err = err - r.ch <- constants.TaskSignalError - return - } - - // success - r.ch <- constants.TaskSignalFinish -} - -// updateTask update and get updated info of task (Runner.t) -func (r *Runner) updateTask(status string, e error) (err error) { - if r.t != nil && status != "" { - // update task status - r.t.SetStatus(status) - if e != nil { - r.t.SetError(e.Error()) - } - if r.svc.GetNodeConfigService().IsMaster() { - if err := delegate.NewModelDelegate(r.t).Save(); err != nil { - return err - } - } else { - if err := client.NewModelDelegate(r.t, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { - return err - } - } - - // send notification - go r.sendNotification() - - // update stats - go func() { - r._updateTaskStat(status) - r._updateSpiderStat(status) - }() - } - - // get task - r.t, err = r.svc.GetTaskById(r.tid) - if err != nil { - return err - } - - return nil -} - -func (r *Runner) initSub() (err error) { - r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) - if err != nil { - return trace.TraceError(err) - } - return nil -} - -func (r *Runner) writeLogLines(lines []string) { - data, err := json.Marshal(&entity.StreamMessageTaskData{ - TaskId: r.tid, - Logs: lines, - }) - if err != nil { - trace.PrintError(err) - return - } - msg := &grpc.StreamMessage{ - Code: grpc.StreamMessageCode_INSERT_LOGS, - Data: data, - } - if err := r.sub.Send(msg); err != nil { - trace.PrintError(err) - return - } -} - -func (r *Runner) _updateTaskStat(status string) { - ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) - if err != nil { - trace.PrintError(err) - return - } - switch status { - case constants.TaskStatusPending: - // do nothing - case constants.TaskStatusRunning: - ts.SetStartTs(time.Now()) - ts.SetWaitDuration(ts.GetStartTs().Sub(ts.GetCreateTs()).Milliseconds()) - case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: - ts.SetEndTs(time.Now()) - ts.SetRuntimeDuration(ts.GetEndTs().Sub(ts.GetStartTs()).Milliseconds()) - ts.SetTotalDuration(ts.GetEndTs().Sub(ts.GetCreateTs()).Milliseconds()) - } - if r.svc.GetNodeConfigService().IsMaster() { - if err := delegate.NewModelDelegate(ts).Save(); err != nil { - trace.PrintError(err) - return - } - } else { - if err := client.NewModelDelegate(ts, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { - trace.PrintError(err) - return - } - } -} - -func (r *Runner) sendNotification() { - data, err := json.Marshal(r.t) - if err != nil { - trace.PrintError(err) - return - } - req := &grpc.Request{ - NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), - Data: data, - } - _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) - if err != nil { - trace.PrintError(err) - return - } -} - -func (r *Runner) _updateSpiderStat(status string) { - // task stat - ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) - if err != nil { - trace.PrintError(err) - return - } - - // update - var update bson.M - switch status { - case constants.TaskStatusPending, constants.TaskStatusRunning: - update = bson.M{ - "$set": bson.M{ - "last_task_id": r.tid, // last task id - }, - "$inc": bson.M{ - "tasks": 1, // task count - "wait_duration": ts.GetWaitDuration(), // wait duration - }, - } - case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: - update = bson.M{ - "$inc": bson.M{ - "results": ts.GetResultCount(), // results - "runtime_duration": ts.GetRuntimeDuration() / 1000, // runtime duration - "total_duration": ts.GetTotalDuration() / 1000, // total duration - }, - } - default: - trace.PrintError(errors.ErrorTaskInvalidType) - return - } - - // perform update - if r.svc.GetNodeConfigService().IsMaster() { - if err := mongo.GetMongoCol(interfaces.ModelColNameSpiderStat).UpdateId(r.s.GetId(), update); err != nil { - trace.PrintError(err) - return - } - } else { - modelSvc, err := client.NewBaseServiceDelegate( - client.WithBaseServiceModelId(interfaces.ModelIdSpiderStat), - client.WithBaseServiceConfigPath(r.svc.GetConfigPath()), - ) - if err != nil { - trace.PrintError(err) - return - } - if err := modelSvc.UpdateById(r.s.GetId(), update); err != nil { - trace.PrintError(err) - return - } - } - -} - -func NewTaskRunner(id primitive.ObjectID, svc interfaces.TaskHandlerService, opts ...RunnerOption) (r2 interfaces.TaskRunner, err error) { - // validate options - if id.IsZero() { - return nil, constants.ErrInvalidOptions - } - - // runner - r := &Runner{ - subscribeTimeout: 30 * time.Second, - bufferSize: 1024 * 1024, - svc: svc, - tid: id, - ch: make(chan constants.TaskSignal), - logBatchSize: 20, - } - - // apply options - for _, opt := range opts { - opt(r) - } - - // task - r.t, err = svc.GetTaskById(id) - if err != nil { - return nil, err - } - - // spider - r.s, err = svc.GetSpiderById(r.t.GetSpiderId()) - if err != nil { - return nil, err - } - - // task fs service - r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.GetId().Hex())) - - // dependency injection - if err := container.GetContainer().Invoke(func( - c interfaces.GrpcClient, - ) { - r.c = c - }); err != nil { - return nil, trace.TraceError(err) - } - - _ = container.GetContainer().Invoke(func(hookSvc interfaces.TaskHookService) { - r.hookSvc = hookSvc - }) - - // initialize task runner - if err := r.Init(); err != nil { - return r, err - } - - return r, nil -} diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go index 7424a2c1..3788661f 100644 --- a/core/task/handler/runner_v2.go +++ b/core/task/handler/runner_v2.go @@ -4,24 +4,24 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "github.com/apex/log" "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/entity" - "github.com/crawlab-team/crawlab/core/errors" fs2 "github.com/crawlab-team/crawlab/core/fs" + client2 "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" service2 "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/sys_exec" "github.com/crawlab-team/crawlab/core/utils" - grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" "github.com/shirou/gopsutil/process" - "github.com/sirupsen/logrus" "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -48,13 +48,13 @@ type RunnerV2 struct { cmd *exec.Cmd // process command instance pid int // process id tid primitive.ObjectID // task id - t *models.TaskV2 // task model.Task - s *models.SpiderV2 // spider model.Spider + t *models2.TaskV2 // task model.Task + s *models2.SpiderV2 // spider model.Spider ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 err error // standard process error envs []models.Env // environment variables cwd string // working directory - c interfaces.GrpcClient // grpc client + c *client2.GrpcClientV2 // grpc client sub grpc.TaskService_SubscribeClient // grpc task service stream client // log internals @@ -71,16 +71,8 @@ func (r *RunnerV2) Init() (err error) { // start grpc client if !r.c.IsStarted() { - r.c.Start() - } - - // working directory - workspacePath := viper.GetString("workspace") - r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) - - // sync files from master - if !utils.IsMaster() { - if err := r.syncFiles(); err != nil { + err := r.c.Start() + if err != nil { return err } } @@ -97,8 +89,21 @@ func (r *RunnerV2) Run() (err error) { // log task started log.Infof("task[%s] started", r.tid.Hex()) + // configure working directory + r.configureCwd() + + // sync files worker nodes + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return r.updateTask(constants.TaskStatusError, err) + } + } + // configure cmd - r.configureCmd() + err = r.configureCmd() + if err != nil { + return r.updateTask(constants.TaskStatusError, err) + } // configure environment variables r.configureEnv() @@ -176,7 +181,7 @@ func (r *RunnerV2) Cancel() (err error) { // make sure the process does not exist op := func() error { if exists, _ := process.PidExists(int32(r.pid)); exists { - return errors.ErrorTaskProcessStillExists + return errors.New(fmt.Sprintf("task process %d still exists", r.pid)) } return nil } @@ -184,7 +189,8 @@ func (r *RunnerV2) Cancel() (err error) { defer cancel() b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) if err := backoff.Retry(op, b); err != nil { - return trace.TraceError(errors.ErrorTaskUnableToCancel) + log.Errorf("Error canceling task %s: %v", r.tid, err) + return trace.TraceError(err) } return nil @@ -203,7 +209,7 @@ func (r *RunnerV2) GetTaskId() (id primitive.ObjectID) { return r.tid } -func (r *RunnerV2) configureCmd() { +func (r *RunnerV2) configureCmd() (err error) { var cmdStr string // customized spider @@ -221,13 +227,17 @@ func (r *RunnerV2) configureCmd() { } // get cmd instance - r.cmd = sys_exec.BuildCmd(cmdStr) + r.cmd, err = sys_exec.BuildCmd(cmdStr) + if err != nil { + log.Errorf("Error building command: %v", err) + trace.PrintError(err) + return err + } // set working directory r.cmd.Dir = r.cwd - // configure pgid to allow killing sub processes - //sys_exec.SetPgid(r.cmd) + return nil } func (r *RunnerV2) configureLogging() { @@ -306,7 +316,7 @@ func (r *RunnerV2) configureEnv() { } // global environment variables - envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) + envs, err := client.NewModelServiceV2[models2.EnvironmentV2]().GetMany(nil, nil) if err != nil { trace.PrintError(err) return @@ -317,26 +327,33 @@ func (r *RunnerV2) configureEnv() { } func (r *RunnerV2) syncFiles() (err error) { - masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.Id.Hex()) - workspacePath := viper.GetString("workspace") - workerDir := filepath.Join(workspacePath, r.s.Id.Hex()) + var id string + var workingDir string + if r.s.GitId.IsZero() { + id = r.s.Id.Hex() + workingDir = "" + } else { + id = r.s.GitId.Hex() + workingDir = r.s.GitRootPath + } + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), id) // get file list from master - resp, err := http.Get(masterURL + "/scan") + resp, err := http.Get(masterURL + "/scan?path=" + workingDir) if err != nil { - fmt.Println("Error getting file list from master:", err) + log.Errorf("Error getting file list from master: %v", err) return trace.TraceError(err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - fmt.Println("Error reading response body:", err) + log.Errorf("Error reading response body: %v", err) return trace.TraceError(err) } var masterFiles map[string]entity.FsFileInfo err = json.Unmarshal(body, &masterFiles) if err != nil { - fmt.Println("Error unmarshaling JSON:", err) + log.Errorf("Error unmarshaling JSON: %v", err) return trace.TraceError(err) } @@ -346,80 +363,115 @@ func (r *RunnerV2) syncFiles() (err error) { masterFilesMap[file.Path] = file } - // create worker directory if not exists - if _, err := os.Stat(workerDir); os.IsNotExist(err) { - if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { - fmt.Println("Error creating worker directory:", err) + // create working directory if not exists + if _, err := os.Stat(r.cwd); os.IsNotExist(err) { + if err := os.MkdirAll(r.cwd, os.ModePerm); err != nil { + log.Errorf("Error creating worker directory: %v", err) return trace.TraceError(err) } } // get file list from worker - workerFiles, err := utils.ScanDirectory(workerDir) + workerFiles, err := utils.ScanDirectory(r.cwd) if err != nil { - fmt.Println("Error scanning worker directory:", err) + log.Errorf("Error scanning worker directory: %v", err) return trace.TraceError(err) } - // set up wait group and error channel - var wg sync.WaitGroup - errCh := make(chan error, 1) - // delete files that are deleted on master node for path, workerFile := range workerFiles { if _, exists := masterFilesMap[path]; !exists { - fmt.Println("Deleting file:", path) + log.Infof("Deleting file: %s", path) err := os.Remove(workerFile.FullPath) if err != nil { - fmt.Println("Error deleting file:", err) + log.Errorf("Error deleting file: %v", err) } } } + // set up wait group and error channel + var wg sync.WaitGroup + pool := make(chan struct{}, 10) + // download files that are new or modified on master node for path, masterFile := range masterFilesMap { workerFile, exists := workerFiles[path] if !exists || masterFile.Hash != workerFile.Hash { wg.Add(1) - go func(path string, masterFile entity.FsFileInfo) { + + // acquire token + pool <- struct{}{} + + // start goroutine to synchronize file or directory + go func(path string, masterFile *entity.FsFileInfo) { defer wg.Done() - logrus.Infof("File needs to be synchronized: %s", path) - err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) - if err != nil { - logrus.Errorf("Error downloading file: %v", err) - select { - case errCh <- err: - default: + + if masterFile.IsDir { + log.Infof("Directory needs to be synchronized: %s", path) + _err := os.MkdirAll(filepath.Join(r.cwd, path), masterFile.Mode) + if _err != nil { + log.Errorf("Error creating directory: %v", _err) + err = errors.Join(err, _err) + } + } else { + log.Infof("File needs to be synchronized: %s", path) + _err := r.downloadFile(masterURL+"/download?path="+filepath.Join(workingDir, path), filepath.Join(r.cwd, path), masterFile) + if _err != nil { + log.Errorf("Error downloading file: %v", _err) + err = errors.Join(err, _err) } } - }(path, masterFile) + + // release token + <-pool + + }(path, &masterFile) } } + // wait for all files and directories to be synchronized wg.Wait() - close(errCh) - if err := <-errCh; err != nil { - return err - } - return nil + return err } -func (r *RunnerV2) downloadFile(url string, filePath string) error { +func (r *RunnerV2) downloadFile(url string, filePath string, fileInfo *entity.FsFileInfo) error { + // get file response resp, err := http.Get(url) if err != nil { + log.Errorf("Error getting file response: %v", err) return err } + if resp.StatusCode != http.StatusOK { + log.Errorf("Error downloading file: %s", resp.Status) + return errors.New(resp.Status) + } defer resp.Body.Close() - out, err := os.Create(filePath) + // create directory if not exists + dirPath := filepath.Dir(filePath) + utils.Exists(dirPath) + err = os.MkdirAll(dirPath, os.ModePerm) if err != nil { + log.Errorf("Error creating directory: %v", err) + return err + } + + // create local file + out, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fileInfo.Mode) + if err != nil { + log.Errorf("Error creating file: %v", err) return err } defer out.Close() + // copy file content to local file _, err = io.Copy(out, resp.Body) - return err + if err != nil { + log.Errorf("Error copying file: %v", err) + return err + } + return nil } // wait for process to finish and send task signal (constants.TaskSignal) @@ -427,7 +479,8 @@ func (r *RunnerV2) downloadFile(url string, filePath string) error { func (r *RunnerV2) wait() { // wait for process to finish if err := r.cmd.Wait(); err != nil { - exitError, ok := err.(*exec.ExitError) + var exitError *exec.ExitError + ok := errors.As(err, &exitError) if !ok { r.ch <- constants.TaskSignalError return @@ -458,25 +511,23 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { r.t.Error = e.Error() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = service2.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } else { - err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + err = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(r.t.Id, *r.t) if err != nil { return err } } + // update stats + r._updateTaskStat(status) + r._updateSpiderStat(status) + // send notification go r.sendNotification() - - // update stats - go func() { - r._updateTaskStat(status) - r._updateSpiderStat(status) - }() } // get task @@ -489,7 +540,7 @@ func (r *RunnerV2) updateTask(status string, e error) (err error) { } func (r *RunnerV2) initSub() (err error) { - r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + r.sub, err = r.c.TaskClient.Subscribe(context.Background()) if err != nil { return trace.TraceError(err) } @@ -516,7 +567,7 @@ func (r *RunnerV2) writeLogLines(lines []string) { } func (r *RunnerV2) _updateTaskStat(status string) { - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -526,24 +577,24 @@ func (r *RunnerV2) _updateTaskStat(status string) { // do nothing case constants.TaskStatusRunning: ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: if ts.StartTs.IsZero() { ts.StartTs = time.Now() - ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + ts.WaitDuration = ts.StartTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } ts.EndTs = time.Now() ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() - ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.BaseModelV2.CreatedAt).Milliseconds() } if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = service2.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + err = client.NewModelServiceV2[models2.TaskStatV2]().ReplaceById(ts.Id, *ts) if err != nil { trace.PrintError(err) return @@ -552,17 +603,13 @@ func (r *RunnerV2) _updateTaskStat(status string) { } func (r *RunnerV2) sendNotification() { - data, err := json.Marshal(r.t) - if err != nil { - trace.PrintError(err) - return - } - req := &grpc.Request{ + req := &grpc.TaskServiceSendNotificationRequest{ NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), - Data: data, + TaskId: r.tid.Hex(), } - _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + _, err := r.c.TaskClient.SendNotification(context.Background(), req) if err != nil { + log.Errorf("Error sending notification: %v", err) trace.PrintError(err) return } @@ -570,7 +617,7 @@ func (r *RunnerV2) sendNotification() { func (r *RunnerV2) _updateSpiderStat(status string) { // task stat - ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + ts, err := client.NewModelServiceV2[models2.TaskStatV2]().GetById(r.tid) if err != nil { trace.PrintError(err) return @@ -601,25 +648,36 @@ func (r *RunnerV2) _updateSpiderStat(status string) { }, } default: - trace.PrintError(errors.ErrorTaskInvalidType) + log.Errorf("Invalid task status: %s", status) + trace.PrintError(errors.New("invalid task status")) return } // perform update if r.svc.GetNodeConfigService().IsMaster() { - err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = service2.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } else { - err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + err = client.NewModelServiceV2[models2.SpiderStatV2]().UpdateById(r.s.Id, update) if err != nil { trace.PrintError(err) return } } +} +func (r *RunnerV2) configureCwd() { + workspacePath := viper.GetString("workspace") + if r.s.GitId.IsZero() { + // not git + r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) + } else { + // git + r.cwd = filepath.Join(workspacePath, r.s.GitId.Hex(), r.s.GitRootPath) + } } func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err error) { @@ -653,14 +711,8 @@ func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err e // task fs service r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) - // dependency injection - if err := container.GetContainer().Invoke(func( - c interfaces.GrpcClient, - ) { - r.c = c - }); err != nil { - return nil, trace.TraceError(err) - } + // grpc client + r.c = client2.GetGrpcClientV2() // initialize task runner if err := r.Init(); err != nil { diff --git a/core/task/handler/service.go b/core/task/handler/service.go deleted file mode 100644 index 69a99884..00000000 --- a/core/task/handler/service.go +++ /dev/null @@ -1,506 +0,0 @@ -package handler - -import ( - "context" - "encoding/json" - "errors" - "github.com/apex/log" - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - errors2 "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "sync" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - cfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - clientModelSvc interfaces.GrpcClientModelService - clientModelNodeSvc interfaces.GrpcClientModelNodeService - clientModelSpiderSvc interfaces.GrpcClientModelSpiderService - clientModelTaskSvc interfaces.GrpcClientModelTaskService - clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService - clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService - c interfaces.GrpcClient // grpc client - - // settings - //maxRunners int - exitWatchDuration time.Duration - reportInterval time.Duration - fetchInterval time.Duration - fetchTimeout time.Duration - cancelTimeout time.Duration - - // internals variables - stopped bool - mu sync.Mutex - runners sync.Map // pool of task runners started - syncLocks sync.Map // files sync locks map of task runners -} - -func (svc *Service) Start() { - // Initialize gRPC if not started - if !svc.c.IsStarted() { - svc.c.Start() - } - - go svc.ReportStatus() - go svc.Fetch() -} - -func (svc *Service) Run(taskId primitive.ObjectID) (err error) { - return svc.run(taskId) -} - -func (svc *Service) Reset() { - svc.mu.Lock() - defer svc.mu.Unlock() -} - -func (svc *Service) Cancel(taskId primitive.ObjectID) (err error) { - r, err := svc.getRunner(taskId) - if err != nil { - return err - } - if err := r.Cancel(); err != nil { - return err - } - return nil -} - -func (svc *Service) Fetch() { - for { - // wait - time.Sleep(svc.fetchInterval) - - // current node - n, err := svc.GetCurrentNode() - if err != nil { - continue - } - - // skip if node is not active or enabled - if !n.GetActive() || !n.GetEnabled() { - continue - } - - // validate if there are available runners - if svc.getRunnerCount() >= n.GetMaxRunners() { - continue - } - - // stop - if svc.stopped { - return - } - - // fetch task - tid, err := svc.fetch() - if err != nil { - trace.PrintError(err) - continue - } - - // skip if no task id - if tid.IsZero() { - continue - } - - // run task - if err := svc.run(tid); err != nil { - trace.PrintError(err) - t, err := svc.GetTaskById(tid) - if err == nil && t.GetStatus() != constants.TaskStatusCancelled { - t.SetError(err.Error()) - _ = svc.SaveTask(t, constants.TaskStatusError) - continue - } - continue - } - } -} - -func (svc *Service) ReportStatus() { - for { - if svc.stopped { - return - } - - // report handler status - if err := svc.reportStatus(); err != nil { - trace.PrintError(err) - } - - // wait - time.Sleep(svc.reportInterval) - } -} - -func (svc *Service) IsSyncLocked(path string) (ok bool) { - _, ok = svc.syncLocks.Load(path) - return ok -} - -func (svc *Service) LockSync(path string) { - svc.syncLocks.Store(path, true) -} - -func (svc *Service) UnlockSync(path string) { - svc.syncLocks.Delete(path) -} - -//func (svc *Service) GetMaxRunners() (maxRunners int) { -// return svc.maxRunners -//} -// -//func (svc *Service) SetMaxRunners(maxRunners int) { -// svc.maxRunners = maxRunners -//} - -func (svc *Service) GetExitWatchDuration() (duration time.Duration) { - return svc.exitWatchDuration -} - -func (svc *Service) SetExitWatchDuration(duration time.Duration) { - svc.exitWatchDuration = duration -} - -func (svc *Service) GetFetchInterval() (interval time.Duration) { - return svc.fetchInterval -} - -func (svc *Service) SetFetchInterval(interval time.Duration) { - svc.fetchInterval = interval -} - -func (svc *Service) GetReportInterval() (interval time.Duration) { - return svc.reportInterval -} - -func (svc *Service) SetReportInterval(interval time.Duration) { - svc.reportInterval = interval -} - -func (svc *Service) GetCancelTimeout() (timeout time.Duration) { - return svc.cancelTimeout -} - -func (svc *Service) SetCancelTimeout(timeout time.Duration) { - svc.cancelTimeout = timeout -} - -func (svc *Service) GetModelService() (modelSvc interfaces.GrpcClientModelService) { - return svc.clientModelSvc -} - -func (svc *Service) GetModelSpiderService() (modelSpiderSvc interfaces.GrpcClientModelSpiderService) { - return svc.clientModelSpiderSvc -} - -func (svc *Service) GetModelTaskService() (modelTaskSvc interfaces.GrpcClientModelTaskService) { - return svc.clientModelTaskSvc -} - -func (svc *Service) GetModelTaskStatService() (modelTaskSvc interfaces.GrpcClientModelTaskStatService) { - return svc.clientModelTaskStatSvc -} - -func (svc *Service) GetModelEnvironmentService() (modelTaskSvc interfaces.GrpcClientModelEnvironmentService) { - return svc.clientModelEnvironmentSvc -} - -func (svc *Service) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { - return svc.cfgSvc -} - -func (svc *Service) GetCurrentNode() (n interfaces.Node, err error) { - // node key - nodeKey := svc.cfgSvc.GetNodeKey() - - // current node - if svc.cfgSvc.IsMaster() { - n, err = svc.modelSvc.GetNodeByKey(nodeKey, nil) - } else { - n, err = svc.clientModelNodeSvc.GetNodeByKey(nodeKey) - } - if err != nil { - return nil, err - } - - return n, nil -} - -func (svc *Service) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { - if svc.cfgSvc.IsMaster() { - t, err = svc.modelSvc.GetTaskById(id) - } else { - t, err = svc.clientModelTaskSvc.GetTaskById(id) - } - if err != nil { - return nil, err - } - - return t, nil -} - -func (svc *Service) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { - if svc.cfgSvc.IsMaster() { - s, err = svc.modelSvc.GetSpiderById(id) - } else { - s, err = svc.clientModelSpiderSvc.GetSpiderById(id) - } - if err != nil { - return nil, err - } - - return s, nil -} - -func (svc *Service) getRunners() (runners []*Runner) { - svc.mu.Lock() - defer svc.mu.Unlock() - svc.runners.Range(func(key, value interface{}) bool { - r := value.(Runner) - runners = append(runners, &r) - return true - }) - return runners -} - -func (svc *Service) getRunnerCount() (count int) { - n, err := svc.GetCurrentNode() - if err != nil { - trace.PrintError(err) - return - } - query := bson.M{ - "node_id": n.GetId(), - "status": constants.TaskStatusRunning, - } - if svc.cfgSvc.IsMaster() { - count, err = svc.modelSvc.GetBaseService(interfaces.ModelIdTask).Count(query) - if err != nil { - trace.PrintError(err) - return - } - } else { - count, err = svc.clientModelTaskSvc.Count(query) - if err != nil { - trace.PrintError(err) - return - } - } - return count -} - -func (svc *Service) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { - log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) - v, ok := svc.runners.Load(taskId) - if !ok { - return nil, trace.TraceError(errors2.ErrorTaskNotExists) - } - switch v.(type) { - case interfaces.TaskRunner: - r = v.(interfaces.TaskRunner) - default: - return nil, trace.TraceError(errors2.ErrorModelInvalidType) - } - return r, nil -} - -func (svc *Service) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { - log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) - svc.runners.Store(taskId, r) -} - -func (svc *Service) deleteRunner(taskId primitive.ObjectID) { - log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) - svc.runners.Delete(taskId) -} - -func (svc *Service) saveTask(t interfaces.Task, status string) (err error) { - // normalize status - if status == "" { - status = constants.TaskStatusPending - } - - // set task status - t.SetStatus(status) - - // attempt to get task from database - _, err = svc.clientModelTaskSvc.GetTaskById(t.GetId()) - if err != nil { - // if task does not exist, add to database - if err == mongo.ErrNoDocuments { - if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Add(); err != nil { - return err - } - return nil - } else { - return err - } - } else { - // otherwise, update - if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Save(); err != nil { - return err - } - return nil - } -} - -func (svc *Service) reportStatus() (err error) { - // current node - n, err := svc.GetCurrentNode() - if err != nil { - return err - } - - // available runners of handler - ar := n.GetMaxRunners() - svc.getRunnerCount() - - // set available runners - n.SetAvailableRunners(ar) - - // save node - if svc.cfgSvc.IsMaster() { - err = delegate.NewModelDelegate(n).Save() - } else { - err = client.NewModelDelegate(n, client.WithDelegateConfigPath(svc.GetConfigPath())).Save() - } - if err != nil { - return err - } - - return nil -} - -func (svc *Service) fetch() (tid primitive.ObjectID, err error) { - ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) - defer cancel() - res, err := svc.c.GetTaskClient().Fetch(ctx, svc.c.NewRequest(nil)) - if err != nil { - return tid, trace.TraceError(err) - } - if err := json.Unmarshal(res.Data, &tid); err != nil { - return tid, trace.TraceError(err) - } - return tid, nil -} - -func (svc *Service) run(taskId primitive.ObjectID) (err error) { - // attempt to get runner from pool - _, ok := svc.runners.Load(taskId) - if ok { - return trace.TraceError(errors2.ErrorTaskAlreadyExists) - } - - // create a new task runner - r, err := NewTaskRunner(taskId, svc) - if err != nil { - return trace.TraceError(err) - } - - // add runner to pool - svc.addRunner(taskId, r) - - // create a goroutine to run task - go func() { - // delete runner from pool - defer svc.deleteRunner(r.GetTaskId()) - defer func(r interfaces.TaskRunner) { - err := r.CleanUp() - if err != nil { - log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) - } - }(r) - // run task process (blocking) - // error or finish after task runner ends - if err := r.Run(); err != nil { - switch { - case errors.Is(err, constants.ErrTaskError): - log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) - case errors.Is(err, constants.ErrTaskCancelled): - log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) - default: - log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) - } - } - log.Infof("task[%s] finished", r.GetTaskId().Hex()) - }() - - return nil -} - -func NewTaskHandlerService() (svc2 interfaces.TaskHandlerService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - TaskBaseService: baseSvc, - exitWatchDuration: 60 * time.Second, - fetchInterval: 1 * time.Second, - fetchTimeout: 15 * time.Second, - reportInterval: 5 * time.Second, - cancelTimeout: 5 * time.Second, - mu: sync.Mutex{}, - runners: sync.Map{}, - syncLocks: sync.Map{}, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - cfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - clientModelSvc interfaces.GrpcClientModelService, - clientModelNodeSvc interfaces.GrpcClientModelNodeService, - clientModelSpiderSvc interfaces.GrpcClientModelSpiderService, - clientModelTaskSvc interfaces.GrpcClientModelTaskService, - clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService, - clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService, - c interfaces.GrpcClient, - ) { - svc.cfgSvc = cfgSvc - svc.modelSvc = modelSvc - svc.clientModelSvc = clientModelSvc - svc.clientModelNodeSvc = clientModelNodeSvc - svc.clientModelSpiderSvc = clientModelSpiderSvc - svc.clientModelTaskSvc = clientModelTaskSvc - svc.clientModelTaskStatSvc = clientModelTaskStatSvc - svc.clientModelEnvironmentSvc = clientModelEnvironmentSvc - svc.c = c - }); err != nil { - return nil, trace.TraceError(err) - } - - log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) - - return svc, nil -} - -var _service interfaces.TaskHandlerService - -func GetTaskHandlerService() (svr interfaces.TaskHandlerService, err error) { - if _service != nil { - return _service, nil - } - _service, err = NewTaskHandlerService() - if err != nil { - return nil, err - } - return _service, nil -} diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go index ad0dcdd9..28c2c73c 100644 --- a/core/task/handler/service_v2.go +++ b/core/task/handler/service_v2.go @@ -10,7 +10,7 @@ import ( grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" "github.com/crawlab-team/crawlab/core/interfaces" "github.com/crawlab-team/crawlab/core/models/client" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" "github.com/crawlab-team/crawlab/trace" @@ -43,7 +43,10 @@ type ServiceV2 struct { func (svc *ServiceV2) Start() { // Initialize gRPC if not started if !svc.c.IsStarted() { - svc.c.Start() + err := svc.c.Start() + if err != nil { + return + } } go svc.ReportStatus() @@ -71,9 +74,10 @@ func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) { } func (svc *ServiceV2) Fetch() { + ticker := time.NewTicker(svc.fetchInterval) for { // wait - time.Sleep(svc.fetchInterval) + <-ticker.C // current node n, err := svc.GetCurrentNode() @@ -93,6 +97,7 @@ func (svc *ServiceV2) Fetch() { // stop if svc.stopped { + ticker.Stop() return } @@ -112,10 +117,11 @@ func (svc *ServiceV2) Fetch() { if err := svc.run(tid); err != nil { trace.PrintError(err) t, err := svc.GetTaskById(tid) - if err == nil && t.Status != constants.TaskStatusCancelled { + if err != nil && t.Status != constants.TaskStatusCancelled { t.Error = err.Error() t.Status = constants.TaskStatusError t.SetUpdated(t.CreatedBy) + _ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) continue } continue @@ -196,15 +202,15 @@ func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigServic return svc.cfgSvc } -func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { +func (svc *ServiceV2) GetCurrentNode() (n *models2.NodeV2, err error) { // node key nodeKey := svc.cfgSvc.GetNodeKey() // current node if svc.cfgSvc.IsMaster() { - n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = service.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } else { - n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + n, err = client.NewModelServiceV2[models2.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) } if err != nil { return nil, err @@ -213,11 +219,11 @@ func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { return n, nil } -func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) { +func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models2.TaskV2, err error) { if svc.cfgSvc.IsMaster() { - t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = service.NewModelServiceV2[models2.TaskV2]().GetById(id) } else { - t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err = client.NewModelServiceV2[models2.TaskV2]().GetById(id) } if err != nil { return nil, err @@ -226,11 +232,11 @@ func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err return t, nil } -func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) { +func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2, err error) { if svc.cfgSvc.IsMaster() { - s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = service.NewModelServiceV2[models2.SpiderV2]().GetById(id) } else { - s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id) + s, err = client.NewModelServiceV2[models2.SpiderV2]().GetById(id) } if err != nil { return nil, err @@ -239,11 +245,11 @@ func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, return s, nil } -func (svc *ServiceV2) getRunners() (runners []*Runner) { +func (svc *ServiceV2) getRunners() (runners []*RunnerV2) { svc.mu.Lock() defer svc.mu.Unlock() svc.runners.Range(func(key, value interface{}) bool { - r := value.(Runner) + r := value.(RunnerV2) runners = append(runners, &r) return true }) @@ -261,13 +267,13 @@ func (svc *ServiceV2) getRunnerCount() (count int) { "status": constants.TaskStatusRunning, } if svc.cfgSvc.IsMaster() { - count, err = service.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = service.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return } } else { - count, err = client.NewModelServiceV2[models.TaskV2]().Count(query) + count, err = client.NewModelServiceV2[models2.TaskV2]().Count(query) if err != nil { trace.PrintError(err) return @@ -317,9 +323,9 @@ func (svc *ServiceV2) reportStatus() (err error) { // save node n.SetUpdated(n.CreatedBy) if svc.cfgSvc.IsMaster() { - err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = service.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } else { - err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + err = client.NewModelServiceV2[models2.NodeV2]().ReplaceById(n.Id, *n) } if err != nil { return err @@ -385,7 +391,7 @@ func (svc *ServiceV2) run(taskId primitive.ObjectID) (err error) { return nil } -func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { +func newTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { // service svc := &ServiceV2{ exitWatchDuration: 60 * time.Second, @@ -402,10 +408,7 @@ func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { svc.cfgSvc = nodeconfig.GetNodeConfigService() // grpc client - svc.c, err = grpcclient.NewGrpcClientV2() - if err != nil { - return nil, err - } + svc.c = grpcclient.GetGrpcClientV2() log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) @@ -413,12 +416,18 @@ func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { } var _serviceV2 *ServiceV2 +var _serviceV2Once = new(sync.Once) func GetTaskHandlerServiceV2() (svr *ServiceV2, err error) { if _serviceV2 != nil { return _serviceV2, nil } - _serviceV2, err = NewTaskHandlerServiceV2() + _serviceV2Once.Do(func() { + _serviceV2, err = newTaskHandlerServiceV2() + if err != nil { + log.Errorf("failed to create task handler service: %v", err) + } + }) if err != nil { return nil, err } diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go deleted file mode 100644 index 3644aa17..00000000 --- a/core/task/scheduler/service.go +++ /dev/null @@ -1,264 +0,0 @@ -package scheduler - -import ( - "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/errors" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/delegate" - "github.com/crawlab-team/crawlab/core/models/models" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/db/mongo" - grpc "github.com/crawlab-team/crawlab/grpc" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - mongo2 "go.mongodb.org/mongo-driver/mongo" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - svr interfaces.GrpcServer - handlerSvc interfaces.TaskHandlerService - - // settings - interval time.Duration -} - -func (svc *Service) Start() { - go svc.initTaskStatus() - go svc.cleanupTasks() - svc.Wait() - svc.Stop() -} - -func (svc *Service) Enqueue(t interfaces.Task) (t2 interfaces.Task, err error) { - // set task status - t.SetStatus(constants.TaskStatusPending) - - // user - var u *models.User - if !t.GetUserId().IsZero() { - u, _ = svc.modelSvc.GetUserById(t.GetUserId()) - } - - // add task - if err = delegate.NewModelDelegate(t, u).Add(); err != nil { - return nil, err - } - - // task queue item - tq := &models.TaskQueueItem{ - Id: t.GetId(), - Priority: t.GetPriority(), - NodeId: t.GetNodeId(), - } - - // task stat - ts := &models.TaskStat{ - Id: t.GetId(), - CreateTs: time.Now(), - } - - // enqueue task - _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Insert(tq) - if err != nil { - return nil, trace.TraceError(err) - } - - // add task stat - _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Insert(ts) - if err != nil { - return nil, trace.TraceError(err) - } - - // success - return t, nil -} - -func (svc *Service) Cancel(id primitive.ObjectID, args ...interface{}) (err error) { - // task - t, err := svc.modelSvc.GetTaskById(id) - if err != nil { - return trace.TraceError(err) - } - - // initial status - initialStatus := t.Status - - // set task status as "cancelled" - _ = svc.SaveTask(t, constants.TaskStatusCancelled) - - // set status of pending tasks as "cancelled" and remove from task item queue - if initialStatus == constants.TaskStatusPending { - // remove from task item queue - if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).DeleteId(t.GetId()); err != nil { - return trace.TraceError(err) - } - return nil - } - - // whether task is running on master node - isMasterTask, err := svc.isMasterNode(t) - if err != nil { - // when error, force status being set as "cancelled" - return svc.SaveTask(t, constants.TaskStatusCancelled) - } - - // node - n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) - if err != nil { - return trace.TraceError(err) - } - - if isMasterTask { - // cancel task on master - if err := svc.handlerSvc.Cancel(id); err != nil { - return trace.TraceError(err) - } - // cancel success - return nil - } else { - // send to cancel task on worker nodes - if err := svc.svr.SendStreamMessageWithData("node:"+n.GetKey(), grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { - return trace.TraceError(err) - } - // cancel success - return nil - } -} - -func (svc *Service) SetInterval(interval time.Duration) { - svc.interval = interval -} - -// initTaskStatus initialize task status of existing tasks -func (svc *Service) initTaskStatus() { - // set status of running tasks as TaskStatusAbnormal - runningTasks, err := svc.modelSvc.GetTaskList(bson.M{ - "status": bson.M{ - "$in": []string{ - constants.TaskStatusPending, - constants.TaskStatusRunning, - }, - }, - }, nil) - if err != nil { - if err == mongo2.ErrNoDocuments { - return - } - trace.PrintError(err) - } - for _, t := range runningTasks { - go func(t *models.Task) { - if err := svc.SaveTask(t, constants.TaskStatusAbnormal); err != nil { - trace.PrintError(err) - } - }(&t) - } - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskQueue).DeleteList(nil); err != nil { - return - } -} - -func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) { - if t.GetNodeId().IsZero() { - return false, trace.TraceError(errors.ErrorTaskNoNodeId) - } - n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) - if err != nil { - if err == mongo2.ErrNoDocuments { - return false, trace.TraceError(errors.ErrorTaskNodeNotFound) - } - return false, trace.TraceError(err) - } - return n.IsMaster, nil -} - -func (svc *Service) cleanupTasks() { - for { - // task stats over 30 days ago - taskStats, err := svc.modelSvc.GetTaskStatList(bson.M{ - "create_ts": bson.M{ - "$lt": time.Now().Add(-30 * 24 * time.Hour), - }, - }, nil) - if err != nil { - time.Sleep(30 * time.Minute) - continue - } - - // task ids - var ids []primitive.ObjectID - for _, ts := range taskStats { - ids = append(ids, ts.Id) - } - - if len(ids) > 0 { - // remove tasks - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTask).DeleteList(bson.M{ - "_id": bson.M{"$in": ids}, - }); err != nil { - trace.PrintError(err) - } - - // remove task stats - if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskStat).DeleteList(bson.M{ - "_id": bson.M{"$in": ids}, - }); err != nil { - trace.PrintError(err) - } - } - - time.Sleep(30 * time.Minute) - } -} - -func NewTaskSchedulerService() (svc2 interfaces.TaskSchedulerService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - TaskBaseService: baseSvc, - interval: 5 * time.Second, - } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - modelSvc service.ModelService, - svr interfaces.GrpcServer, - handlerSvc interfaces.TaskHandlerService, - ) { - svc.nodeCfgSvc = nodeCfgSvc - svc.modelSvc = modelSvc - svc.svr = svr - svc.handlerSvc = handlerSvc - }); err != nil { - return nil, err - } - - return svc, nil -} - -var svc interfaces.TaskSchedulerService - -func GetTaskSchedulerService() (svr interfaces.TaskSchedulerService, err error) { - if svc != nil { - return svc, nil - } - svc, err = NewTaskSchedulerService() - if err != nil { - return nil, err - } - return svc, nil -} diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go index fbf8ad40..ef4d7ac7 100644 --- a/core/task/scheduler/service_v2.go +++ b/core/task/scheduler/service_v2.go @@ -2,12 +2,15 @@ package scheduler import ( errors2 "errors" + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" - "github.com/crawlab-team/crawlab/core/container" "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/grpc/server" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/task/handler" "github.com/crawlab-team/crawlab/core/utils" grpc "github.com/crawlab-team/crawlab/grpc" "github.com/crawlab-team/crawlab/trace" @@ -20,8 +23,8 @@ import ( type ServiceV2 struct { // dependencies nodeCfgSvc interfaces.NodeConfigService - svr interfaces.GrpcServer - handlerSvc interfaces.TaskHandlerService + svr *server.GrpcServerV2 + handlerSvc *handler.ServiceV2 // settings interval time.Duration @@ -33,40 +36,42 @@ func (svc *ServiceV2) Start() { utils.DefaultWait() } -func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) { +func (svc *ServiceV2) Enqueue(t *models2.TaskV2, by primitive.ObjectID) (t2 *models2.TaskV2, err error) { // set task status t.Status = constants.TaskStatusPending - t.SetCreatedBy(by) + t.SetCreated(by) t.SetUpdated(by) // add task - taskModelSvc := service.NewModelServiceV2[models.TaskV2]() - _, err = taskModelSvc.InsertOne(*t) + taskModelSvc := service.NewModelServiceV2[models2.TaskV2]() + id, err := taskModelSvc.InsertOne(*t) if err != nil { return nil, err } // task queue item - tq := models.TaskQueueItemV2{ + tq := models2.TaskQueueItemV2{ Priority: t.Priority, NodeId: t.NodeId, } - tq.SetId(t.Id) + tq.SetId(id) + tq.SetCreated(by) + tq.SetUpdated(by) // task stat - ts := models.TaskStatV2{ - CreateTs: time.Now(), - } - ts.SetId(t.Id) + ts := models2.TaskStatV2{} + ts.SetId(id) + ts.SetCreated(by) + ts.SetUpdated(by) // enqueue task - _, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq) + _, err = service.NewModelServiceV2[models2.TaskQueueItemV2]().InsertOne(tq) if err != nil { return nil, trace.TraceError(err) } // add task stat - _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts) + _, err = service.NewModelServiceV2[models2.TaskStatV2]().InsertOne(ts) if err != nil { return nil, trace.TraceError(err) } @@ -77,7 +82,7 @@ func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *mode func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) { // task - t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(id) if err != nil { return trace.TraceError(err) } @@ -92,7 +97,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err // set status of pending tasks as "cancelled" and remove from task item queue if initialStatus == constants.TaskStatusPending { // remove from task item queue - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteById(t.Id); err != nil { return trace.TraceError(err) } return nil @@ -107,7 +112,7 @@ func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err } // node - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { return trace.TraceError(err) } @@ -133,22 +138,22 @@ func (svc *ServiceV2) SetInterval(interval time.Duration) { svc.interval = interval } -func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) { +func (svc *ServiceV2) SaveTask(t *models2.TaskV2, by primitive.ObjectID) (err error) { if t.Id.IsZero() { t.SetCreated(by) t.SetUpdated(by) - _, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t) + _, err = service.NewModelServiceV2[models2.TaskV2]().InsertOne(*t) return err } else { t.SetUpdated(by) - return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + return service.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t) } } // initTaskStatus initialize task status of existing tasks func (svc *ServiceV2) initTaskStatus() { // set status of running tasks as TaskStatusAbnormal - runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{ + runningTasks, err := service.NewModelServiceV2[models2.TaskV2]().GetMany(bson.M{ "status": bson.M{ "$in": []string{ constants.TaskStatusPending, @@ -163,23 +168,23 @@ func (svc *ServiceV2) initTaskStatus() { trace.PrintError(err) } for _, t := range runningTasks { - go func(t *models.TaskV2) { + go func(t *models2.TaskV2) { t.Status = constants.TaskStatusAbnormal if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { trace.PrintError(err) } }(&t) } - if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil { + if err := service.NewModelServiceV2[models2.TaskQueueItemV2]().DeleteMany(nil); err != nil { return } } -func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { +func (svc *ServiceV2) isMasterNode(t *models2.TaskV2) (ok bool, err error) { if t.NodeId.IsZero() { return false, trace.TraceError(errors.ErrorTaskNoNodeId) } - n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + n, err := service.NewModelServiceV2[models2.NodeV2]().GetById(t.NodeId) if err != nil { if errors2.Is(err, mongo2.ErrNoDocuments) { return false, trace.TraceError(errors.ErrorTaskNodeNotFound) @@ -192,7 +197,7 @@ func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { func (svc *ServiceV2) cleanupTasks() { for { // task stats over 30 days ago - taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{ "create_ts": bson.M{ "$lt": time.Now().Add(-30 * 24 * time.Hour), }, @@ -210,14 +215,14 @@ func (svc *ServiceV2) cleanupTasks() { if len(ids) > 0 { // remove tasks - if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) } // remove task stats - if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{ + if err := service.NewModelServiceV2[models2.TaskStatV2]().DeleteMany(bson.M{ "_id": bson.M{"$in": ids}, }); err != nil { trace.PrintError(err) @@ -233,17 +238,15 @@ func NewTaskSchedulerServiceV2() (svc2 *ServiceV2, err error) { svc := &ServiceV2{ interval: 5 * time.Second, } - - // dependency injection - if err := container.GetContainer().Invoke(func( - nodeCfgSvc interfaces.NodeConfigService, - svr interfaces.GrpcServer, - handlerSvc interfaces.TaskHandlerService, - ) { - svc.nodeCfgSvc = nodeCfgSvc - svc.svr = svr - svc.handlerSvc = handlerSvc - }); err != nil { + svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() + svc.svr, err = server.GetGrpcServerV2() + if err != nil { + log.Errorf("failed to get grpc server: %v", err) + return nil, err + } + svc.handlerSvc, err = handler.GetTaskHandlerServiceV2() + if err != nil { + log.Errorf("failed to get task handler service: %v", err) return nil, err } diff --git a/core/task/stats/service.go b/core/task/stats/service.go deleted file mode 100644 index b6964c29..00000000 --- a/core/task/stats/service.go +++ /dev/null @@ -1,155 +0,0 @@ -package stats - -import ( - "github.com/crawlab-team/crawlab/core/container" - "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/service" - "github.com/crawlab-team/crawlab/core/result" - "github.com/crawlab-team/crawlab/core/task" - "github.com/crawlab-team/crawlab/core/task/log" - "github.com/crawlab-team/crawlab/db/mongo" - "github.com/crawlab-team/crawlab/trace" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "sync" - "time" -) - -type Service struct { - // dependencies - interfaces.TaskBaseService - nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService - - // internals - mu sync.Mutex - resultServices sync.Map - rsTtl time.Duration - logDriver log.Driver -} - -func (svc *Service) Init() (err error) { - go svc.cleanup() - return nil -} - -func (svc *Service) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { - resultSvc, err := svc.getResultService(id) - if err != nil { - return err - } - if err := resultSvc.Insert(records...); err != nil { - return err - } - go svc.updateTaskStats(id, len(records)) - return nil -} - -func (svc *Service) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { - return svc.logDriver.WriteLines(id.Hex(), logs) -} - -func (svc *Service) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { - // atomic operation - svc.mu.Lock() - defer svc.mu.Unlock() - - // attempt to get from cache - res, _ := svc.resultServices.Load(id.Hex()) - if res != nil { - // hit in cache - resultSvc, ok := res.(interfaces.ResultService) - resultSvc.SetTime(time.Now()) - if ok { - return resultSvc, nil - } - } - - // task - t, err := svc.modelSvc.GetTaskById(id) - if err != nil { - return nil, err - } - - // result service - resultSvc, err = result.GetResultService(t.SpiderId) - if err != nil { - return nil, err - } - - // store in cache - svc.resultServices.Store(id.Hex(), resultSvc) - - return resultSvc, nil -} - -func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) { - _ = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).UpdateId(id, bson.M{ - "$inc": bson.M{ - "result_count": resultCount, - }, - }) -} - -func (svc *Service) cleanup() { - for { - // atomic operation - svc.mu.Lock() - - svc.resultServices.Range(func(key, value interface{}) bool { - rs := value.(interfaces.ResultService) - if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { - svc.resultServices.Delete(key) - } - return true - }) - - svc.mu.Unlock() - - time.Sleep(10 * time.Minute) - } -} - -func NewTaskStatsService() (svc2 interfaces.TaskStatsService, err error) { - // base service - baseSvc, err := task.NewBaseService() - if err != nil { - return nil, trace.TraceError(err) - } - - // service - svc := &Service{ - mu: sync.Mutex{}, - TaskBaseService: baseSvc, - resultServices: sync.Map{}, - } - - // dependency injection - if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService) { - svc.nodeCfgSvc = nodeCfgSvc - svc.modelSvc = modelSvc - }); err != nil { - return nil, trace.TraceError(err) - } - - // log driver - svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) - if err != nil { - return nil, err - } - - return svc, nil -} - -var _service interfaces.TaskStatsService - -func GetTaskStatsService() (svr interfaces.TaskStatsService, err error) { - if _service != nil { - return _service, nil - } - _service, err = NewTaskStatsService() - if err != nil { - return nil, err - } - return _service, nil -} diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go index b74b3441..d5a7d73a 100644 --- a/core/task/stats/service_v2.go +++ b/core/task/stats/service_v2.go @@ -1,12 +1,16 @@ package stats import ( + log2 "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/database" + interfaces2 "github.com/crawlab-team/crawlab/core/database/interfaces" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" nodeconfig "github.com/crawlab-team/crawlab/core/node/config" - "github.com/crawlab-team/crawlab/core/result" "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/crawlab/db/mongo" "github.com/crawlab-team/crawlab/trace" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" @@ -14,16 +18,23 @@ import ( "time" ) +type databaseServiceItem struct { + taskId primitive.ObjectID + dbId primitive.ObjectID + dbSvc interfaces2.DatabaseService + tableName string + time time.Time +} + type ServiceV2 struct { // dependencies nodeCfgSvc interfaces.NodeConfigService - modelSvc service.ModelService // internals - mu sync.Mutex - resultServices sync.Map - rsTtl time.Duration - logDriver log.Driver + mu sync.Mutex + databaseServiceItems map[string]*databaseServiceItem + databaseServiceTll time.Duration + logDriver log.Driver } func (svc *ServiceV2) Init() (err error) { @@ -31,15 +42,39 @@ func (svc *ServiceV2) Init() (err error) { return nil } -func (svc *ServiceV2) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { - resultSvc, err := svc.getResultService(id) +func (svc *ServiceV2) InsertData(taskId primitive.ObjectID, records ...map[string]interface{}) (err error) { + count := 0 + + item, err := svc.getDatabaseServiceItem(taskId) if err != nil { return err } - if err := resultSvc.Insert(records...); err != nil { - return err + dbId := item.dbId + dbSvc := item.dbSvc + tableName := item.tableName + if utils.IsPro() && dbSvc != nil { + for _, record := range records { + if err := dbSvc.CreateRow(dbId, "", tableName, record); err != nil { + log2.Errorf("failed to insert data: %v", err) + continue + } + count++ + } + } else { + var records2 []interface{} + for _, record := range records { + records2 = append(records2, record) + } + _, err = mongo.GetMongoCol(tableName).InsertMany(records2) + if err != nil { + log2.Errorf("failed to insert data: %v", err) + return err + } + count = len(records) } - go svc.updateTaskStats(id, len(records)) + + go svc.updateTaskStats(taskId, count) + return nil } @@ -47,42 +82,56 @@ func (svc *ServiceV2) InsertLogs(id primitive.ObjectID, logs ...string) (err err return svc.logDriver.WriteLines(id.Hex(), logs) } -func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { +func (svc *ServiceV2) getDatabaseServiceItem(taskId primitive.ObjectID) (item *databaseServiceItem, err error) { // atomic operation svc.mu.Lock() defer svc.mu.Unlock() // attempt to get from cache - res, _ := svc.resultServices.Load(id.Hex()) - if res != nil { + item, ok := svc.databaseServiceItems[taskId.Hex()] + if ok { // hit in cache - resultSvc, ok := res.(interfaces.ResultService) - resultSvc.SetTime(time.Now()) - if ok { - return resultSvc, nil - } + item.time = time.Now() + return item, nil } // task - t, err := svc.modelSvc.GetTaskById(id) + t, err := service.NewModelServiceV2[models2.TaskV2]().GetById(taskId) if err != nil { return nil, err } - // result service - resultSvc, err = result.GetResultService(t.SpiderId) + // spider + s, err := service.NewModelServiceV2[models2.SpiderV2]().GetById(t.SpiderId) if err != nil { return nil, err } + // database service + var dbSvc interfaces2.DatabaseService + if utils.IsPro() { + if dbRegSvc := database.GetDatabaseRegistryService(); dbRegSvc != nil { + dbSvc, err = dbRegSvc.GetDatabaseService(s.DataSourceId) + if err != nil { + return nil, err + } + } + } + // store in cache - svc.resultServices.Store(id.Hex(), resultSvc) + svc.databaseServiceItems[taskId.Hex()] = &databaseServiceItem{ + taskId: taskId, + dbId: s.DataSourceId, + dbSvc: dbSvc, + tableName: s.ColName, + time: time.Now(), + } - return resultSvc, nil + return item, nil } func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { - err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{ + err := service.NewModelServiceV2[models2.TaskStatV2]().UpdateById(id, bson.M{ "$inc": bson.M{ "result_count": resultCount, }, @@ -97,13 +146,11 @@ func (svc *ServiceV2) cleanup() { // atomic operation svc.mu.Lock() - svc.resultServices.Range(func(key, value interface{}) bool { - rs := value.(interfaces.ResultService) - if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { - svc.resultServices.Delete(key) + for k, v := range svc.databaseServiceItems { + if time.Now().After(v.time.Add(svc.databaseServiceTll)) { + delete(svc.databaseServiceItems, k) } - return true - }) + } svc.mu.Unlock() @@ -114,8 +161,9 @@ func (svc *ServiceV2) cleanup() { func NewTaskStatsServiceV2() (svc2 *ServiceV2, err error) { // service svc := &ServiceV2{ - mu: sync.Mutex{}, - resultServices: sync.Map{}, + mu: sync.Mutex{}, + databaseServiceItems: map[string]*databaseServiceItem{}, + databaseServiceTll: 10 * time.Minute, } svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() diff --git a/core/user/service_v2.go b/core/user/service_v2.go index 7784003c..696b7d95 100644 --- a/core/user/service_v2.go +++ b/core/user/service_v2.go @@ -1,10 +1,11 @@ package user import ( + "github.com/apex/log" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/errors" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/core/models/service" "github.com/crawlab-team/crawlab/core/utils" mongo2 "github.com/crawlab-team/crawlab/db/mongo" @@ -14,6 +15,7 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" + "sync" "time" ) @@ -181,7 +183,7 @@ func (svc *ServiceV2) getSecretFunc() jwt.Keyfunc { } } -func NewUserServiceV2() (svc *ServiceV2, err error) { +func newUserServiceV2() (svc *ServiceV2, err error) { // service svc = &ServiceV2{ modelSvc: service.NewModelServiceV2[models.UserV2](), @@ -191,6 +193,7 @@ func NewUserServiceV2() (svc *ServiceV2, err error) { // initialize if err := svc.Init(); err != nil { + log.Errorf("failed to initialize user service: %v", err) return nil, trace.TraceError(err) } @@ -198,15 +201,14 @@ func NewUserServiceV2() (svc *ServiceV2, err error) { } var userSvcV2 *ServiceV2 +var userSvcV2Once sync.Once func GetUserServiceV2() (svc *ServiceV2, err error) { - if userSvcV2 != nil { - return userSvcV2, nil - } - svc, err = NewUserServiceV2() - if err != nil { - return nil, err - } - userSvcV2 = svc - return svc, nil + userSvcV2Once.Do(func() { + userSvcV2, err = newUserServiceV2() + if err != nil { + return + } + }) + return userSvcV2, nil } diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go index 650c32f1..012da375 100644 --- a/core/utils/cockroachdb.go +++ b/core/utils/cockroachdb.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -27,7 +28,52 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultCockroachdbPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} + +func GetCockroachdbSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getCockroachdbSessionV2(ctx, ds) +} + +func getCockroachdbSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultCockroachdbPort } diff --git a/core/utils/demo.go b/core/utils/demo.go index bf667647..70b45044 100644 --- a/core/utils/demo.go +++ b/core/utils/demo.go @@ -31,7 +31,10 @@ func InitializedDemo() (ok bool) { func ImportDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo import", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } @@ -40,7 +43,10 @@ func ImportDemo() (err error) { func ReimportDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } @@ -49,7 +55,10 @@ func ReimportDemo() (err error) { func CleanupDemo() (err error) { cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) - cmd := sys_exec.BuildCmd(cmdStr) + cmd, err := sys_exec.BuildCmd(cmdStr) + if err != nil { + return err + } if err := cmd.Run(); err != nil { trace.PrintError(err) } diff --git a/core/utils/es.go b/core/utils/es.go index 6636d0da..ee70f187 100644 --- a/core/utils/es.go +++ b/core/utils/es.go @@ -8,6 +8,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/trace" "github.com/elastic/go-elasticsearch/v8" @@ -33,7 +34,70 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultElasticsearchPort + } + + // es hosts + addresses := []string{ + fmt.Sprintf("http://%s:%s", host, port), + } + + // retry backoff + rb := backoff.NewExponentialBackOff() + + // es client options + cfg := elasticsearch.Config{ + Addresses: addresses, + Username: ds.Username, + Password: ds.Password, + RetryBackoff: func(i int) time.Duration { + if i == 1 { + rb.Reset() + } + return rb.NextBackOff() + }, + } + + // es client + done := make(chan struct{}) + go func() { + c, err = elasticsearch.NewClient(cfg) + if err != nil { + return + } + var res *esapi.Response + res, err = c.Info() + fmt.Println(res) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return c, err +} + +func GetElasticsearchClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *elasticsearch.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getElasticsearchClientV2(ctx, ds) +} + +func getElasticsearchClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *elasticsearch.Client, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultElasticsearchPort } @@ -50,34 +114,12 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas Addresses: addresses, Username: ds.Username, Password: ds.Password, - //CloudID: "", - //APIKey: "", - //ServiceToken: "", - //CertificateFingerprint: "", - //Header: nil, - //CACert: nil, - //RetryOnStatus: nil, - //DisableRetry: false, - //EnableRetryOnTimeout: false, - //MaxRetries: 0, - //CompressRequestBody: false, - //DiscoverNodesOnStart: false, - //DiscoverNodesInterval: 0, - //EnableMetrics: false, - //EnableDebugLogger: false, - //EnableCompatibilityMode: false, - //DisableMetaHeader: false, - //UseResponseCheckOnly: false, RetryBackoff: func(i int) time.Duration { if i == 1 { rb.Reset() } return rb.NextBackOff() }, - //Transport: nil, - //Logger: nil, - //Selector: nil, - //ConnectionPoolFunc: nil, } // es client diff --git a/core/utils/file.go b/core/utils/file.go index 216a129a..572730ec 100644 --- a/core/utils/file.go +++ b/core/utils/file.go @@ -348,13 +348,13 @@ func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { if err != nil { return err } - if info.IsDir() { - return nil - } - hash, err := GetFileHash(path) - if err != nil { - return err + var hash string + if !info.IsDir() { + hash, err = GetFileHash(path) + if err != nil { + return err + } } relPath, err := filepath.Rel(dir, path) @@ -367,6 +367,7 @@ func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { Path: relPath, FullPath: path, Extension: filepath.Ext(path), + IsDir: info.IsDir(), FileSize: info.Size(), ModTime: info.ModTime(), Mode: info.Mode(), diff --git a/core/utils/git.go b/core/utils/git.go index 475d2314..272d0691 100644 --- a/core/utils/git.go +++ b/core/utils/git.go @@ -3,7 +3,6 @@ package utils import ( "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/interfaces" - "github.com/crawlab-team/crawlab/core/models/models" vcs "github.com/crawlab-team/crawlab/vcs" ) @@ -20,17 +19,3 @@ func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { gitClient.SetPrivateKey(g.GetPassword()) } } - -func InitGitClientAuthV2(g *models.GitV2, gitClient *vcs.GitClient) { - // set auth - switch g.AuthType { - case constants.GitAuthTypeHttp: - gitClient.SetAuthType(vcs.GitAuthTypeHTTP) - gitClient.SetUsername(g.Username) - gitClient.SetPassword(g.Password) - case constants.GitAuthTypeSsh: - gitClient.SetAuthType(vcs.GitAuthTypeSSH) - gitClient.SetUsername(g.Username) - gitClient.SetPrivateKey(g.Password) - } -} diff --git a/core/utils/hash.go b/core/utils/hash.go new file mode 100644 index 00000000..8fcbf0a1 --- /dev/null +++ b/core/utils/hash.go @@ -0,0 +1,12 @@ +package utils + +import "encoding/json" + +func GetObjectHash(obj any) string { + data, _ := json.Marshal(obj) + if data == nil { + // random hash + return EncryptMd5(NewUUIDString()) + } + return EncryptMd5(string(data)) +} diff --git a/core/utils/kafka.go b/core/utils/kafka.go index 1bf005f2..c07f19ea 100644 --- a/core/utils/kafka.go +++ b/core/utils/kafka.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/segmentio/kafka-go" "time" ) @@ -26,7 +27,34 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultKafkaPort + } + + // kafka connection address + network := "tcp" + address := fmt.Sprintf("%s:%s", host, port) + topic := ds.Database + partition := 0 // TODO: parameterize + + // kafka connection + return kafka.DialLeader(ctx, network, address, topic, partition) +} + +func GetKafkaConnectionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *kafka.Conn, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getKafkaConnectionV2(ctx, ds) +} + +func getKafkaConnectionV2(ctx context.Context, ds *models2.DatabaseV2) (c *kafka.Conn, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultKafkaPort } diff --git a/core/utils/mongo.go b/core/utils/mongo.go index 5807f24a..f576cec5 100644 --- a/core/utils/mongo.go +++ b/core/utils/mongo.go @@ -4,6 +4,7 @@ import ( "context" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/crawlab-team/crawlab/db/generic" "github.com/crawlab-team/crawlab/db/mongo" "go.mongodb.org/mongo-driver/bson" @@ -54,12 +55,18 @@ func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c return getMongoClient(ctx, ds) } +func GetMongoClientWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (c *mongo2.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMongoClientV2(ctx, ds) +} + func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { // normalize settings if ds.Host == "" { ds.Host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { ds.Port = constants.DefaultMongoPort } @@ -72,7 +79,6 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien opts = append(opts, mongo.WithDb(ds.Database)) opts = append(opts, mongo.WithUsername(ds.Username)) opts = append(opts, mongo.WithPassword(ds.Password)) - opts = append(opts, mongo.WithHosts(ds.Hosts)) // extra if ds.Extra != nil { @@ -92,3 +98,26 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien // client return mongo.GetMongoClient(opts...) } + +func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Client, err error) { + // normalize settings + if ds.Host == "" { + ds.Host = constants.DefaultHost + } + if ds.Port == 0 { + ds.Port = constants.DefaultMongoPort + } + + // options + var opts []mongo.ClientOption + opts = append(opts, mongo.WithContext(ctx)) + opts = append(opts, mongo.WithUri(ds.URI)) + opts = append(opts, mongo.WithHost(ds.Host)) + opts = append(opts, mongo.WithPort(ds.Port)) + opts = append(opts, mongo.WithDb(ds.Database)) + opts = append(opts, mongo.WithUsername(ds.Username)) + opts = append(opts, mongo.WithPassword(ds.Password)) + + // client + return mongo.GetMongoClient(opts...) +} diff --git a/core/utils/mssql.go b/core/utils/mssql.go index 0fb21353..1190a28e 100644 --- a/core/utils/mssql.go +++ b/core/utils/mssql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mssql" "time" @@ -27,7 +28,52 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultMssqlPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} + +func GetMssqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMssqlSessionV2(ctx, ds) +} + +func getMssqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultMssqlPort } diff --git a/core/utils/mysql.go b/core/utils/mysql.go index 8fc9ae18..34c8bcfb 100644 --- a/core/utils/mysql.go +++ b/core/utils/mysql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/mysql" "time" @@ -27,7 +28,52 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultMysqlPort + } + + // connect settings + settings := mysql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mysql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} + +func GetMysqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMysqlSessionV2(ctx, ds) +} + +func getMysqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultMysqlPort } diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go index 40c8a208..0971c10e 100644 --- a/core/utils/postgresql.go +++ b/core/utils/postgresql.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/crawlab-team/crawlab/core/constants" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/postgresql" "time" @@ -27,7 +28,52 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess if ds.Host == "" { host = constants.DefaultHost } - if ds.Port == "" { + if ds.Port == 0 { + port = constants.DefaultPostgresqlPort + } + + // connect settings + settings := postgresql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = postgresql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} + +func GetPostgresqlSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getPostgresqlSessionV2(ctx, ds) +} + +func getPostgresqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == 0 { port = constants.DefaultPostgresqlPort } diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go index 1d6ff682..8e39db2a 100644 --- a/core/utils/sqlite.go +++ b/core/utils/sqlite.go @@ -3,6 +3,7 @@ package utils import ( "context" "github.com/crawlab-team/crawlab/core/models/models" + models2 "github.com/crawlab-team/crawlab/core/models/models/v2" "github.com/upper/db/v4" "github.com/upper/db/v4/adapter/sqlite" "time" @@ -43,3 +44,35 @@ func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, return s, err } + +func GetSqliteSessionWithTimeoutV2(ds *models2.DatabaseV2, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getSqliteSessionV2(ctx, ds) +} + +func getSqliteSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Session, err error) { + // connect settings + settings := sqlite.ConnectionURL{ + Database: ds.Database, + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = sqlite.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/system.go b/core/utils/system.go index 9b161f11..6e501116 100644 --- a/core/utils/system.go +++ b/core/utils/system.go @@ -3,5 +3,5 @@ package utils import "github.com/spf13/viper" func IsPro() bool { - return viper.GetString("info.edition") == "global.edition.pro" + return viper.GetString("edition") == "global.edition.pro" } diff --git a/core/utils/time.go b/core/utils/time.go index 689e2c28..1ddcbdcf 100644 --- a/core/utils/time.go +++ b/core/utils/time.go @@ -1,6 +1,11 @@ package utils import ( + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/trace" + "regexp" + "strconv" "time" ) @@ -16,3 +21,53 @@ func GetLocalTimeString(t time.Time) string { t = GetLocalTime(t) return GetTimeString(t) } + +func GetTimeUnitParts(timeUnit string) (num int, unit string, err error) { + re := regexp.MustCompile(`^(\d+)([a-zA-Z])$`) + groups := re.FindStringSubmatch(timeUnit) + if len(groups) < 3 { + err = errors.New("failed to parse duration text") + log.Errorf("failed to parse duration text: %v", err) + trace.PrintError(err) + return 0, "", err + } + num, err = strconv.Atoi(groups[1]) + if err != nil { + log.Errorf("failed to convert string to int: %v", err) + trace.PrintError(err) + return 0, "", err + } + unit = groups[2] + return num, unit, nil +} + +func GetTimeDuration(num string, unit string) (d time.Duration, err error) { + numInt, err := strconv.Atoi(num) + if err != nil { + log.Errorf("failed to convert string to int: %v", err) + trace.PrintError(err) + return d, err + } + switch unit { + case "s": + d = time.Duration(numInt) * time.Second + case "m": + d = time.Duration(numInt) * time.Minute + case "h": + d = time.Duration(numInt) * time.Hour + case "d": + d = time.Duration(numInt) * 24 * time.Hour + case "w": + d = time.Duration(numInt) * 7 * 24 * time.Hour + case "M": + d = time.Duration(numInt) * 30 * 24 * time.Hour + case "y": + d = time.Duration(numInt) * 365 * 24 * time.Hour + default: + err = errors.New("invalid time unit") + log.Errorf("invalid time unit: %v", unit) + trace.PrintError(err) + return d, err + } + return d, nil +} diff --git a/db/go.mod b/db/go.mod index 861550fc..c0f417aa 100644 --- a/db/go.mod +++ b/db/go.mod @@ -2,51 +2,56 @@ module github.com/crawlab-team/crawlab/db go 1.22 +replace github.com/crawlab-team/crawlab/trace => ../trace + require ( github.com/apex/log v1.9.0 - github.com/cenkalti/backoff/v4 v4.1.0 - github.com/crawlab-team/crawlab/vcs v0.1.1 + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/crawlab-team/crawlab/trace v0.0.0-20240614095218-7b4ee8399ab0 github.com/gomodule/redigo v2.0.0+incompatible github.com/jmoiron/sqlx v1.2.0 - github.com/spf13/viper v1.10.0 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - go.mongodb.org/mongo-driver v1.15.0 + go.mongodb.org/mongo-driver v1.15.1 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/pretty v0.3.1 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/lib/pq v1.10.4 // indirect - github.com/magiconair/properties v1.8.5 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-sqlite3 v1.14.9 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/spf13/afero v1.10.0 // indirect - github.com/spf13/cast v1.4.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.2.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/ztrue/tracerr v0.4.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/db/mongo/client.go b/db/mongo/client.go index e5c03f90..f49a443a 100644 --- a/db/mongo/client.go +++ b/db/mongo/client.go @@ -33,10 +33,10 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) { _opts.Host = "localhost" } } - if _opts.Port == "" { - _opts.Port = viper.GetString("mongo.port") - if _opts.Port == "" { - _opts.Port = "27017" + if _opts.Port == 0 { + _opts.Port = viper.GetInt("mongo.port") + if _opts.Port == 0 { + _opts.Port = 27017 } } if _opts.Db == "" { @@ -123,7 +123,7 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client, mongoOpts.SetHosts(_opts.Hosts) } else { // hosts are unset - mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%s/%s", _opts.Host, _opts.Port, _opts.Db)) + mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%d/%s", _opts.Host, _opts.Port, _opts.Db)) } } diff --git a/db/mongo/client_options.go b/db/mongo/client_options.go index ef664f2a..3395876f 100644 --- a/db/mongo/client_options.go +++ b/db/mongo/client_options.go @@ -8,7 +8,7 @@ type ClientOptions struct { Context context.Context Uri string Host string - Port string + Port int Db string Hosts []string Username string @@ -36,7 +36,7 @@ func WithHost(value string) ClientOption { } } -func WithPort(value string) ClientOption { +func WithPort(value int) ClientOption { return func(options *ClientOptions) { options.Port = value } diff --git a/frontend/index.html b/frontend/index.html index c42f06c8..ccaeab84 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -12,7 +12,6 @@ -