mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
refactor: code cleanup
This commit is contained in:
4
.github/workflows/docker-crawlab.yml
vendored
4
.github/workflows/docker-crawlab.yml
vendored
@@ -10,6 +10,7 @@ on:
|
||||
types: [ docker-crawlab ]
|
||||
|
||||
env:
|
||||
SOURCE_CODE_PATH_CRAWLAB_BACKEND: core
|
||||
IMAGE_PATH_CRAWLAB_BACKEND: backend
|
||||
IMAGE_PATH_CRAWLAB_FRONTEND: frontend
|
||||
GH_PKG_NAME_CRAWLAB_BASE: ${{ github.repository_owner }}/crawlab-base
|
||||
@@ -44,7 +45,6 @@ jobs:
|
||||
with:
|
||||
files_yaml: |
|
||||
backend:
|
||||
- 'backend/**'
|
||||
- 'core/**'
|
||||
- 'db/**'
|
||||
- 'fs/**'
|
||||
@@ -165,7 +165,7 @@ jobs:
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: ${{ env.IMAGE_PATH_CRAWLAB_BACKEND }}
|
||||
context: ${{ env.SOURCE_CODE_PATH_CRAWLAB_BACKEND }}
|
||||
push: true
|
||||
tags: ${{ env.IMAGE_NAME_CRAWLAB_BACKEND }}:${{ needs.setup.outputs.version }}
|
||||
- name: Set output
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
|
||||
|
||||
# Working directory
|
||||
# . or absolute path, please note that the directories following must be under root.
|
||||
root = "."
|
||||
tmp_dir = "/tmp"
|
||||
|
||||
[build]
|
||||
# Just plain old shell command. You could use `make` as well.
|
||||
cmd = "go build -o ../tmp/main ./ "
|
||||
# Binary file yields from `cmd`.
|
||||
bin = "../tmp/main"
|
||||
# Customize binary.
|
||||
full_bin = "../tmp/main master"
|
||||
# Watch these filename extensions.
|
||||
include_ext = ["go", "tpl", "tmpl", "html"]
|
||||
# Ignore these filename extensions or directories.
|
||||
exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
|
||||
# Watch these directories if you specified.
|
||||
include_dir = ["../libs"]
|
||||
# Exclude files.
|
||||
exclude_file = []
|
||||
# This log file places in your tmp_dir.
|
||||
log = "air.log"
|
||||
# It's not necessary to trigger build each time file changes if it's too frequent.
|
||||
delay = 1000 # ms
|
||||
# Stop running old binary when build errors occur.
|
||||
stop_on_error = true
|
||||
# Send Interrupt signal before killing process (windows does not support this feature)
|
||||
send_interrupt = false
|
||||
# Delay after sending Interrupt signal
|
||||
kill_delay = 500 # ms
|
||||
|
||||
[log]
|
||||
# Show log time
|
||||
time = false
|
||||
|
||||
[color]
|
||||
# Customize each part's color. If no color found, use the raw app log.
|
||||
main = "magenta"
|
||||
watcher = "cyan"
|
||||
build = "yellow"
|
||||
runner = "green"
|
||||
|
||||
[misc]
|
||||
# Delete tmp directory on exit
|
||||
clean_on_exit = true
|
||||
@@ -1,47 +0,0 @@
|
||||
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
|
||||
|
||||
# Working directory
|
||||
# . or absolute path, please note that the directories following must be under root.
|
||||
root = "."
|
||||
tmp_dir = "/tmp"
|
||||
|
||||
[build]
|
||||
# Just plain old shell command. You could use `make` as well.
|
||||
cmd = "go build -o ../tmp/main ./ "
|
||||
# Binary file yields from `cmd`.
|
||||
bin = "../tmp/main"
|
||||
# Customize binary.
|
||||
full_bin = "../tmp/main worker"
|
||||
# Watch these filename extensions.
|
||||
include_ext = ["go", "tpl", "tmpl", "html"]
|
||||
# Ignore these filename extensions or directories.
|
||||
exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
|
||||
# Watch these directories if you specified.
|
||||
include_dir = ["../libs"]
|
||||
# Exclude files.
|
||||
exclude_file = []
|
||||
# This log file places in your tmp_dir.
|
||||
log = "air.log"
|
||||
# It's not necessary to trigger build each time file changes if it's too frequent.
|
||||
delay = 1000 # ms
|
||||
# Stop running old binary when build errors occur.
|
||||
stop_on_error = true
|
||||
# Send Interrupt signal before killing process (windows does not support this feature)
|
||||
send_interrupt = false
|
||||
# Delay after sending Interrupt signal
|
||||
kill_delay = 500 # ms
|
||||
|
||||
[log]
|
||||
# Show log time
|
||||
time = false
|
||||
|
||||
[color]
|
||||
# Customize each part's color. If no color found, use the raw app log.
|
||||
main = "magenta"
|
||||
watcher = "cyan"
|
||||
build = "yellow"
|
||||
runner = "green"
|
||||
|
||||
[misc]
|
||||
# Delete tmp directory on exit
|
||||
clean_on_exit = true
|
||||
@@ -1,12 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_size = 4
|
||||
indent_style = tab
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[{*.yml, *.yaml, *.json}]
|
||||
indent_size = 2
|
||||
@@ -1,3 +0,0 @@
|
||||
# crawlab-backend
|
||||
|
||||
Backend (Golang) for Crawlab
|
||||
@@ -1,12 +0,0 @@
|
||||
version: '3'
|
||||
|
||||
tasks:
|
||||
dev:
|
||||
desc: Switch to dev mode for local development.
|
||||
cmds:
|
||||
- cp -f go.mod.dev go.mod
|
||||
|
||||
deploy:
|
||||
desc: Switch to deploy mode for code publish.
|
||||
cmds:
|
||||
- echo 'not implemented'
|
||||
@@ -1,446 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -o errexit -o nounset -o pipefail
|
||||
|
||||
NAT='0|[1-9][0-9]*'
|
||||
ALPHANUM='[0-9]*[A-Za-z-][0-9A-Za-z-]*'
|
||||
IDENT="$NAT|$ALPHANUM"
|
||||
FIELD='[0-9A-Za-z-]+'
|
||||
|
||||
SEMVER_REGEX="\
|
||||
^[vV]?\
|
||||
($NAT)\\.($NAT)\\.($NAT)\
|
||||
(\\-(${IDENT})(\\.(${IDENT}))*)?\
|
||||
(\\+${FIELD}(\\.${FIELD})*)?$"
|
||||
|
||||
PROG=semver
|
||||
PROG_VERSION="3.4.0"
|
||||
|
||||
USAGE="\
|
||||
Usage:
|
||||
$PROG bump major <version>
|
||||
$PROG bump minor <version>
|
||||
$PROG bump patch <version>
|
||||
$PROG bump prerel|prerelease [<prerel>] <version>
|
||||
$PROG bump build <build> <version>
|
||||
$PROG bump release <version>
|
||||
$PROG get major <version>
|
||||
$PROG get minor <version>
|
||||
$PROG get patch <version>
|
||||
$PROG get prerel|prerelease <version>
|
||||
$PROG get build <version>
|
||||
$PROG get release <version>
|
||||
$PROG compare <version> <other_version>
|
||||
$PROG diff <version> <other_version>
|
||||
$PROG validate <version>
|
||||
$PROG --help
|
||||
$PROG --version
|
||||
|
||||
Arguments:
|
||||
<version> A version must match the following regular expression:
|
||||
\"${SEMVER_REGEX}\"
|
||||
In English:
|
||||
-- The version must match X.Y.Z[-PRERELEASE][+BUILD]
|
||||
where X, Y and Z are non-negative integers.
|
||||
-- PRERELEASE is a dot separated sequence of non-negative integers and/or
|
||||
identifiers composed of alphanumeric characters and hyphens (with
|
||||
at least one non-digit). Numeric identifiers must not have leading
|
||||
zeros. A hyphen (\"-\") introduces this optional part.
|
||||
-- BUILD is a dot separated sequence of identifiers composed of alphanumeric
|
||||
characters and hyphens. A plus (\"+\") introduces this optional part.
|
||||
|
||||
<other_version> See <version> definition.
|
||||
|
||||
<prerel> A string as defined by PRERELEASE above. Or, it can be a PRERELEASE
|
||||
prototype string followed by a dot.
|
||||
|
||||
<build> A string as defined by BUILD above.
|
||||
|
||||
Options:
|
||||
-v, --version Print the version of this tool.
|
||||
-h, --help Print this help message.
|
||||
|
||||
Commands:
|
||||
bump Bump by one of major, minor, patch; zeroing or removing
|
||||
subsequent parts. \"bump prerel\" (or its synonym \"bump prerelease\")
|
||||
sets the PRERELEASE part and removes any BUILD part. A trailing dot
|
||||
in the <prerel> argument introduces an incrementing numeric field
|
||||
which is added or bumped. If no <prerel> argument is provided, an
|
||||
incrementing numeric field is introduced/bumped. \"bump build\" sets
|
||||
the BUILD part. \"bump release\" removes any PRERELEASE or BUILD parts.
|
||||
The bumped version is written to stdout.
|
||||
|
||||
get Extract given part of <version>, where part is one of major, minor,
|
||||
patch, prerel (alternatively: prerelease), build, or release.
|
||||
|
||||
compare Compare <version> with <other_version>, output to stdout the
|
||||
following values: -1 if <other_version> is newer, 0 if equal, 1 if
|
||||
older. The BUILD part is not used in comparisons.
|
||||
|
||||
diff Compare <version> with <other_version>, output to stdout the
|
||||
difference between two versions by the release type (MAJOR, MINOR,
|
||||
PATCH, PRERELEASE, BUILD).
|
||||
|
||||
validate Validate if <version> follows the SEMVER pattern (see <version>
|
||||
definition). Print 'valid' to stdout if the version is valid, otherwise
|
||||
print 'invalid'.
|
||||
|
||||
See also:
|
||||
https://semver.org -- Semantic Versioning 2.0.0"
|
||||
|
||||
function error {
|
||||
echo -e "$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
function usage_help {
|
||||
error "$USAGE"
|
||||
}
|
||||
|
||||
function usage_version {
|
||||
echo -e "${PROG}: $PROG_VERSION"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# normalize the "part" keywords to a canonical string. At present,
|
||||
# only "prerelease" is normalized to "prerel".
|
||||
|
||||
function normalize_part {
|
||||
if [ "$1" == "prerelease" ]
|
||||
then
|
||||
echo "prerel"
|
||||
else
|
||||
echo "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
function validate_version {
|
||||
local version=$1
|
||||
if [[ "$version" =~ $SEMVER_REGEX ]]; then
|
||||
# if a second argument is passed, store the result in var named by $2
|
||||
if [ "$#" -eq "2" ]; then
|
||||
local major=${BASH_REMATCH[1]}
|
||||
local minor=${BASH_REMATCH[2]}
|
||||
local patch=${BASH_REMATCH[3]}
|
||||
local prere=${BASH_REMATCH[4]}
|
||||
local build=${BASH_REMATCH[8]}
|
||||
eval "$2=(\"$major\" \"$minor\" \"$patch\" \"$prere\" \"$build\")"
|
||||
else
|
||||
echo "$version"
|
||||
fi
|
||||
else
|
||||
error "version $version does not match the semver scheme 'X.Y.Z(-PRERELEASE)(+BUILD)'. See help for more information."
|
||||
fi
|
||||
}
|
||||
|
||||
function is_nat {
|
||||
[[ "$1" =~ ^($NAT)$ ]]
|
||||
}
|
||||
|
||||
function is_null {
|
||||
[ -z "$1" ]
|
||||
}
|
||||
|
||||
function order_nat {
|
||||
[ "$1" -lt "$2" ] && { echo -1 ; return ; }
|
||||
[ "$1" -gt "$2" ] && { echo 1 ; return ; }
|
||||
echo 0
|
||||
}
|
||||
|
||||
function order_string {
|
||||
[[ $1 < $2 ]] && { echo -1 ; return ; }
|
||||
[[ $1 > $2 ]] && { echo 1 ; return ; }
|
||||
echo 0
|
||||
}
|
||||
|
||||
# given two (named) arrays containing NAT and/or ALPHANUM fields, compare them
|
||||
# one by one according to semver 2.0.0 spec. Return -1, 0, 1 if left array ($1)
|
||||
# is less-than, equal, or greater-than the right array ($2). The longer array
|
||||
# is considered greater-than the shorter if the shorter is a prefix of the longer.
|
||||
#
|
||||
function compare_fields {
|
||||
local l="$1[@]"
|
||||
local r="$2[@]"
|
||||
local leftfield=( "${!l}" )
|
||||
local rightfield=( "${!r}" )
|
||||
local left
|
||||
local right
|
||||
|
||||
local i=$(( -1 ))
|
||||
local order=$(( 0 ))
|
||||
|
||||
while true
|
||||
do
|
||||
[ $order -ne 0 ] && { echo $order ; return ; }
|
||||
|
||||
: $(( i++ ))
|
||||
left="${leftfield[$i]}"
|
||||
right="${rightfield[$i]}"
|
||||
|
||||
is_null "$left" && is_null "$right" && { echo 0 ; return ; }
|
||||
is_null "$left" && { echo -1 ; return ; }
|
||||
is_null "$right" && { echo 1 ; return ; }
|
||||
|
||||
is_nat "$left" && is_nat "$right" && { order=$(order_nat "$left" "$right") ; continue ; }
|
||||
is_nat "$left" && { echo -1 ; return ; }
|
||||
is_nat "$right" && { echo 1 ; return ; }
|
||||
{ order=$(order_string "$left" "$right") ; continue ; }
|
||||
done
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2206 # checked by "validate"; ok to expand prerel id's into array
|
||||
function compare_version {
|
||||
local order
|
||||
validate_version "$1" V
|
||||
validate_version "$2" V_
|
||||
|
||||
# compare major, minor, patch
|
||||
|
||||
local left=( "${V[0]}" "${V[1]}" "${V[2]}" )
|
||||
local right=( "${V_[0]}" "${V_[1]}" "${V_[2]}" )
|
||||
|
||||
order=$(compare_fields left right)
|
||||
[ "$order" -ne 0 ] && { echo "$order" ; return ; }
|
||||
|
||||
# compare pre-release ids when M.m.p are equal
|
||||
|
||||
local prerel="${V[3]:1}"
|
||||
local prerel_="${V_[3]:1}"
|
||||
local left=( ${prerel//./ } )
|
||||
local right=( ${prerel_//./ } )
|
||||
|
||||
# if left and right have no pre-release part, then left equals right
|
||||
# if only one of left/right has pre-release part, that one is less than simple M.m.p
|
||||
|
||||
[ -z "$prerel" ] && [ -z "$prerel_" ] && { echo 0 ; return ; }
|
||||
[ -z "$prerel" ] && { echo 1 ; return ; }
|
||||
[ -z "$prerel_" ] && { echo -1 ; return ; }
|
||||
|
||||
# otherwise, compare the pre-release id's
|
||||
|
||||
compare_fields left right
|
||||
}
|
||||
|
||||
# render_prerel -- return a prerel field with a trailing numeric string
|
||||
# usage: render_prerel numeric [prefix-string]
|
||||
#
|
||||
function render_prerel {
|
||||
if [ -z "$2" ]
|
||||
then
|
||||
echo "${1}"
|
||||
else
|
||||
echo "${2}${1}"
|
||||
fi
|
||||
}
|
||||
|
||||
# extract_prerel -- extract prefix and trailing numeric portions of a pre-release part
|
||||
# usage: extract_prerel prerel prerel_parts
|
||||
# The prefix and trailing numeric parts are returned in "prerel_parts".
|
||||
#
|
||||
PREFIX_ALPHANUM='[.0-9A-Za-z-]*[.A-Za-z-]'
|
||||
DIGITS='[0-9][0-9]*'
|
||||
EXTRACT_REGEX="^(${PREFIX_ALPHANUM})*(${DIGITS})$"
|
||||
|
||||
function extract_prerel {
|
||||
local prefix; local numeric;
|
||||
|
||||
if [[ "$1" =~ $EXTRACT_REGEX ]]
|
||||
then # found prefix and trailing numeric parts
|
||||
prefix="${BASH_REMATCH[1]}"
|
||||
numeric="${BASH_REMATCH[2]}"
|
||||
else # no numeric part
|
||||
prefix="${1}"
|
||||
numeric=
|
||||
fi
|
||||
|
||||
eval "$2=(\"$prefix\" \"$numeric\")"
|
||||
}
|
||||
|
||||
# bump_prerel -- return the new pre-release part based on previous pre-release part
|
||||
# and prototype for bump
|
||||
# usage: bump_prerel proto previous
|
||||
#
|
||||
function bump_prerel {
|
||||
local proto; local prev_prefix; local prev_numeric;
|
||||
|
||||
# case one: no trailing dot in prototype => simply replace previous with proto
|
||||
if [[ ! ( "$1" =~ \.$ ) ]]
|
||||
then
|
||||
echo "$1"
|
||||
return
|
||||
fi
|
||||
|
||||
proto="${1%.}" # discard trailing dot marker from prototype
|
||||
|
||||
extract_prerel "${2#-}" prerel_parts # extract parts of previous pre-release
|
||||
# shellcheck disable=SC2154
|
||||
prev_prefix="${prerel_parts[0]}"
|
||||
prev_numeric="${prerel_parts[1]}"
|
||||
|
||||
# case two: bump or append numeric to previous pre-release part
|
||||
if [ "$proto" == "+" ] # dummy "+" indicates no prototype argument provided
|
||||
then
|
||||
if [ -n "$prev_numeric" ]
|
||||
then
|
||||
: $(( ++prev_numeric )) # previous pre-release is already numbered, bump it
|
||||
render_prerel "$prev_numeric" "$prev_prefix"
|
||||
else
|
||||
render_prerel 1 "$prev_prefix" # append starting number
|
||||
fi
|
||||
return
|
||||
fi
|
||||
|
||||
# case three: set, bump, or append using prototype prefix
|
||||
if [ "$prev_prefix" != "$proto" ]
|
||||
then
|
||||
render_prerel 1 "$proto" # proto not same pre-release; set and start at '1'
|
||||
elif [ -n "$prev_numeric" ]
|
||||
then
|
||||
: $(( ++prev_numeric )) # pre-release is numbered; bump it
|
||||
render_prerel "$prev_numeric" "$prev_prefix"
|
||||
else
|
||||
render_prerel 1 "$prev_prefix" # start pre-release at number '1'
|
||||
fi
|
||||
}
|
||||
|
||||
function command_bump {
|
||||
local new; local version; local sub_version; local command;
|
||||
|
||||
command="$(normalize_part "$1")"
|
||||
|
||||
case $# in
|
||||
2) case "$command" in
|
||||
major|minor|patch|prerel|release) sub_version="+."; version=$2;;
|
||||
*) usage_help;;
|
||||
esac ;;
|
||||
3) case "$command" in
|
||||
prerel|build) sub_version=$2 version=$3 ;;
|
||||
*) usage_help;;
|
||||
esac ;;
|
||||
*) usage_help;;
|
||||
esac
|
||||
|
||||
validate_version "$version" parts
|
||||
# shellcheck disable=SC2154
|
||||
local major="${parts[0]}"
|
||||
local minor="${parts[1]}"
|
||||
local patch="${parts[2]}"
|
||||
local prere="${parts[3]}"
|
||||
local build="${parts[4]}"
|
||||
|
||||
case "$command" in
|
||||
major) new="$((major + 1)).0.0";;
|
||||
minor) new="${major}.$((minor + 1)).0";;
|
||||
patch) new="${major}.${minor}.$((patch + 1))";;
|
||||
release) new="${major}.${minor}.${patch}";;
|
||||
prerel) new=$(validate_version "${major}.${minor}.${patch}-$(bump_prerel "$sub_version" "$prere")");;
|
||||
build) new=$(validate_version "${major}.${minor}.${patch}${prere}+${sub_version}");;
|
||||
*) usage_help ;;
|
||||
esac
|
||||
|
||||
echo "$new"
|
||||
exit 0
|
||||
}
|
||||
|
||||
function command_compare {
|
||||
local v; local v_;
|
||||
|
||||
case $# in
|
||||
2) v=$(validate_version "$1"); v_=$(validate_version "$2") ;;
|
||||
*) usage_help ;;
|
||||
esac
|
||||
|
||||
set +u # need unset array element to evaluate to null
|
||||
compare_version "$v" "$v_"
|
||||
exit 0
|
||||
}
|
||||
|
||||
function command_diff {
|
||||
validate_version "$1" v1_parts
|
||||
# shellcheck disable=SC2154
|
||||
local v1_major="${v1_parts[0]}"
|
||||
local v1_minor="${v1_parts[1]}"
|
||||
local v1_patch="${v1_parts[2]}"
|
||||
local v1_prere="${v1_parts[3]}"
|
||||
local v1_build="${v1_parts[4]}"
|
||||
|
||||
validate_version "$2" v2_parts
|
||||
# shellcheck disable=SC2154
|
||||
local v2_major="${v2_parts[0]}"
|
||||
local v2_minor="${v2_parts[1]}"
|
||||
local v2_patch="${v2_parts[2]}"
|
||||
local v2_prere="${v2_parts[3]}"
|
||||
local v2_build="${v2_parts[4]}"
|
||||
|
||||
if [ "${v1_major}" != "${v2_major}" ]; then
|
||||
echo "major"
|
||||
elif [ "${v1_minor}" != "${v2_minor}" ]; then
|
||||
echo "minor"
|
||||
elif [ "${v1_patch}" != "${v2_patch}" ]; then
|
||||
echo "patch"
|
||||
elif [ "${v1_prere}" != "${v2_prere}" ]; then
|
||||
echo "prerelease"
|
||||
elif [ "${v1_build}" != "${v2_build}" ]; then
|
||||
echo "build"
|
||||
fi
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
function command_get {
|
||||
local part version
|
||||
|
||||
if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then
|
||||
usage_help
|
||||
exit 0
|
||||
fi
|
||||
|
||||
part="$1"
|
||||
version="$2"
|
||||
|
||||
validate_version "$version" parts
|
||||
local major="${parts[0]}"
|
||||
local minor="${parts[1]}"
|
||||
local patch="${parts[2]}"
|
||||
local prerel="${parts[3]:1}"
|
||||
local build="${parts[4]:1}"
|
||||
local release="${major}.${minor}.${patch}"
|
||||
|
||||
part="$(normalize_part "$part")"
|
||||
|
||||
case "$part" in
|
||||
major|minor|patch|release|prerel|build) echo "${!part}" ;;
|
||||
*) usage_help ;;
|
||||
esac
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
function command_validate {
|
||||
if [[ "$#" -ne "1" ]]; then
|
||||
usage_help
|
||||
fi
|
||||
|
||||
if [[ "$1" =~ $SEMVER_REGEX ]]; then
|
||||
echo "valid"
|
||||
else
|
||||
echo "invalid"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
case $# in
|
||||
0) echo "Unknown command: $*"; usage_help;;
|
||||
esac
|
||||
|
||||
case $1 in
|
||||
--help|-h) echo -e "$USAGE"; exit 0;;
|
||||
--version|-v) usage_version ;;
|
||||
bump) shift; command_bump "$@";;
|
||||
get) shift; command_get "$@";;
|
||||
compare) shift; command_compare "$@";;
|
||||
diff) shift; command_diff "$@";;
|
||||
validate) shift; command_validate "$@";;
|
||||
*) echo "Unknown arguments: $*"; usage_help;;
|
||||
esac
|
||||
@@ -1,4 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
go get -u github.com/crawlab-team/crawlab/core@main
|
||||
go mod tidy
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
# update version type (major, minor, patch, prerelease)
|
||||
update_version_type="prerelease"
|
||||
if [ -n "$1" ]; then
|
||||
update_version_type="$1"
|
||||
fi
|
||||
|
||||
# current version
|
||||
current_version=$(grep -oEi 'version: v([0-9\.?]+)' conf/config.yml | sed -E 's/version: v//g')
|
||||
|
||||
# next version
|
||||
next_version=$(./bin/semver.sh bump $update_version_type $current_version)
|
||||
|
||||
# update next version to conf/config.yml
|
||||
sed -i '' "s/version: v$current_version/version: v$next_version/g" conf/config.yml
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Crawlab Configuration File
|
||||
edition: global.edition.community
|
||||
version: v0.7.0
|
||||
|
||||
mongo:
|
||||
host: localhost
|
||||
port: 27017
|
||||
db: crawlab_test
|
||||
username: ""
|
||||
password: ""
|
||||
authSource: "admin"
|
||||
|
||||
server:
|
||||
host: 0.0.0.0
|
||||
port: 8000
|
||||
|
||||
grpc:
|
||||
address: localhost:9666
|
||||
server:
|
||||
address: 0.0.0.0:9666
|
||||
authKey: Crawlab2021!
|
||||
|
||||
api:
|
||||
endpoint: http://localhost:8000
|
||||
|
||||
log:
|
||||
path: /var/log/crawlab
|
||||
149
backend/go.mod
149
backend/go.mod
@@ -1,149 +0,0 @@
|
||||
module crawlab
|
||||
|
||||
go 1.22
|
||||
|
||||
require github.com/crawlab-team/crawlab/core v0.0.0-20241017131019-1646df71366c
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.0 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.7.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.5.0 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.0.0 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.9.2 // indirect
|
||||
github.com/ReneKroon/ttlcache v1.7.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/apex/log v1.9.0 // indirect
|
||||
github.com/bytedance/sonic v1.11.6 // indirect
|
||||
github.com/bytedance/sonic/loader v0.1.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/crawlab-team/crawlab/db v0.0.0-20241017131019-1646df71366c // indirect
|
||||
github.com/crawlab-team/crawlab/fs v0.0.0-20241017131019-1646df71366c // indirect
|
||||
github.com/crawlab-team/crawlab/grpc v0.0.0-20241017131019-1646df71366c // indirect
|
||||
github.com/crawlab-team/crawlab/trace v0.0.0-20241017131019-1646df71366c // indirect
|
||||
github.com/crawlab-team/crawlab/vcs v0.0.0-20241017131019-1646df71366c // indirect
|
||||
github.com/crawlab-team/goseaweedfs v0.6.3 // indirect
|
||||
github.com/denisenkom/go-mssqldb v0.11.0 // indirect
|
||||
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
|
||||
github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.7.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/gin-gonic/gin v1.10.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
github.com/go-git/go-git/v5 v5.12.0 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 // indirect
|
||||
github.com/google/s2a-go v0.1.8 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/imroc/req v0.3.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
|
||||
github.com/jackc/pgconn v1.14.3 // indirect
|
||||
github.com/jackc/pgio v1.0.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgproto3/v2 v2.3.3 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/pgtype v1.14.0 // indirect
|
||||
github.com/jackc/pgx/v4 v4.18.2 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.17.7 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lib/pq v1.10.4 // indirect
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.9 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.4.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/segmentio/fasthash v1.0.3 // indirect
|
||||
github.com/segmentio/kafka-go v0.4.39 // indirect
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
|
||||
github.com/skeema/knownhosts v1.2.2 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/cobra v1.3.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.19.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/thoas/go-funk v0.9.1 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/upper/db/v4 v4.6.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.3 // indirect
|
||||
github.com/ztrue/tracerr v0.4.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.15.1 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
|
||||
go.opentelemetry.io/otel v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.28.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.28.0 // indirect
|
||||
go.uber.org/dig v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.25.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/oauth2 v0.21.0 // indirect
|
||||
golang.org/x/sync v0.7.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
google.golang.org/api v0.189.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect
|
||||
google.golang.org/grpc v1.65.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -1,20 +0,0 @@
|
||||
module crawlab
|
||||
|
||||
go 1.16
|
||||
|
||||
replace (
|
||||
github.com/crawlab-team/crawlab/core => ../../crawlab-core
|
||||
github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs
|
||||
github.com/crawlab-team/crawlab/fs => ../../crawlab-fs
|
||||
github.com/crawlab-team/crawlab/db => ../../crawlab-db
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apex/log v1.9.0
|
||||
github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200
|
||||
github.com/crawlab-team/crawlab/vcs v0.1.1
|
||||
github.com/gin-gonic/gin v1.7.1
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/viper v1.7.1
|
||||
go.uber.org/dig v1.10.0
|
||||
)
|
||||
@@ -1,20 +0,0 @@
|
||||
module crawlab
|
||||
|
||||
go 1.16
|
||||
|
||||
replace (
|
||||
github.com/crawlab-team/crawlab/core => /libs/crawlab-team/crawlab-core
|
||||
github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs
|
||||
github.com/crawlab-team/crawlab/fs => /libs/crawlab-team/crawlab-fs
|
||||
github.com/crawlab-team/crawlab/db => /libs/crawlab-team/crawlab-db
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apex/log v1.9.0
|
||||
github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200
|
||||
github.com/crawlab-team/crawlab/vcs v0.1.1
|
||||
github.com/gin-gonic/gin v1.7.1
|
||||
github.com/spf13/cobra v1.1.3
|
||||
github.com/spf13/viper v1.7.1
|
||||
go.uber.org/dig v1.10.0
|
||||
)
|
||||
1286
backend/go.sum
1286
backend/go.sum
File diff suppressed because it is too large
Load Diff
@@ -1,9 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/cmd"
|
||||
)
|
||||
|
||||
func main() {
|
||||
_ = cmd.Execute()
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"key": "master",
|
||||
"is_master": true,
|
||||
"name": "Master Node",
|
||||
"ip": "",
|
||||
"mac": "",
|
||||
"hostname": "",
|
||||
"description": "",
|
||||
"auth_key": "Crawlab2021!"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"key": "worker-01",
|
||||
"is_master": false,
|
||||
"name": "Worker Node 01",
|
||||
"ip": "",
|
||||
"mac": "",
|
||||
"hostname": "",
|
||||
"description": "",
|
||||
"auth_key": "Crawlab2021!"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"key": "worker-02",
|
||||
"is_master": false,
|
||||
"name": "Worker Node 02",
|
||||
"ip": "",
|
||||
"mac": "",
|
||||
"hostname": "",
|
||||
"description": "",
|
||||
"auth_key": "Crawlab2021!"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"key": "worker-03",
|
||||
"is_master": false,
|
||||
"name": "Worker Node 03",
|
||||
"ip": "",
|
||||
"mac": "",
|
||||
"hostname": "",
|
||||
"description": "",
|
||||
"auth_key": "Crawlab2021!"
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"key": "worker-invalid-auth-key",
|
||||
"is_master": false,
|
||||
"name": "worker",
|
||||
"ip": "",
|
||||
"mac": "",
|
||||
"hostname": "",
|
||||
"description": "",
|
||||
"auth_key": "invalid-auth-key"
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
var statsDefaultQuery = bson.M{
|
||||
"create_ts": bson.M{
|
||||
"created_ts": bson.M{
|
||||
"$gte": time.Now().Add(-30 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -32,11 +32,13 @@ type GrpcClientV2 struct {
|
||||
timeout time.Duration
|
||||
|
||||
// internals
|
||||
conn *grpc.ClientConn
|
||||
stream grpc2.NodeService_SubscribeClient
|
||||
msgCh chan *grpc2.StreamMessage
|
||||
err error
|
||||
once sync.Once
|
||||
conn *grpc.ClientConn
|
||||
stream grpc2.NodeService_SubscribeClient
|
||||
msgCh chan *grpc2.StreamMessage
|
||||
err error
|
||||
once sync.Once
|
||||
stopped bool
|
||||
stop chan struct{}
|
||||
|
||||
// clients
|
||||
NodeClient grpc2.NodeServiceClient
|
||||
@@ -71,6 +73,11 @@ func (c *GrpcClientV2) Start() (err error) {
|
||||
}
|
||||
|
||||
func (c *GrpcClientV2) Stop() (err error) {
|
||||
// set stopped flag
|
||||
c.stopped = true
|
||||
c.stop <- struct{}{}
|
||||
log.Infof("[GrpcClient] stopped")
|
||||
|
||||
// skip if connection is nil
|
||||
if c.conn == nil {
|
||||
return nil
|
||||
@@ -186,11 +193,23 @@ func (c *GrpcClientV2) connect() (err error) {
|
||||
|
||||
func (c *GrpcClientV2) subscribe() (err error) {
|
||||
op := func() error {
|
||||
// skip if stopped
|
||||
if c.stopped {
|
||||
return nil
|
||||
}
|
||||
|
||||
// request
|
||||
req := c.NewRequest(&entity.NodeInfo{
|
||||
Key: c.nodeCfgSvc.GetNodeKey(),
|
||||
IsMaster: false,
|
||||
})
|
||||
c.stream, err = c.NodeClient.Subscribe(context.Background(), req)
|
||||
|
||||
// timeout context
|
||||
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
|
||||
defer cancel()
|
||||
|
||||
// subscribe
|
||||
c.stream, err = c.NodeClient.Subscribe(ctx, req)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
@@ -200,50 +219,57 @@ func (c *GrpcClientV2) subscribe() (err error) {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe"))
|
||||
}
|
||||
|
||||
func (c *GrpcClientV2) handleStreamMessage() {
|
||||
log.Infof("[GrpcClient] start handling stream message...")
|
||||
for {
|
||||
// resubscribe if stream is set to nil
|
||||
if c.stream == nil {
|
||||
if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil {
|
||||
log.Errorf("subscribe")
|
||||
return
|
||||
select {
|
||||
case <-c.stop:
|
||||
return
|
||||
|
||||
default:
|
||||
// resubscribe if stream is set to nil
|
||||
if c.stream == nil {
|
||||
if err := c.subscribe(); err != nil {
|
||||
log.Errorf("subscribe")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// receive stream message
|
||||
msg, err := c.stream.Recv()
|
||||
log.Debugf("[GrpcClient] received message: %v", msg)
|
||||
if err != nil {
|
||||
// set error
|
||||
c.err = err
|
||||
|
||||
// end
|
||||
if err == io.EOF {
|
||||
log.Infof("[GrpcClient] received EOF signal, disconnecting")
|
||||
return
|
||||
}
|
||||
|
||||
// connection closed
|
||||
if c.IsClosed() {
|
||||
return
|
||||
}
|
||||
|
||||
// error
|
||||
trace.PrintError(err)
|
||||
c.stream = nil
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
// send stream message to channel
|
||||
c.msgCh <- msg
|
||||
|
||||
// reset error
|
||||
c.err = nil
|
||||
}
|
||||
|
||||
// receive stream message
|
||||
msg, err := c.stream.Recv()
|
||||
log.Debugf("[GrpcClient] received message: %v", msg)
|
||||
if err != nil {
|
||||
// set error
|
||||
c.err = err
|
||||
|
||||
// end
|
||||
if err == io.EOF {
|
||||
log.Infof("[GrpcClient] received EOF signal, disconnecting")
|
||||
return
|
||||
}
|
||||
|
||||
// connection closed
|
||||
if c.IsClosed() {
|
||||
return
|
||||
}
|
||||
|
||||
// error
|
||||
trace.PrintError(err)
|
||||
c.stream = nil
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
// send stream message to channel
|
||||
c.msgCh <- msg
|
||||
|
||||
// reset error
|
||||
c.err = nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +280,7 @@ func newGrpcClientV2() (c *GrpcClientV2) {
|
||||
Port: constants.DefaultGrpcClientRemotePort,
|
||||
}),
|
||||
timeout: 10 * time.Second,
|
||||
stop: make(chan struct{}),
|
||||
msgCh: make(chan *grpc2.StreamMessage),
|
||||
}
|
||||
client.nodeCfgSvc = nodeconfig.GetNodeConfigService()
|
||||
|
||||
12
core/main.go
12
core/main.go
@@ -1,4 +1,16 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/crawlab/core/cmd"
|
||||
"github.com/crawlab-team/crawlab/core/utils"
|
||||
)
|
||||
|
||||
func main() {
|
||||
go func() {
|
||||
err := cmd.Execute()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
utils.DefaultWait()
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/apex/log"
|
||||
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
|
||||
"github.com/crawlab-team/crawlab/core/models/service"
|
||||
"github.com/crawlab-team/crawlab/db/mongo"
|
||||
@@ -10,8 +13,58 @@ import (
|
||||
)
|
||||
|
||||
func CreateIndexesV2() {
|
||||
// Helper function to recreate indexes only if different
|
||||
recreateIndexes := func(col *mongo.Col, desiredIndexes []mongo2.IndexModel) {
|
||||
cur, err := col.GetCollection().Indexes().List(col.GetContext())
|
||||
if err != nil {
|
||||
log.Errorf("error listing indexes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var existingIndexes []bson.M
|
||||
err = cur.All(col.GetContext(), &existingIndexes)
|
||||
if err != nil {
|
||||
log.Errorf("error listing indexes: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Compare and recreate only if different
|
||||
needsUpdate := false
|
||||
existingKeys := make(map[string]bool)
|
||||
|
||||
// Skip _id index when comparing
|
||||
for _, idx := range existingIndexes {
|
||||
if name, ok := idx["name"].(string); ok && name != "_id_" {
|
||||
key := idx["key"].(bson.M)
|
||||
keyStr := fmt.Sprintf("%v", key)
|
||||
existingKeys[keyStr] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Check if desired indexes exist
|
||||
for _, idx := range desiredIndexes {
|
||||
keyStr := fmt.Sprintf("%v", idx.Keys)
|
||||
if !existingKeys[keyStr] {
|
||||
needsUpdate = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if needsUpdate {
|
||||
// Drop all existing indexes (except _id)
|
||||
err := col.DeleteAllIndexes()
|
||||
if err != nil {
|
||||
log.Errorf("error dropping indexes: %v", err)
|
||||
}
|
||||
|
||||
// Create new indexes
|
||||
col.MustCreateIndexes(desiredIndexes)
|
||||
log.Infof("recreated indexes for collection: %s", col.GetCollection().Name())
|
||||
}
|
||||
}
|
||||
|
||||
// nodes
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"key": 1}}, // key
|
||||
{Keys: bson.M{"name": 1}}, // name
|
||||
{Keys: bson.M{"is_master": 1}}, // is_master
|
||||
@@ -21,12 +74,12 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// projects
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
})
|
||||
|
||||
// spiders
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
{Keys: bson.M{"type": 1}},
|
||||
{Keys: bson.M{"col_id": 1}},
|
||||
@@ -34,7 +87,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// tasks
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"spider_id": 1}},
|
||||
{Keys: bson.M{"status": 1}},
|
||||
{Keys: bson.M{"node_id": 1}},
|
||||
@@ -44,72 +97,72 @@ func CreateIndexesV2() {
|
||||
{Keys: bson.M{"priority": 1}},
|
||||
{Keys: bson.M{"parent_id": 1}},
|
||||
{Keys: bson.M{"has_sub": 1}},
|
||||
{Keys: bson.M{"create_ts": -1}},
|
||||
{Keys: bson.M{"created_ts": -1}, Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30)},
|
||||
})
|
||||
|
||||
// task stats
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
{Keys: bson.M{"create_ts": 1}},
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"created_ts": -1}, Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30)},
|
||||
})
|
||||
|
||||
// schedules
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
{Keys: bson.M{"spider_id": 1}},
|
||||
{Keys: bson.M{"enabled": 1}},
|
||||
})
|
||||
|
||||
// users
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"username": 1}},
|
||||
{Keys: bson.M{"role": 1}},
|
||||
{Keys: bson.M{"email": 1}},
|
||||
})
|
||||
|
||||
// settings
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
|
||||
})
|
||||
|
||||
// tokens
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
})
|
||||
|
||||
// data sources
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
})
|
||||
|
||||
// data collections
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.M{"name": 1}},
|
||||
})
|
||||
|
||||
// roles
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
|
||||
})
|
||||
|
||||
// user role relations
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
|
||||
{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
|
||||
})
|
||||
|
||||
// permissions
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
|
||||
})
|
||||
|
||||
// role permission relations
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})), []mongo2.IndexModel{
|
||||
{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
|
||||
{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
|
||||
})
|
||||
|
||||
// dependencies
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"type", 1},
|
||||
@@ -121,7 +174,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// dependency settings
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"type", 1},
|
||||
@@ -133,7 +186,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// dependency logs
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{{"task_id", 1}},
|
||||
},
|
||||
@@ -144,7 +197,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// dependency tasks
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"update_ts", 1},
|
||||
@@ -154,7 +207,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// metrics
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"created_ts", -1},
|
||||
@@ -174,7 +227,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// notification requests
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"created_ts", -1},
|
||||
@@ -199,7 +252,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// databases
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"data_source_id", 1},
|
||||
@@ -208,7 +261,7 @@ func CreateIndexesV2() {
|
||||
})
|
||||
|
||||
// database metrics
|
||||
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
|
||||
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})), []mongo2.IndexModel{
|
||||
{
|
||||
Keys: bson.D{
|
||||
{"created_ts", -1},
|
||||
|
||||
@@ -87,6 +87,7 @@ func (svc *MasterServiceV2) Wait() {
|
||||
|
||||
func (svc *MasterServiceV2) Stop() {
|
||||
_ = svc.server.Stop()
|
||||
svc.handlerSvc.Stop()
|
||||
log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey())
|
||||
}
|
||||
|
||||
|
||||
@@ -71,6 +71,7 @@ func (svc *WorkerServiceV2) Wait() {
|
||||
|
||||
func (svc *WorkerServiceV2) Stop() {
|
||||
_ = svc.client.Stop()
|
||||
svc.handlerSvc.Stop()
|
||||
log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey())
|
||||
}
|
||||
|
||||
|
||||
@@ -53,6 +53,10 @@ func (svc *ServiceV2) Start() {
|
||||
go svc.Fetch()
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Stop() {
|
||||
svc.stopped = true
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) Run(taskId primitive.ObjectID) (err error) {
|
||||
return svc.run(taskId)
|
||||
}
|
||||
@@ -76,128 +80,82 @@ func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) {
|
||||
func (svc *ServiceV2) Fetch() {
|
||||
ticker := time.NewTicker(svc.fetchInterval)
|
||||
for {
|
||||
// wait
|
||||
<-ticker.C
|
||||
|
||||
// current node
|
||||
n, err := svc.GetCurrentNode()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip if node is not active or enabled
|
||||
if !n.Active || !n.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
// validate if there are available runners
|
||||
if svc.getRunnerCount() >= n.MaxRunners {
|
||||
continue
|
||||
}
|
||||
|
||||
// stop
|
||||
if svc.stopped {
|
||||
ticker.Stop()
|
||||
return
|
||||
}
|
||||
|
||||
// fetch task
|
||||
tid, err := svc.fetch()
|
||||
if err != nil {
|
||||
trace.PrintError(err)
|
||||
continue
|
||||
}
|
||||
|
||||
// skip if no task id
|
||||
if tid.IsZero() {
|
||||
continue
|
||||
}
|
||||
|
||||
// run task
|
||||
if err := svc.run(tid); err != nil {
|
||||
trace.PrintError(err)
|
||||
t, err := svc.GetTaskById(tid)
|
||||
if err != nil && t.Status != constants.TaskStatusCancelled {
|
||||
t.Error = err.Error()
|
||||
t.Status = constants.TaskStatusError
|
||||
t.SetUpdated(t.CreatedBy)
|
||||
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
// current node
|
||||
n, err := svc.GetCurrentNode()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// skip if node is not active or enabled
|
||||
if !n.Active || !n.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
// validate if there are available runners
|
||||
if svc.getRunnerCount() >= n.MaxRunners {
|
||||
continue
|
||||
}
|
||||
|
||||
// fetch task
|
||||
tid, err := svc.fetch()
|
||||
if err != nil {
|
||||
trace.PrintError(err)
|
||||
continue
|
||||
}
|
||||
|
||||
// skip if no task id
|
||||
if tid.IsZero() {
|
||||
continue
|
||||
}
|
||||
|
||||
// run task
|
||||
if err := svc.run(tid); err != nil {
|
||||
trace.PrintError(err)
|
||||
t, err := svc.GetTaskById(tid)
|
||||
if err != nil && t.Status != constants.TaskStatusCancelled {
|
||||
t.Error = err.Error()
|
||||
t.Status = constants.TaskStatusError
|
||||
t.SetUpdated(t.CreatedBy)
|
||||
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) ReportStatus() {
|
||||
ticker := time.NewTicker(svc.reportInterval)
|
||||
for {
|
||||
if svc.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
// report handler status
|
||||
if err := svc.reportStatus(); err != nil {
|
||||
trace.PrintError(err)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
// report handler status
|
||||
if err := svc.reportStatus(); err != nil {
|
||||
trace.PrintError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// wait
|
||||
time.Sleep(svc.reportInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) IsSyncLocked(path string) (ok bool) {
|
||||
_, ok = svc.syncLocks.Load(path)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) LockSync(path string) {
|
||||
svc.syncLocks.Store(path, true)
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) UnlockSync(path string) {
|
||||
svc.syncLocks.Delete(path)
|
||||
}
|
||||
|
||||
//func (svc *ServiceV2) GetMaxRunners() (maxRunners int) {
|
||||
// return svc.maxRunners
|
||||
//}
|
||||
//
|
||||
//func (svc *ServiceV2) SetMaxRunners(maxRunners int) {
|
||||
// svc.maxRunners = maxRunners
|
||||
//}
|
||||
|
||||
func (svc *ServiceV2) GetExitWatchDuration() (duration time.Duration) {
|
||||
return svc.exitWatchDuration
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) SetExitWatchDuration(duration time.Duration) {
|
||||
svc.exitWatchDuration = duration
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetFetchInterval() (interval time.Duration) {
|
||||
return svc.fetchInterval
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) SetFetchInterval(interval time.Duration) {
|
||||
svc.fetchInterval = interval
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetReportInterval() (interval time.Duration) {
|
||||
return svc.reportInterval
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) SetReportInterval(interval time.Duration) {
|
||||
svc.reportInterval = interval
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetCancelTimeout() (timeout time.Duration) {
|
||||
return svc.cancelTimeout
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) SetCancelTimeout(timeout time.Duration) {
|
||||
svc.cancelTimeout = timeout
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) {
|
||||
return svc.cfgSvc
|
||||
}
|
||||
@@ -245,17 +203,6 @@ func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2,
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) getRunners() (runners []*RunnerV2) {
|
||||
svc.mu.Lock()
|
||||
defer svc.mu.Unlock()
|
||||
svc.runners.Range(func(key, value interface{}) bool {
|
||||
r := value.(RunnerV2)
|
||||
runners = append(runners, &r)
|
||||
return true
|
||||
})
|
||||
return runners
|
||||
}
|
||||
|
||||
func (svc *ServiceV2) getRunnerCount() (count int) {
|
||||
n, err := svc.GetCurrentNode()
|
||||
if err != nil {
|
||||
@@ -401,7 +348,6 @@ func newTaskHandlerServiceV2() (svc2 *ServiceV2, err error) {
|
||||
cancelTimeout: 5 * time.Second,
|
||||
mu: sync.Mutex{},
|
||||
runners: sync.Map{},
|
||||
syncLocks: sync.Map{},
|
||||
}
|
||||
|
||||
// dependency injection
|
||||
|
||||
@@ -198,7 +198,7 @@ func (svc *ServiceV2) cleanupTasks() {
|
||||
for {
|
||||
// task stats over 30 days ago
|
||||
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
|
||||
"create_ts": bson.M{
|
||||
"created_ts": bson.M{
|
||||
"$lt": time.Now().Add(-30 * 24 * time.Hour),
|
||||
},
|
||||
}, nil)
|
||||
|
||||
15
go.work.sum
15
go.work.sum
@@ -748,6 +748,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOC
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4=
|
||||
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE=
|
||||
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
|
||||
@@ -831,6 +832,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe
|
||||
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E=
|
||||
github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
|
||||
github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ=
|
||||
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
|
||||
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
|
||||
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
@@ -961,9 +963,11 @@ github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/knz/go-libedit v1.10.1 h1:0pHpWtx9vcvC0xGZqEQlQdfSQs7WRlAjuPvk3fOZDCo=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
|
||||
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
|
||||
github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
|
||||
github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M=
|
||||
github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY=
|
||||
@@ -1044,12 +1048,15 @@ github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129 h1:ORfYNrFCM8HM
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
|
||||
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q=
|
||||
github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI=
|
||||
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
|
||||
github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
|
||||
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
|
||||
@@ -1057,12 +1064,15 @@ github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M=
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9mdZU6shPewugMyH5+GV6LNgW8w=
|
||||
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
|
||||
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
|
||||
github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
|
||||
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ=
|
||||
@@ -1136,6 +1146,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
|
||||
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
@@ -1174,7 +1185,9 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
@@ -1201,6 +1214,8 @@ golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
|
||||
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
|
||||
Reference in New Issue
Block a user