refactor: code cleanup

This commit is contained in:
Marvin Zhang
2024-10-29 12:59:45 +08:00
parent 43931a3e48
commit 1c03cb3e5c
31 changed files with 232 additions and 2327 deletions

View File

@@ -10,6 +10,7 @@ on:
types: [ docker-crawlab ]
env:
SOURCE_CODE_PATH_CRAWLAB_BACKEND: core
IMAGE_PATH_CRAWLAB_BACKEND: backend
IMAGE_PATH_CRAWLAB_FRONTEND: frontend
GH_PKG_NAME_CRAWLAB_BASE: ${{ github.repository_owner }}/crawlab-base
@@ -44,7 +45,6 @@ jobs:
with:
files_yaml: |
backend:
- 'backend/**'
- 'core/**'
- 'db/**'
- 'fs/**'
@@ -165,7 +165,7 @@ jobs:
- name: Build and push image
uses: docker/build-push-action@v5
with:
context: ${{ env.IMAGE_PATH_CRAWLAB_BACKEND }}
context: ${{ env.SOURCE_CODE_PATH_CRAWLAB_BACKEND }}
push: true
tags: ${{ env.IMAGE_NAME_CRAWLAB_BACKEND }}:${{ needs.setup.outputs.version }}
- name: Set output

View File

@@ -1,47 +0,0 @@
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
# Working directory
# . or absolute path, please note that the directories following must be under root.
root = "."
tmp_dir = "/tmp"
[build]
# Just plain old shell command. You could use `make` as well.
cmd = "go build -o ../tmp/main ./ "
# Binary file yields from `cmd`.
bin = "../tmp/main"
# Customize binary.
full_bin = "../tmp/main master"
# Watch these filename extensions.
include_ext = ["go", "tpl", "tmpl", "html"]
# Ignore these filename extensions or directories.
exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
# Watch these directories if you specified.
include_dir = ["../libs"]
# Exclude files.
exclude_file = []
# This log file places in your tmp_dir.
log = "air.log"
# It's not necessary to trigger build each time file changes if it's too frequent.
delay = 1000 # ms
# Stop running old binary when build errors occur.
stop_on_error = true
# Send Interrupt signal before killing process (windows does not support this feature)
send_interrupt = false
# Delay after sending Interrupt signal
kill_delay = 500 # ms
[log]
# Show log time
time = false
[color]
# Customize each part's color. If no color found, use the raw app log.
main = "magenta"
watcher = "cyan"
build = "yellow"
runner = "green"
[misc]
# Delete tmp directory on exit
clean_on_exit = true

View File

@@ -1,47 +0,0 @@
# Config file for [Air](https://github.com/cosmtrek/air) in TOML format
# Working directory
# . or absolute path, please note that the directories following must be under root.
root = "."
tmp_dir = "/tmp"
[build]
# Just plain old shell command. You could use `make` as well.
cmd = "go build -o ../tmp/main ./ "
# Binary file yields from `cmd`.
bin = "../tmp/main"
# Customize binary.
full_bin = "../tmp/main worker"
# Watch these filename extensions.
include_ext = ["go", "tpl", "tmpl", "html"]
# Ignore these filename extensions or directories.
exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"]
# Watch these directories if you specified.
include_dir = ["../libs"]
# Exclude files.
exclude_file = []
# This log file places in your tmp_dir.
log = "air.log"
# It's not necessary to trigger build each time file changes if it's too frequent.
delay = 1000 # ms
# Stop running old binary when build errors occur.
stop_on_error = true
# Send Interrupt signal before killing process (windows does not support this feature)
send_interrupt = false
# Delay after sending Interrupt signal
kill_delay = 500 # ms
[log]
# Show log time
time = false
[color]
# Customize each part's color. If no color found, use the raw app log.
main = "magenta"
watcher = "cyan"
build = "yellow"
runner = "green"
[misc]
# Delete tmp directory on exit
clean_on_exit = true

View File

@@ -1,12 +0,0 @@
root = true
[*]
charset = utf-8
end_of_line = lf
indent_size = 4
indent_style = tab
insert_final_newline = true
trim_trailing_whitespace = true
[{*.yml, *.yaml, *.json}]
indent_size = 2

View File

@@ -1,3 +0,0 @@
# crawlab-backend
Backend (Golang) for Crawlab

View File

@@ -1,12 +0,0 @@
version: '3'
tasks:
dev:
desc: Switch to dev mode for local development.
cmds:
- cp -f go.mod.dev go.mod
deploy:
desc: Switch to deploy mode for code publish.
cmds:
- echo 'not implemented'

View File

@@ -1,446 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
set -o errexit -o nounset -o pipefail
NAT='0|[1-9][0-9]*'
ALPHANUM='[0-9]*[A-Za-z-][0-9A-Za-z-]*'
IDENT="$NAT|$ALPHANUM"
FIELD='[0-9A-Za-z-]+'
SEMVER_REGEX="\
^[vV]?\
($NAT)\\.($NAT)\\.($NAT)\
(\\-(${IDENT})(\\.(${IDENT}))*)?\
(\\+${FIELD}(\\.${FIELD})*)?$"
PROG=semver
PROG_VERSION="3.4.0"
USAGE="\
Usage:
$PROG bump major <version>
$PROG bump minor <version>
$PROG bump patch <version>
$PROG bump prerel|prerelease [<prerel>] <version>
$PROG bump build <build> <version>
$PROG bump release <version>
$PROG get major <version>
$PROG get minor <version>
$PROG get patch <version>
$PROG get prerel|prerelease <version>
$PROG get build <version>
$PROG get release <version>
$PROG compare <version> <other_version>
$PROG diff <version> <other_version>
$PROG validate <version>
$PROG --help
$PROG --version
Arguments:
<version> A version must match the following regular expression:
\"${SEMVER_REGEX}\"
In English:
-- The version must match X.Y.Z[-PRERELEASE][+BUILD]
where X, Y and Z are non-negative integers.
-- PRERELEASE is a dot separated sequence of non-negative integers and/or
identifiers composed of alphanumeric characters and hyphens (with
at least one non-digit). Numeric identifiers must not have leading
zeros. A hyphen (\"-\") introduces this optional part.
-- BUILD is a dot separated sequence of identifiers composed of alphanumeric
characters and hyphens. A plus (\"+\") introduces this optional part.
<other_version> See <version> definition.
<prerel> A string as defined by PRERELEASE above. Or, it can be a PRERELEASE
prototype string followed by a dot.
<build> A string as defined by BUILD above.
Options:
-v, --version Print the version of this tool.
-h, --help Print this help message.
Commands:
bump Bump by one of major, minor, patch; zeroing or removing
subsequent parts. \"bump prerel\" (or its synonym \"bump prerelease\")
sets the PRERELEASE part and removes any BUILD part. A trailing dot
in the <prerel> argument introduces an incrementing numeric field
which is added or bumped. If no <prerel> argument is provided, an
incrementing numeric field is introduced/bumped. \"bump build\" sets
the BUILD part. \"bump release\" removes any PRERELEASE or BUILD parts.
The bumped version is written to stdout.
get Extract given part of <version>, where part is one of major, minor,
patch, prerel (alternatively: prerelease), build, or release.
compare Compare <version> with <other_version>, output to stdout the
following values: -1 if <other_version> is newer, 0 if equal, 1 if
older. The BUILD part is not used in comparisons.
diff Compare <version> with <other_version>, output to stdout the
difference between two versions by the release type (MAJOR, MINOR,
PATCH, PRERELEASE, BUILD).
validate Validate if <version> follows the SEMVER pattern (see <version>
definition). Print 'valid' to stdout if the version is valid, otherwise
print 'invalid'.
See also:
https://semver.org -- Semantic Versioning 2.0.0"
function error {
echo -e "$1" >&2
exit 1
}
function usage_help {
error "$USAGE"
}
function usage_version {
echo -e "${PROG}: $PROG_VERSION"
exit 0
}
# normalize the "part" keywords to a canonical string. At present,
# only "prerelease" is normalized to "prerel".
function normalize_part {
if [ "$1" == "prerelease" ]
then
echo "prerel"
else
echo "$1"
fi
}
function validate_version {
local version=$1
if [[ "$version" =~ $SEMVER_REGEX ]]; then
# if a second argument is passed, store the result in var named by $2
if [ "$#" -eq "2" ]; then
local major=${BASH_REMATCH[1]}
local minor=${BASH_REMATCH[2]}
local patch=${BASH_REMATCH[3]}
local prere=${BASH_REMATCH[4]}
local build=${BASH_REMATCH[8]}
eval "$2=(\"$major\" \"$minor\" \"$patch\" \"$prere\" \"$build\")"
else
echo "$version"
fi
else
error "version $version does not match the semver scheme 'X.Y.Z(-PRERELEASE)(+BUILD)'. See help for more information."
fi
}
function is_nat {
[[ "$1" =~ ^($NAT)$ ]]
}
function is_null {
[ -z "$1" ]
}
function order_nat {
[ "$1" -lt "$2" ] && { echo -1 ; return ; }
[ "$1" -gt "$2" ] && { echo 1 ; return ; }
echo 0
}
function order_string {
[[ $1 < $2 ]] && { echo -1 ; return ; }
[[ $1 > $2 ]] && { echo 1 ; return ; }
echo 0
}
# given two (named) arrays containing NAT and/or ALPHANUM fields, compare them
# one by one according to semver 2.0.0 spec. Return -1, 0, 1 if left array ($1)
# is less-than, equal, or greater-than the right array ($2). The longer array
# is considered greater-than the shorter if the shorter is a prefix of the longer.
#
function compare_fields {
local l="$1[@]"
local r="$2[@]"
local leftfield=( "${!l}" )
local rightfield=( "${!r}" )
local left
local right
local i=$(( -1 ))
local order=$(( 0 ))
while true
do
[ $order -ne 0 ] && { echo $order ; return ; }
: $(( i++ ))
left="${leftfield[$i]}"
right="${rightfield[$i]}"
is_null "$left" && is_null "$right" && { echo 0 ; return ; }
is_null "$left" && { echo -1 ; return ; }
is_null "$right" && { echo 1 ; return ; }
is_nat "$left" && is_nat "$right" && { order=$(order_nat "$left" "$right") ; continue ; }
is_nat "$left" && { echo -1 ; return ; }
is_nat "$right" && { echo 1 ; return ; }
{ order=$(order_string "$left" "$right") ; continue ; }
done
}
# shellcheck disable=SC2206 # checked by "validate"; ok to expand prerel id's into array
function compare_version {
local order
validate_version "$1" V
validate_version "$2" V_
# compare major, minor, patch
local left=( "${V[0]}" "${V[1]}" "${V[2]}" )
local right=( "${V_[0]}" "${V_[1]}" "${V_[2]}" )
order=$(compare_fields left right)
[ "$order" -ne 0 ] && { echo "$order" ; return ; }
# compare pre-release ids when M.m.p are equal
local prerel="${V[3]:1}"
local prerel_="${V_[3]:1}"
local left=( ${prerel//./ } )
local right=( ${prerel_//./ } )
# if left and right have no pre-release part, then left equals right
# if only one of left/right has pre-release part, that one is less than simple M.m.p
[ -z "$prerel" ] && [ -z "$prerel_" ] && { echo 0 ; return ; }
[ -z "$prerel" ] && { echo 1 ; return ; }
[ -z "$prerel_" ] && { echo -1 ; return ; }
# otherwise, compare the pre-release id's
compare_fields left right
}
# render_prerel -- return a prerel field with a trailing numeric string
# usage: render_prerel numeric [prefix-string]
#
function render_prerel {
if [ -z "$2" ]
then
echo "${1}"
else
echo "${2}${1}"
fi
}
# extract_prerel -- extract prefix and trailing numeric portions of a pre-release part
# usage: extract_prerel prerel prerel_parts
# The prefix and trailing numeric parts are returned in "prerel_parts".
#
PREFIX_ALPHANUM='[.0-9A-Za-z-]*[.A-Za-z-]'
DIGITS='[0-9][0-9]*'
EXTRACT_REGEX="^(${PREFIX_ALPHANUM})*(${DIGITS})$"
function extract_prerel {
local prefix; local numeric;
if [[ "$1" =~ $EXTRACT_REGEX ]]
then # found prefix and trailing numeric parts
prefix="${BASH_REMATCH[1]}"
numeric="${BASH_REMATCH[2]}"
else # no numeric part
prefix="${1}"
numeric=
fi
eval "$2=(\"$prefix\" \"$numeric\")"
}
# bump_prerel -- return the new pre-release part based on previous pre-release part
# and prototype for bump
# usage: bump_prerel proto previous
#
function bump_prerel {
local proto; local prev_prefix; local prev_numeric;
# case one: no trailing dot in prototype => simply replace previous with proto
if [[ ! ( "$1" =~ \.$ ) ]]
then
echo "$1"
return
fi
proto="${1%.}" # discard trailing dot marker from prototype
extract_prerel "${2#-}" prerel_parts # extract parts of previous pre-release
# shellcheck disable=SC2154
prev_prefix="${prerel_parts[0]}"
prev_numeric="${prerel_parts[1]}"
# case two: bump or append numeric to previous pre-release part
if [ "$proto" == "+" ] # dummy "+" indicates no prototype argument provided
then
if [ -n "$prev_numeric" ]
then
: $(( ++prev_numeric )) # previous pre-release is already numbered, bump it
render_prerel "$prev_numeric" "$prev_prefix"
else
render_prerel 1 "$prev_prefix" # append starting number
fi
return
fi
# case three: set, bump, or append using prototype prefix
if [ "$prev_prefix" != "$proto" ]
then
render_prerel 1 "$proto" # proto not same pre-release; set and start at '1'
elif [ -n "$prev_numeric" ]
then
: $(( ++prev_numeric )) # pre-release is numbered; bump it
render_prerel "$prev_numeric" "$prev_prefix"
else
render_prerel 1 "$prev_prefix" # start pre-release at number '1'
fi
}
function command_bump {
local new; local version; local sub_version; local command;
command="$(normalize_part "$1")"
case $# in
2) case "$command" in
major|minor|patch|prerel|release) sub_version="+."; version=$2;;
*) usage_help;;
esac ;;
3) case "$command" in
prerel|build) sub_version=$2 version=$3 ;;
*) usage_help;;
esac ;;
*) usage_help;;
esac
validate_version "$version" parts
# shellcheck disable=SC2154
local major="${parts[0]}"
local minor="${parts[1]}"
local patch="${parts[2]}"
local prere="${parts[3]}"
local build="${parts[4]}"
case "$command" in
major) new="$((major + 1)).0.0";;
minor) new="${major}.$((minor + 1)).0";;
patch) new="${major}.${minor}.$((patch + 1))";;
release) new="${major}.${minor}.${patch}";;
prerel) new=$(validate_version "${major}.${minor}.${patch}-$(bump_prerel "$sub_version" "$prere")");;
build) new=$(validate_version "${major}.${minor}.${patch}${prere}+${sub_version}");;
*) usage_help ;;
esac
echo "$new"
exit 0
}
function command_compare {
local v; local v_;
case $# in
2) v=$(validate_version "$1"); v_=$(validate_version "$2") ;;
*) usage_help ;;
esac
set +u # need unset array element to evaluate to null
compare_version "$v" "$v_"
exit 0
}
function command_diff {
validate_version "$1" v1_parts
# shellcheck disable=SC2154
local v1_major="${v1_parts[0]}"
local v1_minor="${v1_parts[1]}"
local v1_patch="${v1_parts[2]}"
local v1_prere="${v1_parts[3]}"
local v1_build="${v1_parts[4]}"
validate_version "$2" v2_parts
# shellcheck disable=SC2154
local v2_major="${v2_parts[0]}"
local v2_minor="${v2_parts[1]}"
local v2_patch="${v2_parts[2]}"
local v2_prere="${v2_parts[3]}"
local v2_build="${v2_parts[4]}"
if [ "${v1_major}" != "${v2_major}" ]; then
echo "major"
elif [ "${v1_minor}" != "${v2_minor}" ]; then
echo "minor"
elif [ "${v1_patch}" != "${v2_patch}" ]; then
echo "patch"
elif [ "${v1_prere}" != "${v2_prere}" ]; then
echo "prerelease"
elif [ "${v1_build}" != "${v2_build}" ]; then
echo "build"
fi
}
# shellcheck disable=SC2034
function command_get {
local part version
if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then
usage_help
exit 0
fi
part="$1"
version="$2"
validate_version "$version" parts
local major="${parts[0]}"
local minor="${parts[1]}"
local patch="${parts[2]}"
local prerel="${parts[3]:1}"
local build="${parts[4]:1}"
local release="${major}.${minor}.${patch}"
part="$(normalize_part "$part")"
case "$part" in
major|minor|patch|release|prerel|build) echo "${!part}" ;;
*) usage_help ;;
esac
exit 0
}
function command_validate {
if [[ "$#" -ne "1" ]]; then
usage_help
fi
if [[ "$1" =~ $SEMVER_REGEX ]]; then
echo "valid"
else
echo "invalid"
fi
exit 0
}
case $# in
0) echo "Unknown command: $*"; usage_help;;
esac
case $1 in
--help|-h) echo -e "$USAGE"; exit 0;;
--version|-v) usage_version ;;
bump) shift; command_bump "$@";;
get) shift; command_get "$@";;
compare) shift; command_compare "$@";;
diff) shift; command_diff "$@";;
validate) shift; command_validate "$@";;
*) echo "Unknown arguments: $*"; usage_help;;
esac

View File

@@ -1,4 +0,0 @@
#!/bin/bash
go get -u github.com/crawlab-team/crawlab/core@main
go mod tidy

View File

@@ -1,17 +0,0 @@
#!/bin/sh
# update version type (major, minor, patch, prerelease)
update_version_type="prerelease"
if [ -n "$1" ]; then
update_version_type="$1"
fi
# current version
current_version=$(grep -oEi 'version: v([0-9\.?]+)' conf/config.yml | sed -E 's/version: v//g')
# next version
next_version=$(./bin/semver.sh bump $update_version_type $current_version)
# update next version to conf/config.yml
sed -i '' "s/version: v$current_version/version: v$next_version/g" conf/config.yml

View File

@@ -1,27 +0,0 @@
# Crawlab Configuration File
edition: global.edition.community
version: v0.7.0
mongo:
host: localhost
port: 27017
db: crawlab_test
username: ""
password: ""
authSource: "admin"
server:
host: 0.0.0.0
port: 8000
grpc:
address: localhost:9666
server:
address: 0.0.0.0:9666
authKey: Crawlab2021!
api:
endpoint: http://localhost:8000
log:
path: /var/log/crawlab

View File

@@ -1,149 +0,0 @@
module crawlab
go 1.22
require github.com/crawlab-team/crawlab/core v0.0.0-20241017131019-1646df71366c
require (
dario.cat/mergo v1.0.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
)
require (
cloud.google.com/go/auth v0.7.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/PuerkitoBio/goquery v1.9.2 // indirect
github.com/ReneKroon/ttlcache v1.7.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/apex/log v1.9.0 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/crawlab-team/crawlab/db v0.0.0-20241017131019-1646df71366c // indirect
github.com/crawlab-team/crawlab/fs v0.0.0-20241017131019-1646df71366c // indirect
github.com/crawlab-team/crawlab/grpc v0.0.0-20241017131019-1646df71366c // indirect
github.com/crawlab-team/crawlab/trace v0.0.0-20241017131019-1646df71366c // indirect
github.com/crawlab-team/crawlab/vcs v0.0.0-20241017131019-1646df71366c // indirect
github.com/crawlab-team/goseaweedfs v0.6.3 // indirect
github.com/denisenkom/go-mssqldb v0.11.0 // indirect
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
github.com/elastic/go-elasticsearch/v8 v8.14.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/gin-gonic/gin v1.10.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/go-git/go-git/v5 v5.12.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomarkdown/markdown v0.0.0-20240626202925-2eda941fd024 // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imroc/req v0.3.2 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.14.3 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.3 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.14.0 // indirect
github.com/jackc/pgx/v4 v4.18.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lib/pq v1.10.4 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-sqlite3 v1.14.9 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/robfig/cron/v3 v3.0.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/segmentio/fasthash v1.0.3 // indirect
github.com/segmentio/kafka-go v0.4.39 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/cobra v1.3.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.19.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/thoas/go-funk v0.9.1 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/upper/db/v4 v4.6.0 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
github.com/ztrue/tracerr v0.4.0 // indirect
go.mongodb.org/mongo-driver v1.15.1 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.uber.org/dig v1.10.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
google.golang.org/api v0.189.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240725223205-93522f1f2a9f // indirect
google.golang.org/grpc v1.65.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -1,20 +0,0 @@
module crawlab
go 1.16
replace (
github.com/crawlab-team/crawlab/core => ../../crawlab-core
github.com/crawlab-team/crawlab-vcs => ../../crawlab-vcs
github.com/crawlab-team/crawlab/fs => ../../crawlab-fs
github.com/crawlab-team/crawlab/db => ../../crawlab-db
)
require (
github.com/apex/log v1.9.0
github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200
github.com/crawlab-team/crawlab/vcs v0.1.1
github.com/gin-gonic/gin v1.7.1
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1
go.uber.org/dig v1.10.0
)

View File

@@ -1,20 +0,0 @@
module crawlab
go 1.16
replace (
github.com/crawlab-team/crawlab/core => /libs/crawlab-team/crawlab-core
github.com/crawlab-team/crawlab-vcs => /libs/crawlab-team/crawlab-vcs
github.com/crawlab-team/crawlab/fs => /libs/crawlab-team/crawlab-fs
github.com/crawlab-team/crawlab/db => /libs/crawlab-team/crawlab-db
)
require (
github.com/apex/log v1.9.0
github.com/crawlab-team/crawlab/core v0.6.0-beta.20211230.1200
github.com/crawlab-team/crawlab/vcs v0.1.1
github.com/gin-gonic/gin v1.7.1
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.1
go.uber.org/dig v1.10.0
)

File diff suppressed because it is too large Load Diff

View File

@@ -1,9 +0,0 @@
package main
import (
"github.com/crawlab-team/crawlab/core/cmd"
)
func main() {
_ = cmd.Execute()
}

View File

@@ -1,10 +0,0 @@
{
"key": "master",
"is_master": true,
"name": "Master Node",
"ip": "",
"mac": "",
"hostname": "",
"description": "",
"auth_key": "Crawlab2021!"
}

View File

@@ -1,10 +0,0 @@
{
"key": "worker-01",
"is_master": false,
"name": "Worker Node 01",
"ip": "",
"mac": "",
"hostname": "",
"description": "",
"auth_key": "Crawlab2021!"
}

View File

@@ -1,10 +0,0 @@
{
"key": "worker-02",
"is_master": false,
"name": "Worker Node 02",
"ip": "",
"mac": "",
"hostname": "",
"description": "",
"auth_key": "Crawlab2021!"
}

View File

@@ -1,10 +0,0 @@
{
"key": "worker-03",
"is_master": false,
"name": "Worker Node 03",
"ip": "",
"mac": "",
"hostname": "",
"description": "",
"auth_key": "Crawlab2021!"
}

View File

@@ -1,10 +0,0 @@
{
"key": "worker-invalid-auth-key",
"is_master": false,
"name": "worker",
"ip": "",
"mac": "",
"hostname": "",
"description": "",
"auth_key": "invalid-auth-key"
}

View File

@@ -8,7 +8,7 @@ import (
)
var statsDefaultQuery = bson.M{
"create_ts": bson.M{
"created_ts": bson.M{
"$gte": time.Now().Add(-30 * 24 * time.Hour),
},
}

View File

@@ -32,11 +32,13 @@ type GrpcClientV2 struct {
timeout time.Duration
// internals
conn *grpc.ClientConn
stream grpc2.NodeService_SubscribeClient
msgCh chan *grpc2.StreamMessage
err error
once sync.Once
conn *grpc.ClientConn
stream grpc2.NodeService_SubscribeClient
msgCh chan *grpc2.StreamMessage
err error
once sync.Once
stopped bool
stop chan struct{}
// clients
NodeClient grpc2.NodeServiceClient
@@ -71,6 +73,11 @@ func (c *GrpcClientV2) Start() (err error) {
}
func (c *GrpcClientV2) Stop() (err error) {
// set stopped flag
c.stopped = true
c.stop <- struct{}{}
log.Infof("[GrpcClient] stopped")
// skip if connection is nil
if c.conn == nil {
return nil
@@ -186,11 +193,23 @@ func (c *GrpcClientV2) connect() (err error) {
func (c *GrpcClientV2) subscribe() (err error) {
op := func() error {
// skip if stopped
if c.stopped {
return nil
}
// request
req := c.NewRequest(&entity.NodeInfo{
Key: c.nodeCfgSvc.GetNodeKey(),
IsMaster: false,
})
c.stream, err = c.NodeClient.Subscribe(context.Background(), req)
// timeout context
ctx, cancel := context.WithTimeout(context.Background(), c.timeout)
defer cancel()
// subscribe
c.stream, err = c.NodeClient.Subscribe(ctx, req)
if err != nil {
return trace.TraceError(err)
}
@@ -200,50 +219,57 @@ func (c *GrpcClientV2) subscribe() (err error) {
return nil
}
return backoff.RetryNotify(op, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe"))
}
func (c *GrpcClientV2) handleStreamMessage() {
log.Infof("[GrpcClient] start handling stream message...")
for {
// resubscribe if stream is set to nil
if c.stream == nil {
if err := backoff.RetryNotify(c.subscribe, backoff.NewExponentialBackOff(), utils.BackoffErrorNotify("grpc client subscribe")); err != nil {
log.Errorf("subscribe")
return
select {
case <-c.stop:
return
default:
// resubscribe if stream is set to nil
if c.stream == nil {
if err := c.subscribe(); err != nil {
log.Errorf("subscribe")
return
}
}
// receive stream message
msg, err := c.stream.Recv()
log.Debugf("[GrpcClient] received message: %v", msg)
if err != nil {
// set error
c.err = err
// end
if err == io.EOF {
log.Infof("[GrpcClient] received EOF signal, disconnecting")
return
}
// connection closed
if c.IsClosed() {
return
}
// error
trace.PrintError(err)
c.stream = nil
time.Sleep(1 * time.Second)
continue
}
// send stream message to channel
c.msgCh <- msg
// reset error
c.err = nil
}
// receive stream message
msg, err := c.stream.Recv()
log.Debugf("[GrpcClient] received message: %v", msg)
if err != nil {
// set error
c.err = err
// end
if err == io.EOF {
log.Infof("[GrpcClient] received EOF signal, disconnecting")
return
}
// connection closed
if c.IsClosed() {
return
}
// error
trace.PrintError(err)
c.stream = nil
time.Sleep(1 * time.Second)
continue
}
// send stream message to channel
c.msgCh <- msg
// reset error
c.err = nil
}
}
@@ -254,6 +280,7 @@ func newGrpcClientV2() (c *GrpcClientV2) {
Port: constants.DefaultGrpcClientRemotePort,
}),
timeout: 10 * time.Second,
stop: make(chan struct{}),
msgCh: make(chan *grpc2.StreamMessage),
}
client.nodeCfgSvc = nodeconfig.GetNodeConfigService()

View File

@@ -1,4 +1,16 @@
package main
import (
"github.com/crawlab-team/crawlab/core/cmd"
"github.com/crawlab-team/crawlab/core/utils"
)
func main() {
go func() {
err := cmd.Execute()
if err != nil {
panic(err)
}
}()
utils.DefaultWait()
}

View File

@@ -1,6 +1,9 @@
package common
import (
"fmt"
"github.com/apex/log"
models2 "github.com/crawlab-team/crawlab/core/models/models/v2"
"github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/db/mongo"
@@ -10,8 +13,58 @@ import (
)
func CreateIndexesV2() {
// Helper function to recreate indexes only if different
recreateIndexes := func(col *mongo.Col, desiredIndexes []mongo2.IndexModel) {
cur, err := col.GetCollection().Indexes().List(col.GetContext())
if err != nil {
log.Errorf("error listing indexes: %v", err)
return
}
var existingIndexes []bson.M
err = cur.All(col.GetContext(), &existingIndexes)
if err != nil {
log.Errorf("error listing indexes: %v", err)
return
}
// Compare and recreate only if different
needsUpdate := false
existingKeys := make(map[string]bool)
// Skip _id index when comparing
for _, idx := range existingIndexes {
if name, ok := idx["name"].(string); ok && name != "_id_" {
key := idx["key"].(bson.M)
keyStr := fmt.Sprintf("%v", key)
existingKeys[keyStr] = true
}
}
// Check if desired indexes exist
for _, idx := range desiredIndexes {
keyStr := fmt.Sprintf("%v", idx.Keys)
if !existingKeys[keyStr] {
needsUpdate = true
break
}
}
if needsUpdate {
// Drop all existing indexes (except _id)
err := col.DeleteAllIndexes()
if err != nil {
log.Errorf("error dropping indexes: %v", err)
}
// Create new indexes
col.MustCreateIndexes(desiredIndexes)
log.Infof("recreated indexes for collection: %s", col.GetCollection().Name())
}
}
// nodes
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NodeV2{})), []mongo2.IndexModel{
{Keys: bson.M{"key": 1}}, // key
{Keys: bson.M{"name": 1}}, // name
{Keys: bson.M{"is_master": 1}}, // is_master
@@ -21,12 +74,12 @@ func CreateIndexesV2() {
})
// projects
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ProjectV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// spiders
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SpiderV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"type": 1}},
{Keys: bson.M{"col_id": 1}},
@@ -34,7 +87,7 @@ func CreateIndexesV2() {
})
// tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskV2{})), []mongo2.IndexModel{
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"status": 1}},
{Keys: bson.M{"node_id": 1}},
@@ -44,72 +97,72 @@ func CreateIndexesV2() {
{Keys: bson.M{"priority": 1}},
{Keys: bson.M{"parent_id": 1}},
{Keys: bson.M{"has_sub": 1}},
{Keys: bson.M{"create_ts": -1}},
{Keys: bson.M{"created_ts": -1}, Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30)},
})
// task stats
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})).MustCreateIndexes([]mongo2.IndexModel{
{Keys: bson.M{"create_ts": 1}},
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TaskStatV2{})), []mongo2.IndexModel{
{Keys: bson.M{"created_ts": -1}, Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30)},
})
// schedules
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.ScheduleV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
{Keys: bson.M{"spider_id": 1}},
{Keys: bson.M{"enabled": 1}},
})
// users
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserV2{})), []mongo2.IndexModel{
{Keys: bson.M{"username": 1}},
{Keys: bson.M{"role": 1}},
{Keys: bson.M{"email": 1}},
})
// settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.SettingV2{})), []mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// tokens
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.TokenV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// data sources
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// data collections
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DataCollectionV2{})), []mongo2.IndexModel{
{Keys: bson.M{"name": 1}},
})
// roles
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RoleV2{})), []mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// user role relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.UserRoleV2{})), []mongo2.IndexModel{
{Keys: bson.D{{"user_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"role_id", 1}, {"user_id", 1}}, Options: options.Index().SetUnique(true)},
})
// permissions
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.PermissionV2{})), []mongo2.IndexModel{
{Keys: bson.D{{"key", 1}}, Options: options.Index().SetUnique(true)},
})
// role permission relations
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.RolePermissionV2{})), []mongo2.IndexModel{
{Keys: bson.D{{"role_id", 1}, {"permission_id", 1}}, Options: options.Index().SetUnique(true)},
{Keys: bson.D{{"permission_id", 1}, {"role_id", 1}}, Options: options.Index().SetUnique(true)},
})
// dependencies
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -121,7 +174,7 @@ func CreateIndexesV2() {
})
// dependency settings
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencySettingV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"type", 1},
@@ -133,7 +186,7 @@ func CreateIndexesV2() {
})
// dependency logs
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyLogV2{})), []mongo2.IndexModel{
{
Keys: bson.D{{"task_id", 1}},
},
@@ -144,7 +197,7 @@ func CreateIndexesV2() {
})
// dependency tasks
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DependencyTaskV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"update_ts", 1},
@@ -154,7 +207,7 @@ func CreateIndexesV2() {
})
// metrics
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.MetricV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},
@@ -174,7 +227,7 @@ func CreateIndexesV2() {
})
// notification requests
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.NotificationRequestV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},
@@ -199,7 +252,7 @@ func CreateIndexesV2() {
})
// databases
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"data_source_id", 1},
@@ -208,7 +261,7 @@ func CreateIndexesV2() {
})
// database metrics
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
recreateIndexes(mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})), []mongo2.IndexModel{
{
Keys: bson.D{
{"created_ts", -1},

View File

@@ -87,6 +87,7 @@ func (svc *MasterServiceV2) Wait() {
func (svc *MasterServiceV2) Stop() {
_ = svc.server.Stop()
svc.handlerSvc.Stop()
log.Infof("master[%s] service has stopped", svc.GetConfigService().GetNodeKey())
}

View File

@@ -71,6 +71,7 @@ func (svc *WorkerServiceV2) Wait() {
func (svc *WorkerServiceV2) Stop() {
_ = svc.client.Stop()
svc.handlerSvc.Stop()
log.Infof("worker[%s] service has stopped", svc.cfgSvc.GetNodeKey())
}

View File

@@ -53,6 +53,10 @@ func (svc *ServiceV2) Start() {
go svc.Fetch()
}
func (svc *ServiceV2) Stop() {
svc.stopped = true
}
func (svc *ServiceV2) Run(taskId primitive.ObjectID) (err error) {
return svc.run(taskId)
}
@@ -76,128 +80,82 @@ func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) {
func (svc *ServiceV2) Fetch() {
ticker := time.NewTicker(svc.fetchInterval)
for {
// wait
<-ticker.C
// current node
n, err := svc.GetCurrentNode()
if err != nil {
continue
}
// skip if node is not active or enabled
if !n.Active || !n.Enabled {
continue
}
// validate if there are available runners
if svc.getRunnerCount() >= n.MaxRunners {
continue
}
// stop
if svc.stopped {
ticker.Stop()
return
}
// fetch task
tid, err := svc.fetch()
if err != nil {
trace.PrintError(err)
continue
}
// skip if no task id
if tid.IsZero() {
continue
}
// run task
if err := svc.run(tid); err != nil {
trace.PrintError(err)
t, err := svc.GetTaskById(tid)
if err != nil && t.Status != constants.TaskStatusCancelled {
t.Error = err.Error()
t.Status = constants.TaskStatusError
t.SetUpdated(t.CreatedBy)
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
select {
case <-ticker.C:
// current node
n, err := svc.GetCurrentNode()
if err != nil {
continue
}
// skip if node is not active or enabled
if !n.Active || !n.Enabled {
continue
}
// validate if there are available runners
if svc.getRunnerCount() >= n.MaxRunners {
continue
}
// fetch task
tid, err := svc.fetch()
if err != nil {
trace.PrintError(err)
continue
}
// skip if no task id
if tid.IsZero() {
continue
}
// run task
if err := svc.run(tid); err != nil {
trace.PrintError(err)
t, err := svc.GetTaskById(tid)
if err != nil && t.Status != constants.TaskStatusCancelled {
t.Error = err.Error()
t.Status = constants.TaskStatusError
t.SetUpdated(t.CreatedBy)
_ = client.NewModelServiceV2[models2.TaskV2]().ReplaceById(t.Id, *t)
continue
}
continue
}
continue
}
}
}
func (svc *ServiceV2) ReportStatus() {
ticker := time.NewTicker(svc.reportInterval)
for {
if svc.stopped {
return
}
// report handler status
if err := svc.reportStatus(); err != nil {
trace.PrintError(err)
select {
case <-ticker.C:
// report handler status
if err := svc.reportStatus(); err != nil {
trace.PrintError(err)
}
}
// wait
time.Sleep(svc.reportInterval)
}
}
func (svc *ServiceV2) IsSyncLocked(path string) (ok bool) {
_, ok = svc.syncLocks.Load(path)
return ok
}
func (svc *ServiceV2) LockSync(path string) {
svc.syncLocks.Store(path, true)
}
func (svc *ServiceV2) UnlockSync(path string) {
svc.syncLocks.Delete(path)
}
//func (svc *ServiceV2) GetMaxRunners() (maxRunners int) {
// return svc.maxRunners
//}
//
//func (svc *ServiceV2) SetMaxRunners(maxRunners int) {
// svc.maxRunners = maxRunners
//}
func (svc *ServiceV2) GetExitWatchDuration() (duration time.Duration) {
return svc.exitWatchDuration
}
func (svc *ServiceV2) SetExitWatchDuration(duration time.Duration) {
svc.exitWatchDuration = duration
}
func (svc *ServiceV2) GetFetchInterval() (interval time.Duration) {
return svc.fetchInterval
}
func (svc *ServiceV2) SetFetchInterval(interval time.Duration) {
svc.fetchInterval = interval
}
func (svc *ServiceV2) GetReportInterval() (interval time.Duration) {
return svc.reportInterval
}
func (svc *ServiceV2) SetReportInterval(interval time.Duration) {
svc.reportInterval = interval
}
func (svc *ServiceV2) GetCancelTimeout() (timeout time.Duration) {
return svc.cancelTimeout
}
func (svc *ServiceV2) SetCancelTimeout(timeout time.Duration) {
svc.cancelTimeout = timeout
}
func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) {
return svc.cfgSvc
}
@@ -245,17 +203,6 @@ func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models2.SpiderV2,
return s, nil
}
func (svc *ServiceV2) getRunners() (runners []*RunnerV2) {
svc.mu.Lock()
defer svc.mu.Unlock()
svc.runners.Range(func(key, value interface{}) bool {
r := value.(RunnerV2)
runners = append(runners, &r)
return true
})
return runners
}
func (svc *ServiceV2) getRunnerCount() (count int) {
n, err := svc.GetCurrentNode()
if err != nil {
@@ -401,7 +348,6 @@ func newTaskHandlerServiceV2() (svc2 *ServiceV2, err error) {
cancelTimeout: 5 * time.Second,
mu: sync.Mutex{},
runners: sync.Map{},
syncLocks: sync.Map{},
}
// dependency injection

View File

@@ -198,7 +198,7 @@ func (svc *ServiceV2) cleanupTasks() {
for {
// task stats over 30 days ago
taskStats, err := service.NewModelServiceV2[models2.TaskStatV2]().GetMany(bson.M{
"create_ts": bson.M{
"created_ts": bson.M{
"$lt": time.Now().Add(-30 * 24 * time.Hour),
},
}, nil)

View File

@@ -1,7 +1,6 @@
go 1.22
use (
backend
core
db
grpc

View File

@@ -748,6 +748,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOC
github.com/DataDog/datadog-go v3.2.0+incompatible h1:qSG2N4FghB1He/r2mFrWKCaL7dXCilEuNEeAn20fdD4=
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0 h1:oVLqHXhnYtUwM89y9T1fXGaK9wTkXHgNp8/ZNMQzUxE=
github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp v1.5.0/go.mod h1:dppbR7CwXD4pgtV9t3wD1812RaLDcBjtblcDF5f1vI0=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
@@ -831,6 +832,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E=
github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ=
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -961,9 +963,11 @@ github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/knz/go-libedit v1.10.1 h1:0pHpWtx9vcvC0xGZqEQlQdfSQs7WRlAjuPvk3fOZDCo=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M=
github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY=
@@ -1044,12 +1048,15 @@ github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129 h1:ORfYNrFCM8HM
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q=
github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI=
github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
@@ -1057,12 +1064,15 @@ github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M=
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc=
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9mdZU6shPewugMyH5+GV6LNgW8w=
github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI=
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ=
@@ -1136,6 +1146,7 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -1174,7 +1185,9 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -1201,6 +1214,8 @@ golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=