mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-26 17:49:15 +01:00
feat(sync): add gRPC file synchronization service and integrate end-to-end
- add proto/services/sync_service.proto and generate Go pb + grpc bindings - implement SyncServiceServer (streaming file scan + download) with: - request deduplication, in-memory cache (TTL), chunked streaming - concurrent-safe broadcast to waiters and server-side logging - register SyncSvr in gRPC server and expose sync client in GrpcClient: - add syncClient field, registration and safe getters with reconnection-aware timeouts - integrate gRPC sync into runner: - split syncFiles into syncFilesHTTP (legacy) and syncFilesGRPC - Runner now chooses implementation via config flag and performs streaming scan/download - controller improvements: - add semaphore-based rate limiting for sync scan requests with in-flight counters and logs - misc: - add utils.IsSyncGrpcEnabled() config helper - improve HTTP sync error diagnostics (Content-Type validation, response previews) - update/regenerate many protobuf and gRPC generated files (protoc/protoc-gen-go / protoc-gen-go-grpc version bumps)
This commit is contained in:
@@ -15,9 +15,29 @@ import (
|
||||
var (
|
||||
syncDownloadSemaphore = semaphore.NewWeighted(utils.GetSyncDownloadMaxConcurrency())
|
||||
syncDownloadInFlight int64
|
||||
syncScanSemaphore = semaphore.NewWeighted(10) // Limit concurrent scan requests
|
||||
syncScanInFlight int64
|
||||
)
|
||||
|
||||
func GetSyncScan(c *gin.Context) (response *Response[entity.FsFileInfoMap], err error) {
|
||||
ctx := c.Request.Context()
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
// Rate limiting for scan requests
|
||||
if err := syncScanSemaphore.Acquire(ctx, 1); err != nil {
|
||||
logger.Warnf("failed to acquire sync scan slot for id=%s path=%s: %v", c.Param("id"), c.Param("path"), err)
|
||||
return GetErrorResponse[entity.FsFileInfoMap](errors.Annotate(err, "server overloaded, please retry"))
|
||||
}
|
||||
current := atomic.AddInt64(&syncScanInFlight, 1)
|
||||
logger.Debugf("sync scan in-flight=%d id=%s path=%s", current, c.Param("id"), c.Param("path"))
|
||||
defer func() {
|
||||
newVal := atomic.AddInt64(&syncScanInFlight, -1)
|
||||
logger.Debugf("sync scan completed in-flight=%d id=%s path=%s", newVal, c.Param("id"), c.Param("path"))
|
||||
syncScanSemaphore.Release(1)
|
||||
}()
|
||||
|
||||
workspacePath := utils.GetWorkspace()
|
||||
dirPath := filepath.Join(workspacePath, c.Param("id"), c.Param("path"))
|
||||
files, err := utils.ScanDirectory(dirPath)
|
||||
|
||||
Reference in New Issue
Block a user