mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
- add proto/services/sync_service.proto and generate Go pb + grpc bindings - implement SyncServiceServer (streaming file scan + download) with: - request deduplication, in-memory cache (TTL), chunked streaming - concurrent-safe broadcast to waiters and server-side logging - register SyncSvr in gRPC server and expose sync client in GrpcClient: - add syncClient field, registration and safe getters with reconnection-aware timeouts - integrate gRPC sync into runner: - split syncFiles into syncFilesHTTP (legacy) and syncFilesGRPC - Runner now chooses implementation via config flag and performs streaming scan/download - controller improvements: - add semaphore-based rate limiting for sync scan requests with in-flight counters and logs - misc: - add utils.IsSyncGrpcEnabled() config helper - improve HTTP sync error diagnostics (Content-Type validation, response previews) - update/regenerate many protobuf and gRPC generated files (protoc/protoc-gen-go / protoc-gen-go-grpc version bumps)
170 lines
7.0 KiB
Go
170 lines
7.0 KiB
Go
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
|
// versions:
|
|
// - protoc-gen-go-grpc v1.5.1
|
|
// - protoc v5.29.2
|
|
// source: services/sync_service.proto
|
|
|
|
package grpc
|
|
|
|
import (
|
|
context "context"
|
|
grpc "google.golang.org/grpc"
|
|
codes "google.golang.org/grpc/codes"
|
|
status "google.golang.org/grpc/status"
|
|
)
|
|
|
|
// This is a compile-time assertion to ensure that this generated file
|
|
// is compatible with the grpc package it is being compiled against.
|
|
// Requires gRPC-Go v1.64.0 or later.
|
|
const _ = grpc.SupportPackageIsVersion9
|
|
|
|
const (
|
|
SyncService_StreamFileScan_FullMethodName = "/grpc.SyncService/StreamFileScan"
|
|
SyncService_StreamFileDownload_FullMethodName = "/grpc.SyncService/StreamFileDownload"
|
|
)
|
|
|
|
// SyncServiceClient is the client API for SyncService service.
|
|
//
|
|
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
|
type SyncServiceClient interface {
|
|
// Stream file list for synchronization
|
|
StreamFileScan(ctx context.Context, in *FileSyncRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FileScanChunk], error)
|
|
// Stream file download
|
|
StreamFileDownload(ctx context.Context, in *FileDownloadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FileDownloadChunk], error)
|
|
}
|
|
|
|
type syncServiceClient struct {
|
|
cc grpc.ClientConnInterface
|
|
}
|
|
|
|
func NewSyncServiceClient(cc grpc.ClientConnInterface) SyncServiceClient {
|
|
return &syncServiceClient{cc}
|
|
}
|
|
|
|
func (c *syncServiceClient) StreamFileScan(ctx context.Context, in *FileSyncRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FileScanChunk], error) {
|
|
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
|
stream, err := c.cc.NewStream(ctx, &SyncService_ServiceDesc.Streams[0], SyncService_StreamFileScan_FullMethodName, cOpts...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
x := &grpc.GenericClientStream[FileSyncRequest, FileScanChunk]{ClientStream: stream}
|
|
if err := x.ClientStream.SendMsg(in); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := x.ClientStream.CloseSend(); err != nil {
|
|
return nil, err
|
|
}
|
|
return x, nil
|
|
}
|
|
|
|
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
|
type SyncService_StreamFileScanClient = grpc.ServerStreamingClient[FileScanChunk]
|
|
|
|
func (c *syncServiceClient) StreamFileDownload(ctx context.Context, in *FileDownloadRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FileDownloadChunk], error) {
|
|
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
|
stream, err := c.cc.NewStream(ctx, &SyncService_ServiceDesc.Streams[1], SyncService_StreamFileDownload_FullMethodName, cOpts...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
x := &grpc.GenericClientStream[FileDownloadRequest, FileDownloadChunk]{ClientStream: stream}
|
|
if err := x.ClientStream.SendMsg(in); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := x.ClientStream.CloseSend(); err != nil {
|
|
return nil, err
|
|
}
|
|
return x, nil
|
|
}
|
|
|
|
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
|
type SyncService_StreamFileDownloadClient = grpc.ServerStreamingClient[FileDownloadChunk]
|
|
|
|
// SyncServiceServer is the server API for SyncService service.
|
|
// All implementations must embed UnimplementedSyncServiceServer
|
|
// for forward compatibility.
|
|
type SyncServiceServer interface {
|
|
// Stream file list for synchronization
|
|
StreamFileScan(*FileSyncRequest, grpc.ServerStreamingServer[FileScanChunk]) error
|
|
// Stream file download
|
|
StreamFileDownload(*FileDownloadRequest, grpc.ServerStreamingServer[FileDownloadChunk]) error
|
|
mustEmbedUnimplementedSyncServiceServer()
|
|
}
|
|
|
|
// UnimplementedSyncServiceServer must be embedded to have
|
|
// forward compatible implementations.
|
|
//
|
|
// NOTE: this should be embedded by value instead of pointer to avoid a nil
|
|
// pointer dereference when methods are called.
|
|
type UnimplementedSyncServiceServer struct{}
|
|
|
|
func (UnimplementedSyncServiceServer) StreamFileScan(*FileSyncRequest, grpc.ServerStreamingServer[FileScanChunk]) error {
|
|
return status.Errorf(codes.Unimplemented, "method StreamFileScan not implemented")
|
|
}
|
|
func (UnimplementedSyncServiceServer) StreamFileDownload(*FileDownloadRequest, grpc.ServerStreamingServer[FileDownloadChunk]) error {
|
|
return status.Errorf(codes.Unimplemented, "method StreamFileDownload not implemented")
|
|
}
|
|
func (UnimplementedSyncServiceServer) mustEmbedUnimplementedSyncServiceServer() {}
|
|
func (UnimplementedSyncServiceServer) testEmbeddedByValue() {}
|
|
|
|
// UnsafeSyncServiceServer may be embedded to opt out of forward compatibility for this service.
|
|
// Use of this interface is not recommended, as added methods to SyncServiceServer will
|
|
// result in compilation errors.
|
|
type UnsafeSyncServiceServer interface {
|
|
mustEmbedUnimplementedSyncServiceServer()
|
|
}
|
|
|
|
func RegisterSyncServiceServer(s grpc.ServiceRegistrar, srv SyncServiceServer) {
|
|
// If the following call pancis, it indicates UnimplementedSyncServiceServer was
|
|
// embedded by pointer and is nil. This will cause panics if an
|
|
// unimplemented method is ever invoked, so we test this at initialization
|
|
// time to prevent it from happening at runtime later due to I/O.
|
|
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
|
|
t.testEmbeddedByValue()
|
|
}
|
|
s.RegisterService(&SyncService_ServiceDesc, srv)
|
|
}
|
|
|
|
func _SyncService_StreamFileScan_Handler(srv interface{}, stream grpc.ServerStream) error {
|
|
m := new(FileSyncRequest)
|
|
if err := stream.RecvMsg(m); err != nil {
|
|
return err
|
|
}
|
|
return srv.(SyncServiceServer).StreamFileScan(m, &grpc.GenericServerStream[FileSyncRequest, FileScanChunk]{ServerStream: stream})
|
|
}
|
|
|
|
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
|
type SyncService_StreamFileScanServer = grpc.ServerStreamingServer[FileScanChunk]
|
|
|
|
func _SyncService_StreamFileDownload_Handler(srv interface{}, stream grpc.ServerStream) error {
|
|
m := new(FileDownloadRequest)
|
|
if err := stream.RecvMsg(m); err != nil {
|
|
return err
|
|
}
|
|
return srv.(SyncServiceServer).StreamFileDownload(m, &grpc.GenericServerStream[FileDownloadRequest, FileDownloadChunk]{ServerStream: stream})
|
|
}
|
|
|
|
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
|
|
type SyncService_StreamFileDownloadServer = grpc.ServerStreamingServer[FileDownloadChunk]
|
|
|
|
// SyncService_ServiceDesc is the grpc.ServiceDesc for SyncService service.
|
|
// It's only intended for direct use with grpc.RegisterService,
|
|
// and not to be introspected or modified (even as a copy)
|
|
var SyncService_ServiceDesc = grpc.ServiceDesc{
|
|
ServiceName: "grpc.SyncService",
|
|
HandlerType: (*SyncServiceServer)(nil),
|
|
Methods: []grpc.MethodDesc{},
|
|
Streams: []grpc.StreamDesc{
|
|
{
|
|
StreamName: "StreamFileScan",
|
|
Handler: _SyncService_StreamFileScan_Handler,
|
|
ServerStreams: true,
|
|
},
|
|
{
|
|
StreamName: "StreamFileDownload",
|
|
Handler: _SyncService_StreamFileDownload_Handler,
|
|
ServerStreams: true,
|
|
},
|
|
},
|
|
Metadata: "services/sync_service.proto",
|
|
}
|