git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-28 17:50:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(sync): add gRPC file synchronization service and integrate end-to-end

Browse Source 
- add proto/services/sync_service.proto and generate Go pb + grpc bindings
- implement SyncServiceServer (streaming file scan + download) with:
  - request deduplication, in-memory cache (TTL), chunked streaming
  - concurrent-safe broadcast to waiters and server-side logging
- register SyncSvr in gRPC server and expose sync client in GrpcClient:
  - add syncClient field, registration and safe getters with reconnection-aware timeouts
- integrate gRPC sync into runner:
  - split syncFiles into syncFilesHTTP (legacy) and syncFilesGRPC
  - Runner now chooses implementation via config flag and performs streaming scan/download
- controller improvements:
  - add semaphore-based rate limiting for sync scan requests with in-flight counters and logs
- misc:
  - add utils.IsSyncGrpcEnabled() config helper
  - improve HTTP sync error diagnostics (Content-Type validation, response previews)
  - update/regenerate many protobuf and gRPC generated files (protoc/protoc-gen-go / protoc-gen-go-grpc version bumps)
This commit is contained in:
Marvin Zhang
2025-10-20 12:48:53 +08:00
parent 61604e1817
commit f441265cc2
22 changed files with 1860 additions and 1195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
grpc/node_service_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.4.0
// - protoc v5.27.2
// - protoc-gen-go-grpc v1.5.1
// - protoc v5.29.2
// source: services/node_service.proto
package grpc
@@ -15,8 +15,8 @@ import (
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.62.0 or later.
const _ = grpc.SupportPackageIsVersion8
// Requires gRPC-Go v1.64.0 or later.
const _ = grpc.SupportPackageIsVersion9
const (
NodeService_Register_FullMethodName = "/grpc.NodeService/Register"
@@ -30,7 +30,7 @@ const (
type NodeServiceClient interface {
Register(ctx context.Context, in *NodeServiceRegisterRequest, opts ...grpc.CallOption) (*Response, error)
SendHeartbeat(ctx context.Context, in *NodeServiceSendHeartbeatRequest, opts ...grpc.CallOption) (*Response, error)
Subscribe(ctx context.Context, in *NodeServiceSubscribeRequest, opts ...grpc.CallOption) (NodeService_SubscribeClient, error)
Subscribe(ctx context.Context, in *NodeServiceSubscribeRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[NodeServiceSubscribeResponse], error)
}
type nodeServiceClient struct {
@@ -61,13 +61,13 @@ func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *NodeServiceSe
return out, nil
}
func (c *nodeServiceClient) Subscribe(ctx context.Context, in *NodeServiceSubscribeRequest, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) {
func (c *nodeServiceClient) Subscribe(ctx context.Context, in *NodeServiceSubscribeRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[NodeServiceSubscribeResponse], error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], NodeService_Subscribe_FullMethodName, cOpts...)
if err != nil {
return nil, err
}
x := &nodeServiceSubscribeClient{ClientStream: stream}
x := &grpc.GenericClientStream[NodeServiceSubscribeRequest, NodeServiceSubscribeResponse]{ClientStream: stream}
if err := x.ClientStream.SendMsg(in); err != nil {
return nil, err
}
@@ -77,36 +77,25 @@ func (c *nodeServiceClient) Subscribe(ctx context.Context, in *NodeServiceSubscr
return x, nil
}
type NodeService_SubscribeClient interface {
Recv() (*NodeServiceSubscribeResponse, error)
grpc.ClientStream
}
type nodeServiceSubscribeClient struct {
grpc.ClientStream
}
func (x *nodeServiceSubscribeClient) Recv() (*NodeServiceSubscribeResponse, error) {
m := new(NodeServiceSubscribeResponse)
if err := x.ClientStream.RecvMsg(m); err != nil {
return nil, err
}
return m, nil
}
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
type NodeService_SubscribeClient = grpc.ServerStreamingClient[NodeServiceSubscribeResponse]
// NodeServiceServer is the server API for NodeService service.
// All implementations must embed UnimplementedNodeServiceServer
// for forward compatibility
// for forward compatibility.
type NodeServiceServer interface {
Register(context.Context, *NodeServiceRegisterRequest) (*Response, error)
SendHeartbeat(context.Context, *NodeServiceSendHeartbeatRequest) (*Response, error)
Subscribe(*NodeServiceSubscribeRequest, NodeService_SubscribeServer) error
Subscribe(*NodeServiceSubscribeRequest, grpc.ServerStreamingServer[NodeServiceSubscribeResponse]) error
mustEmbedUnimplementedNodeServiceServer()
}
// UnimplementedNodeServiceServer must be embedded to have forward compatible implementations.
type UnimplementedNodeServiceServer struct {
}
// UnimplementedNodeServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedNodeServiceServer struct{}
func (UnimplementedNodeServiceServer) Register(context.Context, *NodeServiceRegisterRequest) (*Response, error) {
return nil, status.Errorf(codes.Unimplemented, "method Register not implemented")
@@ -114,10 +103,11 @@ func (UnimplementedNodeServiceServer) Register(context.Context, *NodeServiceRegi
func (UnimplementedNodeServiceServer) SendHeartbeat(context.Context, *NodeServiceSendHeartbeatRequest) (*Response, error) {
return nil, status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented")
}
func (UnimplementedNodeServiceServer) Subscribe(*NodeServiceSubscribeRequest, NodeService_SubscribeServer) error {
func (UnimplementedNodeServiceServer) Subscribe(*NodeServiceSubscribeRequest, grpc.ServerStreamingServer[NodeServiceSubscribeResponse]) error {
return status.Errorf(codes.Unimplemented, "method Subscribe not implemented")
}
func (UnimplementedNodeServiceServer) mustEmbedUnimplementedNodeServiceServer() {}
func (UnimplementedNodeServiceServer) testEmbeddedByValue() {}
// UnsafeNodeServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to NodeServiceServer will
@@ -127,6 +117,13 @@ type UnsafeNodeServiceServer interface {
}
func RegisterNodeServiceServer(s grpc.ServiceRegistrar, srv NodeServiceServer) {
// If the following call pancis, it indicates UnimplementedNodeServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&NodeService_ServiceDesc, srv)
}
@@ -171,21 +168,11 @@ func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) e
if err := stream.RecvMsg(m); err != nil {
return err
}
return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{ServerStream: stream})
return srv.(NodeServiceServer).Subscribe(m, &grpc.GenericServerStream[NodeServiceSubscribeRequest, NodeServiceSubscribeResponse]{ServerStream: stream})
}
type NodeService_SubscribeServer interface {
Send(*NodeServiceSubscribeResponse) error
grpc.ServerStream
}
type nodeServiceSubscribeServer struct {
grpc.ServerStream
}
func (x *nodeServiceSubscribeServer) Send(m *NodeServiceSubscribeResponse) error {
return x.ServerStream.SendMsg(m)
}
// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name.
type NodeService_SubscribeServer = grpc.ServerStreamingServer[NodeServiceSubscribeResponse]
// NodeService_ServiceDesc is the grpc.ServiceDesc for NodeService service.
// It's only intended for direct use with grpc.RegisterService,