mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-26 17:49:15 +01:00
feat(sync): add gRPC file synchronization service and integrate end-to-end
- add proto/services/sync_service.proto and generate Go pb + grpc bindings - implement SyncServiceServer (streaming file scan + download) with: - request deduplication, in-memory cache (TTL), chunked streaming - concurrent-safe broadcast to waiters and server-side logging - register SyncSvr in gRPC server and expose sync client in GrpcClient: - add syncClient field, registration and safe getters with reconnection-aware timeouts - integrate gRPC sync into runner: - split syncFiles into syncFilesHTTP (legacy) and syncFilesGRPC - Runner now chooses implementation via config flag and performs streaming scan/download - controller improvements: - add semaphore-based rate limiting for sync scan requests with in-flight counters and logs - misc: - add utils.IsSyncGrpcEnabled() config helper - improve HTTP sync error diagnostics (Content-Type validation, response previews) - update/regenerate many protobuf and gRPC generated files (protoc/protoc-gen-go / protoc-gen-go-grpc version bumps)
This commit is contained in:
Marvin Zhang
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.34.2
|
||||
// protoc v5.27.2
|
||||
// protoc-gen-go v1.36.1
|
||||
// protoc v5.29.2
|
||||
// source: services/node_service.proto
|
||||
|
||||
package grpc
|
||||
@@ -64,22 +64,19 @@ func (NodeServiceSubscribeCode) EnumDescriptor() ([]byte, []int) {
|
||||
}
|
||||
|
||||
type NodeServiceRegisterRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"`
|
||||
MaxRunners int32 `protobuf:"varint,3,opt,name=max_runners,json=maxRunners,proto3" json:"max_runners,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
NodeName string `protobuf:"bytes,2,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"`
|
||||
MaxRunners int32 `protobuf:"varint,3,opt,name=max_runners,json=maxRunners,proto3" json:"max_runners,omitempty"`
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *NodeServiceRegisterRequest) Reset() {
|
||||
*x = NodeServiceRegisterRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_services_node_service_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
mi := &file_services_node_service_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *NodeServiceRegisterRequest) String() string {
|
||||
@@ -90,7 +87,7 @@ func (*NodeServiceRegisterRequest) ProtoMessage() {}
|
||||
|
||||
func (x *NodeServiceRegisterRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_services_node_service_proto_msgTypes[0]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -127,20 +124,17 @@ func (x *NodeServiceRegisterRequest) GetMaxRunners() int32 {
|
||||
}
|
||||
|
||||
type NodeServiceSendHeartbeatRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *NodeServiceSendHeartbeatRequest) Reset() {
|
||||
*x = NodeServiceSendHeartbeatRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_services_node_service_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
mi := &file_services_node_service_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *NodeServiceSendHeartbeatRequest) String() string {
|
||||
@@ -151,7 +145,7 @@ func (*NodeServiceSendHeartbeatRequest) ProtoMessage() {}
|
||||
|
||||
func (x *NodeServiceSendHeartbeatRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_services_node_service_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -174,20 +168,17 @@ func (x *NodeServiceSendHeartbeatRequest) GetNodeKey() string {
|
||||
}
|
||||
|
||||
type NodeServiceSubscribeRequest struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"`
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *NodeServiceSubscribeRequest) Reset() {
|
||||
*x = NodeServiceSubscribeRequest{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_services_node_service_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
mi := &file_services_node_service_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *NodeServiceSubscribeRequest) String() string {
|
||||
@@ -198,7 +189,7 @@ func (*NodeServiceSubscribeRequest) ProtoMessage() {}
|
||||
|
||||
func (x *NodeServiceSubscribeRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_services_node_service_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -221,20 +212,17 @@ func (x *NodeServiceSubscribeRequest) GetNodeKey() string {
|
||||
}
|
||||
|
||||
type NodeServiceSubscribeResponse struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Code NodeServiceSubscribeCode `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.NodeServiceSubscribeCode" json:"code,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Code NodeServiceSubscribeCode `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.NodeServiceSubscribeCode" json:"code,omitempty"`
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *NodeServiceSubscribeResponse) Reset() {
|
||||
*x = NodeServiceSubscribeResponse{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_services_node_service_proto_msgTypes[3]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
mi := &file_services_node_service_proto_msgTypes[3]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *NodeServiceSubscribeResponse) String() string {
|
||||
@@ -245,7 +233,7 @@ func (*NodeServiceSubscribeResponse) ProtoMessage() {}
|
||||
|
||||
func (x *NodeServiceSubscribeResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_services_node_service_proto_msgTypes[3]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -358,56 +346,6 @@ func file_services_node_service_proto_init() {
|
||||
return
|
||||
}
|
||||
file_entity_response_proto_init()
|
||||
if !protoimpl.UnsafeEnabled {
|
||||
file_services_node_service_proto_msgTypes[0].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*NodeServiceRegisterRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_services_node_service_proto_msgTypes[1].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*NodeServiceSendHeartbeatRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_services_node_service_proto_msgTypes[2].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*NodeServiceSubscribeRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_services_node_service_proto_msgTypes[3].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*NodeServiceSubscribeResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
|
||||
Reference in New Issue
Block a user