DNF90 服务端与启动器完整源码(含 GM 工具功能增强)
服务端(Go): - GM 塞背包/带属性装备发放:在线角色推送 op14 增量物品行,背包实时刷新,无需回角色选择界面 - GM 改金币/改等级:去掉在线拒绝,落库后推送 op14 钱包/op37 等级状态,在线实时刷新 - 等级端点统一切换到新版 GMSetCharacterLevel(自带在线刷新) - 无限/锁死疲劳模式 + 燃烧疲劳:每房间消耗 2 点、打怪与通关经验双倍 - 燃烧疲劳开关时给所有在线角色推送系统提示(op12 mode 0) - 深渊史诗掉率:GM 全局倍率 + 城镇增益药剂窗口倍率,真实接入掉落结算 启动器(Python/JS): - 标题栏自绘、窗口 1204x717、禁止拖拽改大小、仅最小化/关闭 - GM 工具:疲劳与燃烧、深渊史诗掉率、拍卖行上架(单一定价对齐) - 拍卖行上架前端对齐服务端单一价格语义,修复 400 报错
This commit is contained in:
commit
645b64fe21
4388 files changed
+1405457
No files matched your search
@@ -0,0 +1,345 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"longheng.io/server/internal/platform/replay"
|
||||
"longheng.io/server/pkg/contracts"
|
||||
)
|
||||
|
||||
// 历史背景:LongHeng 的 internal RPC 直到本改动前完全明文 —— Bus / NATS / Redis
|
||||
// PubSub 上谁能连进来就能伪造 RPCRequest。生产部署只能靠网络隔离做防线,
|
||||
// 一旦 sidecar 漏接、test pod 误配置或同集群另一个服务被攻陷,攻击者就能
|
||||
// 直接调用 logic 内部 handler。
|
||||
//
|
||||
// 本文件提供应用层 HMAC-SHA256 鉴权,作为 mTLS / 内网隔离之外的纵深防御:
|
||||
// - 共享 secret 由部署期 Secret/KMS 注入,旋转策略由项目侧实现。
|
||||
// - 签名覆盖关键字段(RequestID / SourceNodeID / TargetNodeID / Route /
|
||||
// SentAt / payload 哈希),任何字段被改都会失效。
|
||||
// - 时间戳偏移检查缩小重放窗口;生产多节点应接入 replay.RedisStore 共享 nonce 表。
|
||||
// - Sign* / Verify* 是无状态纯函数,便于在 AuthenticatedTransport 之外的
|
||||
// 场景(例如直接走 HTTP 的 Admin 内部接口)单独使用。
|
||||
|
||||
const (
|
||||
// AuthSignatureMetadataKey 是签名字段在 RPCRequest.Metadata /
|
||||
// RPCResponse.Metadata 上的固定键。
|
||||
AuthSignatureMetadataKey = "x-longheng-signature"
|
||||
// AuthAlgorithmMetadataKey 用于在 metadata 里声明算法版本,便于未来兼容
|
||||
// 多版本(HMAC-SHA256 / HMAC-SHA512 / Ed25519)。
|
||||
AuthAlgorithmMetadataKey = "x-longheng-signature-alg"
|
||||
// AuthAlgorithmHMACSHA256 是当前唯一支持的算法标识。
|
||||
AuthAlgorithmHMACSHA256 = "HMAC-SHA256"
|
||||
// DefaultAuthMaxSkew 是默认的双向时间偏移容差。生产建议保持 5min 量级,
|
||||
// 太小容易因为 NTP 抖动误拒,太大会扩大重放窗口。
|
||||
DefaultAuthMaxSkew = 5 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrAuthSecretRequired 在构造 Authenticator 时 secret 为空返回。
|
||||
ErrAuthSecretRequired = errors.New("rpcauth: shared secret is required")
|
||||
// ErrAuthSignatureMissing 表示请求 / 响应缺少签名 metadata。
|
||||
ErrAuthSignatureMissing = errors.New("rpcauth: signature missing")
|
||||
// ErrAuthSignatureInvalid 表示签名不匹配(密钥不对或字段被改)。
|
||||
ErrAuthSignatureInvalid = errors.New("rpcauth: signature invalid")
|
||||
// ErrAuthAlgUnsup 表示对端声明了不支持的签名算法。
|
||||
ErrAuthAlgUnsup = errors.New("rpcauth: algorithm not supported")
|
||||
// ErrAuthTimestampSkew 表示请求 / 响应的 SentAt 偏移超出允许窗口。
|
||||
ErrAuthTimestampSkew = errors.New("rpcauth: timestamp outside max skew")
|
||||
// ErrAuthReqNoMeta 在签名时 RPCRequest 为 nil 等场景下返回。
|
||||
ErrAuthReqNoMeta = errors.New("rpcauth: request is nil")
|
||||
// ErrAuthReplayDetected 表示同一 SourceNodeID + RequestID 已在 MaxSkew 窗口内验过;可用 errors.Is 判定。
|
||||
ErrAuthReplayDetected = errors.New("rpcauth: replay detected")
|
||||
// ErrAuthReplayCheckFailed 表示 nonce store 不可用;可用 errors.Is 保留底层 replay/redis 错误。
|
||||
ErrAuthReplayCheckFailed = errors.New("rpcauth: replay check failed")
|
||||
)
|
||||
|
||||
// Authenticator 持有共享密钥和默认偏移窗口,是 RPC 鉴权的核心入口。
|
||||
// 所有方法在常量时间比较签名(使用 hmac.Equal / subtle.ConstantTimeCompare),
|
||||
// 不会通过失败用时长泄露签名长度。
|
||||
type Authenticator struct {
|
||||
secret []byte
|
||||
maxSkew time.Duration
|
||||
now func() time.Time
|
||||
replay replay.Store
|
||||
}
|
||||
|
||||
// AuthenticatorOptions 用于构造 Authenticator。Secret 必填;MaxSkew=0 使用
|
||||
// DefaultAuthMaxSkew;ReplayStore 为空时保持旧的纯签名校验语义。
|
||||
type AuthenticatorOptions struct {
|
||||
Secret []byte
|
||||
MaxSkew time.Duration
|
||||
Now func() time.Time
|
||||
ReplayStore replay.Store
|
||||
}
|
||||
|
||||
// NewAuthenticator 校验配置并返回 Authenticator。secret 为空会返回
|
||||
// ErrAuthSecretRequired,避免装配代码静默退化为"无鉴权"。
|
||||
func NewAuthenticator(opts AuthenticatorOptions) (*Authenticator, error) {
|
||||
if len(opts.Secret) == 0 {
|
||||
return nil, ErrAuthSecretRequired
|
||||
}
|
||||
skew := opts.MaxSkew
|
||||
if skew <= 0 {
|
||||
skew = DefaultAuthMaxSkew
|
||||
}
|
||||
now := opts.Now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
// 复制 secret,避免调用方后续修改影响 Authenticator。
|
||||
secret := make([]byte, len(opts.Secret))
|
||||
copy(secret, opts.Secret)
|
||||
return &Authenticator{secret: secret, maxSkew: skew, now: now, replay: opts.ReplayStore}, nil
|
||||
}
|
||||
|
||||
// SignRequest 计算 RPCRequest 的 HMAC 签名并写到 Metadata。req 不能为 nil。
|
||||
// SentAt 为零值时会自动填当前时间;Metadata 为 nil 时会被初始化。
|
||||
func (a *Authenticator) SignRequest(req *contracts.RPCRequest) error {
|
||||
if a == nil {
|
||||
return ErrAuthSecretRequired
|
||||
}
|
||||
if req == nil {
|
||||
return ErrAuthReqNoMeta
|
||||
}
|
||||
if req.SentAt.IsZero() {
|
||||
req.SentAt = a.now().UTC()
|
||||
}
|
||||
if req.Metadata == nil {
|
||||
req.Metadata = map[string]string{}
|
||||
}
|
||||
sig := a.requestSignature(*req)
|
||||
req.Metadata[AuthSignatureMetadataKey] = sig
|
||||
req.Metadata[AuthAlgorithmMetadataKey] = AuthAlgorithmHMACSHA256
|
||||
return nil
|
||||
}
|
||||
|
||||
// VerifyRequest 检查 RPCRequest 是否有合法签名 + 时间戳在允许偏移内。
|
||||
// 缺签名返回 ErrAuthSignatureMissing;签名不匹配返回 ErrAuthSignatureInvalid;
|
||||
// 时间戳偏移过大返回 ErrAuthTimestampSkew;算法不支持返回 ErrAuthAlgUnsup;
|
||||
// nonce 重复返回 ErrAuthReplayDetected。
|
||||
func (a *Authenticator) VerifyRequest(req contracts.RPCRequest) error {
|
||||
return a.VerifyRequestContext(context.Background(), req)
|
||||
}
|
||||
|
||||
// VerifyRequestContext 与 VerifyRequest 语义一致,但会把 ctx 传给 ReplayStore。
|
||||
// 生产 Redis nonce 表应继承调用链超时,否则鉴权路径可能被 Redis 卡死拖垮 handler。
|
||||
func (a *Authenticator) VerifyRequestContext(ctx context.Context, req contracts.RPCRequest) error {
|
||||
if a == nil {
|
||||
return ErrAuthSecretRequired
|
||||
}
|
||||
sig := strings.TrimSpace(req.Metadata[AuthSignatureMetadataKey])
|
||||
if sig == "" {
|
||||
return ErrAuthSignatureMissing
|
||||
}
|
||||
if alg := strings.TrimSpace(req.Metadata[AuthAlgorithmMetadataKey]); alg != "" && alg != AuthAlgorithmHMACSHA256 {
|
||||
return fmt.Errorf("%w: %s", ErrAuthAlgUnsup, alg)
|
||||
}
|
||||
if strings.TrimSpace(req.RequestID) == "" || strings.TrimSpace(req.SourceNodeID) == "" {
|
||||
return fmt.Errorf("%w: request_id and source_node_id are required", ErrAuthSignatureMissing)
|
||||
}
|
||||
expected := a.requestSignature(req)
|
||||
if subtle.ConstantTimeCompare([]byte(sig), []byte(expected)) != 1 {
|
||||
return ErrAuthSignatureInvalid
|
||||
}
|
||||
if err := a.checkSkew(req.SentAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.checkReplay(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SignResponse 与 SignRequest 对称,用于网关 / logic 之间的响应消息。
|
||||
func (a *Authenticator) SignResponse(resp *contracts.RPCResponse) error {
|
||||
if a == nil {
|
||||
return ErrAuthSecretRequired
|
||||
}
|
||||
if resp == nil {
|
||||
return ErrAuthReqNoMeta
|
||||
}
|
||||
if resp.SentAt.IsZero() {
|
||||
resp.SentAt = a.now().UTC()
|
||||
}
|
||||
if resp.Metadata == nil {
|
||||
resp.Metadata = map[string]string{}
|
||||
}
|
||||
sig := a.responseSignature(*resp)
|
||||
resp.Metadata[AuthSignatureMetadataKey] = sig
|
||||
resp.Metadata[AuthAlgorithmMetadataKey] = AuthAlgorithmHMACSHA256
|
||||
return nil
|
||||
}
|
||||
|
||||
// VerifyResponse 校验 RPCResponse 的签名。语义与 VerifyRequest 一致。
|
||||
func (a *Authenticator) VerifyResponse(resp contracts.RPCResponse) error {
|
||||
return a.VerifyResponseContext(context.Background(), resp)
|
||||
}
|
||||
|
||||
// VerifyResponseContext 与 VerifyResponse 语义一致,但会把 ctx 传给 ReplayStore。
|
||||
func (a *Authenticator) VerifyResponseContext(ctx context.Context, resp contracts.RPCResponse) error {
|
||||
if a == nil {
|
||||
return ErrAuthSecretRequired
|
||||
}
|
||||
sig := strings.TrimSpace(resp.Metadata[AuthSignatureMetadataKey])
|
||||
if sig == "" {
|
||||
return ErrAuthSignatureMissing
|
||||
}
|
||||
if alg := strings.TrimSpace(resp.Metadata[AuthAlgorithmMetadataKey]); alg != "" && alg != AuthAlgorithmHMACSHA256 {
|
||||
return fmt.Errorf("%w: %s", ErrAuthAlgUnsup, alg)
|
||||
}
|
||||
if strings.TrimSpace(resp.RequestID) == "" || strings.TrimSpace(resp.SourceNodeID) == "" || strings.TrimSpace(resp.TargetNodeID) == "" {
|
||||
return fmt.Errorf("%w: request_id, source_node_id and target_node_id are required", ErrAuthSignatureMissing)
|
||||
}
|
||||
expected := a.responseSignature(resp)
|
||||
if subtle.ConstantTimeCompare([]byte(sig), []byte(expected)) != 1 {
|
||||
return ErrAuthSignatureInvalid
|
||||
}
|
||||
if err := a.checkSkew(resp.SentAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := a.checkResponseReplay(ctx, resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaxSkew 暴露当前的偏移窗口,便于装配代码做断言或日志。
|
||||
func (a *Authenticator) MaxSkew() time.Duration {
|
||||
if a == nil {
|
||||
return 0
|
||||
}
|
||||
return a.maxSkew
|
||||
}
|
||||
|
||||
func (a *Authenticator) checkSkew(at time.Time) error {
|
||||
if a.maxSkew <= 0 {
|
||||
return nil
|
||||
}
|
||||
if at.IsZero() {
|
||||
return ErrAuthTimestampSkew
|
||||
}
|
||||
now := a.now().UTC()
|
||||
diff := now.Sub(at.UTC())
|
||||
if diff > a.maxSkew || diff < -a.maxSkew {
|
||||
return ErrAuthTimestampSkew
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Authenticator) checkReplay(ctx context.Context, req contracts.RPCRequest) error {
|
||||
if a.replay == nil {
|
||||
return nil
|
||||
}
|
||||
return a.checkReplayKey(ctx, replayKey(req))
|
||||
}
|
||||
|
||||
func (a *Authenticator) checkResponseReplay(ctx context.Context, resp contracts.RPCResponse) error {
|
||||
if a.replay == nil {
|
||||
return nil
|
||||
}
|
||||
return a.checkReplayKey(ctx, responseReplayKey(resp))
|
||||
}
|
||||
|
||||
func (a *Authenticator) checkReplayKey(ctx context.Context, key string) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
first, err := a.replay.Seen(ctx, key, authReplayTTL(a.maxSkew))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", ErrAuthReplayCheckFailed, err)
|
||||
}
|
||||
if !first {
|
||||
return ErrAuthReplayDetected
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replayKey(req contracts.RPCRequest) string {
|
||||
source := strings.TrimSpace(req.SourceNodeID)
|
||||
requestID := strings.TrimSpace(req.RequestID)
|
||||
if source == "" || requestID == "" {
|
||||
return ""
|
||||
}
|
||||
return "v1:" + encodeReplayKeyPart(source) + ":" + encodeReplayKeyPart(requestID)
|
||||
}
|
||||
|
||||
func responseReplayKey(resp contracts.RPCResponse) string {
|
||||
source := strings.TrimSpace(resp.SourceNodeID)
|
||||
target := strings.TrimSpace(resp.TargetNodeID)
|
||||
requestID := strings.TrimSpace(resp.RequestID)
|
||||
if source == "" || target == "" || requestID == "" {
|
||||
return ""
|
||||
}
|
||||
return "v1:resp:" + encodeReplayKeyPart(source) + ":" + encodeReplayKeyPart(target) + ":" + encodeReplayKeyPart(requestID)
|
||||
}
|
||||
|
||||
func encodeReplayKeyPart(value string) string {
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(value))
|
||||
}
|
||||
|
||||
func authReplayTTL(maxSkew time.Duration) time.Duration {
|
||||
if maxSkew <= 0 {
|
||||
return 2*DefaultAuthMaxSkew + time.Minute
|
||||
}
|
||||
return 2*maxSkew + time.Minute
|
||||
}
|
||||
|
||||
// requestSignature 拼出 canonical string 然后 HMAC。canonical 故意排除
|
||||
// Metadata(避免循环依赖签名字段本身),只覆盖 routing / identity / 时间
|
||||
// / payload 哈希这几个攻击面最大的字段。
|
||||
func (a *Authenticator) requestSignature(req contracts.RPCRequest) string {
|
||||
canonical := strings.Join([]string{
|
||||
"rpc.request",
|
||||
req.RequestID,
|
||||
req.SourceService,
|
||||
req.SourceNodeID,
|
||||
req.TargetNodeID,
|
||||
req.Route,
|
||||
req.SentAt.UTC().Format(time.RFC3339Nano),
|
||||
hashPayload(req.Payload),
|
||||
}, "\n")
|
||||
return a.compute(canonical)
|
||||
}
|
||||
|
||||
// responseSignature 与 request 对称,但用 Error 字段替代 Route,强制响应里
|
||||
// 即使内容相同的失败也对应不同的签名(攻击者无法复制一个 OK 响应去顶替一个失败的)。
|
||||
func (a *Authenticator) responseSignature(resp contracts.RPCResponse) string {
|
||||
canonical := strings.Join([]string{
|
||||
"rpc.response",
|
||||
resp.RequestID,
|
||||
resp.SourceNodeID,
|
||||
resp.TargetNodeID,
|
||||
resp.Error,
|
||||
resp.SentAt.UTC().Format(time.RFC3339Nano),
|
||||
hashPayload(resp.Payload),
|
||||
}, "\n")
|
||||
return a.compute(canonical)
|
||||
}
|
||||
|
||||
func (a *Authenticator) compute(canonical string) string {
|
||||
mac := hmac.New(sha256.New, a.secret)
|
||||
_, _ = mac.Write([]byte(canonical))
|
||||
return hex.EncodeToString(mac.Sum(nil))
|
||||
}
|
||||
|
||||
// hashPayload 把 payload 摘要成 sha256,避免 canonical string 巨大、CPU 浪费在
|
||||
// HMAC 内部分块上。空 payload 显式返回固定标记,便于跨语言重现。
|
||||
func hashPayload(payload []byte) string {
|
||||
if len(payload) == 0 {
|
||||
return "empty"
|
||||
}
|
||||
sum := sha256.Sum256(payload)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"longheng.io/server/pkg/contracts"
|
||||
)
|
||||
|
||||
// RequireMode 描述对端没有签名时是否允许通过;签名"错误"始终拒绝。
|
||||
//
|
||||
// RequireAll 缺签名或签名错误都拒绝。生产推荐。
|
||||
// RequirePresent 缺签名放行(灰度),签名错误仍拒绝。仅用于"双方还没全部接入鉴权"
|
||||
// 的滚动升级窗口。完成滚动后应立刻切到 RequireAll。
|
||||
type RequireMode string
|
||||
|
||||
const (
|
||||
// RequireAll 表示缺签名和签名错误都拒绝,是生产推荐模式。
|
||||
RequireAll RequireMode = "all"
|
||||
// RequirePresent 表示缺签名暂时放行,但签名错误仍拒绝。
|
||||
RequirePresent RequireMode = "present"
|
||||
)
|
||||
|
||||
// AuthTransportMetrics 是 AuthenticatedTransport 的指标钩子。所有字段都是
|
||||
// 64-bit counter,调用方可以原子读取作为 Prometheus / scope 输出。
|
||||
// 调用方可以 nil,AuthenticatedTransport 自己持有一个匿名实例做内部统计。
|
||||
type AuthTransportMetrics struct {
|
||||
SignedRequests atomic.Uint64
|
||||
SignedResponses atomic.Uint64
|
||||
AcceptedRequests atomic.Uint64
|
||||
AcceptedResponses atomic.Uint64
|
||||
RejectedMissing atomic.Uint64
|
||||
RejectedInvalid atomic.Uint64
|
||||
RejectedSkew atomic.Uint64
|
||||
RejectedAlgorithm atomic.Uint64
|
||||
RejectedReplay atomic.Uint64
|
||||
RejectedReplayCheck atomic.Uint64
|
||||
GraceLetThrough atomic.Uint64
|
||||
SignFailures atomic.Uint64
|
||||
}
|
||||
|
||||
// AuthenticatedTransport 包装任意 Transport,发送侧自动签名,接收侧自动验签。
|
||||
// 不引入新的拓扑或网络层,对调用方完全透明。
|
||||
type AuthenticatedTransport struct {
|
||||
inner Transport
|
||||
auth *Authenticator
|
||||
require RequireMode
|
||||
metrics *AuthTransportMetrics
|
||||
}
|
||||
|
||||
// AuthTransportOptions 是装配参数。Auth 必填;Inner 通常是 BusTransport
|
||||
// 或 GRPCTransport。Require 缺省 RequireAll。
|
||||
type AuthTransportOptions struct {
|
||||
Inner Transport
|
||||
Auth *Authenticator
|
||||
Require RequireMode
|
||||
Metrics *AuthTransportMetrics
|
||||
}
|
||||
|
||||
// NewAuthenticatedTransport 构造一个签名 / 验签包装。inner 或 auth 为 nil 返回错误,
|
||||
// 拒绝静默退化为"不鉴权"。
|
||||
func NewAuthenticatedTransport(opts AuthTransportOptions) (*AuthenticatedTransport, error) {
|
||||
if opts.Inner == nil {
|
||||
return nil, errors.New("rpcauth: inner transport is required")
|
||||
}
|
||||
if opts.Auth == nil {
|
||||
return nil, errors.New("rpcauth: authenticator is required")
|
||||
}
|
||||
require := opts.Require
|
||||
switch require {
|
||||
case "":
|
||||
require = RequireAll
|
||||
case RequireAll, RequirePresent:
|
||||
default:
|
||||
return nil, fmt.Errorf("rpcauth: unsupported require mode %q", require)
|
||||
}
|
||||
metrics := opts.Metrics
|
||||
if metrics == nil {
|
||||
metrics = &AuthTransportMetrics{}
|
||||
}
|
||||
return &AuthenticatedTransport{
|
||||
inner: opts.Inner,
|
||||
auth: opts.Auth,
|
||||
require: require,
|
||||
metrics: metrics,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Metrics 暴露内部计数器,便于 wiring 层挂到 metrics.Scope 上。
|
||||
func (t *AuthenticatedTransport) Metrics() *AuthTransportMetrics {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
return t.metrics
|
||||
}
|
||||
|
||||
// Require 暴露当前模式,主要用于自检 / 日志。
|
||||
func (t *AuthenticatedTransport) Require() RequireMode {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.require
|
||||
}
|
||||
|
||||
// SubscribeRequests 订阅请求并在交给业务 handler 前完成验签。
|
||||
func (t *AuthenticatedTransport) SubscribeRequests(ctx context.Context, nodeID string, handler RequestHandler) (TransportSubscription, error) {
|
||||
if t == nil {
|
||||
return nil, errors.New("rpcauth: transport is nil")
|
||||
}
|
||||
return t.inner.SubscribeRequests(ctx, nodeID, func(ctx context.Context, req contracts.RPCRequest) error {
|
||||
if err := t.auth.VerifyRequestContext(ctx, req); err != nil {
|
||||
t.recordVerifyFailure(err)
|
||||
if t.shouldAcceptDespite(err) {
|
||||
return handler(ctx, req)
|
||||
}
|
||||
return err
|
||||
}
|
||||
t.metrics.AcceptedRequests.Add(1)
|
||||
return handler(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
// SubscribeResponses 订阅响应并在交给等待方前完成验签。
|
||||
func (t *AuthenticatedTransport) SubscribeResponses(ctx context.Context, nodeID string, handler ResponseHandler) (TransportSubscription, error) {
|
||||
if t == nil {
|
||||
return nil, errors.New("rpcauth: transport is nil")
|
||||
}
|
||||
return t.inner.SubscribeResponses(ctx, nodeID, func(ctx context.Context, resp contracts.RPCResponse) error {
|
||||
if err := t.auth.VerifyResponseContext(ctx, resp); err != nil {
|
||||
t.recordVerifyFailure(err)
|
||||
if t.shouldAcceptDespite(err) {
|
||||
return handler(ctx, resp)
|
||||
}
|
||||
return err
|
||||
}
|
||||
t.metrics.AcceptedResponses.Add(1)
|
||||
return handler(ctx, resp)
|
||||
})
|
||||
}
|
||||
|
||||
// PublishRequest 对请求签名后交给底层 transport 发布。
|
||||
func (t *AuthenticatedTransport) PublishRequest(ctx context.Context, targetNodeID string, request contracts.RPCRequest) error {
|
||||
if t == nil {
|
||||
return errors.New("rpcauth: transport is nil")
|
||||
}
|
||||
if err := t.auth.SignRequest(&request); err != nil {
|
||||
t.metrics.SignFailures.Add(1)
|
||||
return err
|
||||
}
|
||||
t.metrics.SignedRequests.Add(1)
|
||||
return t.inner.PublishRequest(ctx, targetNodeID, request)
|
||||
}
|
||||
|
||||
// PublishResponse 对响应签名后交给底层 transport 发布。
|
||||
func (t *AuthenticatedTransport) PublishResponse(ctx context.Context, targetNodeID string, response contracts.RPCResponse) error {
|
||||
if t == nil {
|
||||
return errors.New("rpcauth: transport is nil")
|
||||
}
|
||||
if err := t.auth.SignResponse(&response); err != nil {
|
||||
t.metrics.SignFailures.Add(1)
|
||||
return err
|
||||
}
|
||||
t.metrics.SignedResponses.Add(1)
|
||||
return t.inner.PublishResponse(ctx, targetNodeID, response)
|
||||
}
|
||||
|
||||
// shouldAcceptDespite 区分"可灰度通过"和"必须拒绝"的失败。
|
||||
// 只有 ErrAuthSignatureMissing 在 RequirePresent 模式下允许放行;
|
||||
// invalid / skew / algorithm 一律拒绝,无论什么模式。
|
||||
func (t *AuthenticatedTransport) shouldAcceptDespite(err error) bool {
|
||||
if t.require != RequirePresent {
|
||||
return false
|
||||
}
|
||||
if !errors.Is(err, ErrAuthSignatureMissing) {
|
||||
return false
|
||||
}
|
||||
t.metrics.GraceLetThrough.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
func (t *AuthenticatedTransport) recordVerifyFailure(err error) {
|
||||
switch {
|
||||
case errors.Is(err, ErrAuthSignatureMissing):
|
||||
t.metrics.RejectedMissing.Add(1)
|
||||
case errors.Is(err, ErrAuthSignatureInvalid):
|
||||
t.metrics.RejectedInvalid.Add(1)
|
||||
case errors.Is(err, ErrAuthTimestampSkew):
|
||||
t.metrics.RejectedSkew.Add(1)
|
||||
case errors.Is(err, ErrAuthAlgUnsup):
|
||||
t.metrics.RejectedAlgorithm.Add(1)
|
||||
case errors.Is(err, ErrAuthReplayDetected):
|
||||
t.metrics.RejectedReplay.Add(1)
|
||||
case errors.Is(err, ErrAuthReplayCheckFailed):
|
||||
t.metrics.RejectedReplayCheck.Add(1)
|
||||
default:
|
||||
// 未知错误不分类,但仍然计入 invalid,避免悄悄漏统计。
|
||||
t.metrics.RejectedInvalid.Add(1)
|
||||
// 兜底,让调用方在日志里能看到具体类型。
|
||||
_ = strings.TrimSpace(err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Package rpc 负责在内存、本地 Bus 和远端传输端点之间路由内部服务调用。
|
||||
//
|
||||
// Endpoint API 会保留请求元数据、追踪上下文、pending 调用上限和 payload 预算。
|
||||
// Reliable 调用只负责重试临时失败;是否具备持久投递能力取决于配置的 transport / bus 后端。
|
||||
package rpc
|
||||
@@ -0,0 +1,892 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"longheng.io/server/internal/platform/bus"
|
||||
"longheng.io/server/internal/platform/discovery"
|
||||
"longheng.io/server/internal/platform/metrics"
|
||||
"longheng.io/server/internal/platform/registry"
|
||||
"longheng.io/server/internal/platform/tracing"
|
||||
"longheng.io/server/pkg/contracts"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrInvalidEndpoint 表示 RPC endpoint 为空或没有完成基础初始化。
|
||||
ErrInvalidEndpoint = errors.New("rpc endpoint is invalid")
|
||||
// ErrNotStarted 表示 RPC endpoint 还没有启动或已经停止。
|
||||
ErrNotStarted = errors.New("rpc endpoint is not started")
|
||||
// ErrInvalidNodeID 表示当前节点 ID 或来源节点 ID 为空。
|
||||
ErrInvalidNodeID = errors.New("rpc node id is required")
|
||||
// ErrInvalidTarget 表示目标节点 ID 为空。
|
||||
ErrInvalidTarget = errors.New("rpc target node id is required")
|
||||
// ErrInvalidService 表示目标服务名为空。
|
||||
ErrInvalidService = errors.New("rpc target service is required")
|
||||
// ErrInvalidRoute 表示 RPC 路由为空。
|
||||
ErrInvalidRoute = errors.New("rpc route is required")
|
||||
// ErrInvalidRequestID 表示请求 ID 为空。
|
||||
ErrInvalidRequestID = errors.New("rpc request id is required")
|
||||
// ErrNoTargetNodes 表示服务发现或路由解析没有可用目标节点。
|
||||
ErrNoTargetNodes = errors.New("rpc target service has no nodes")
|
||||
// ErrTooManyPending 表示等待响应的调用数量已经达到上限。
|
||||
ErrTooManyPending = errors.New("rpc pending call limit exceeded")
|
||||
// ErrPayloadTooLarge 表示请求或响应 payload 超过配置上限。
|
||||
ErrPayloadTooLarge = errors.New("rpc payload exceeds max size")
|
||||
)
|
||||
|
||||
// Handler 是 endpoint 收到业务 RPC 请求后的处理函数。
|
||||
type Handler func(context.Context, contracts.RPCRequest) (contracts.RPCResponse, error)
|
||||
|
||||
// Resolver 根据服务名和分片 key 解析目标节点 ID。
|
||||
type Resolver func(context.Context, string, string) (string, bool, error)
|
||||
|
||||
// Options 控制 endpoint 的调用超时、并发等待上限和 payload 大小预算。
|
||||
type Options struct {
|
||||
CallTimeout time.Duration
|
||||
MaxPending int
|
||||
MaxPayloadBytes int
|
||||
}
|
||||
|
||||
// RetryOptions 控制可靠调用的重试次数、退避和远端错误重试策略。
|
||||
type RetryOptions struct {
|
||||
Attempts int
|
||||
Backoff time.Duration
|
||||
MaxBackoff time.Duration
|
||||
IdempotencyKey string
|
||||
RetryRemoteErrors bool
|
||||
}
|
||||
|
||||
// Snapshot 是 endpoint 当前运行状态的只读快照。
|
||||
type Snapshot struct {
|
||||
ServiceName string `json:"service_name"`
|
||||
NodeID string `json:"node_id"`
|
||||
Started bool `json:"started"`
|
||||
Pending int `json:"pending"`
|
||||
Handlers []string `json:"handlers"`
|
||||
CallTimeoutSeconds float64 `json:"call_timeout_seconds"`
|
||||
MaxPending int `json:"max_pending"`
|
||||
MaxPayloadBytes int `json:"max_payload_bytes"`
|
||||
}
|
||||
|
||||
// RemoteError 表示目标节点 handler 返回的远端业务错误。
|
||||
type RemoteError struct {
|
||||
RequestID string
|
||||
TargetNodeID string
|
||||
Route string
|
||||
Message string
|
||||
}
|
||||
|
||||
// Error 返回包含目标节点和路由的远端错误文本。
|
||||
func (e RemoteError) Error() string {
|
||||
if e.Route == "" {
|
||||
return fmt.Sprintf("rpc remote error from %s: %s", e.TargetNodeID, e.Message)
|
||||
}
|
||||
return fmt.Sprintf("rpc remote error from %s route %s: %s", e.TargetNodeID, e.Route, e.Message)
|
||||
}
|
||||
|
||||
// Endpoint 是服务内 RPC 的注册、调用、响应等待和节点路由入口。
|
||||
type Endpoint struct {
|
||||
serviceName string
|
||||
nodeID string
|
||||
transport Transport
|
||||
logger *slog.Logger
|
||||
resolver Resolver
|
||||
discovery *discovery.Resolver
|
||||
tracer *tracing.Tracer
|
||||
options Options
|
||||
metrics *metrics.Registry
|
||||
metricLabels map[string]string
|
||||
|
||||
seq uint64
|
||||
|
||||
mu sync.RWMutex
|
||||
started bool
|
||||
handlers map[string]Handler
|
||||
pending map[string]chan contracts.RPCResponse
|
||||
subs []TransportSubscription
|
||||
}
|
||||
|
||||
// NewEndpoint 使用 bus transport 创建 endpoint。
|
||||
func NewEndpoint(serviceName, nodeID string, eventBus bus.Bus, logger *slog.Logger) *Endpoint {
|
||||
return NewEndpointWithTransport(serviceName, nodeID, NewBusTransport(eventBus), logger)
|
||||
}
|
||||
|
||||
// NewEndpointWithTransport 使用指定 transport 创建 endpoint。
|
||||
func NewEndpointWithTransport(serviceName, nodeID string, transport Transport, logger *slog.Logger) *Endpoint {
|
||||
return &Endpoint{
|
||||
serviceName: strings.TrimSpace(serviceName),
|
||||
nodeID: strings.TrimSpace(nodeID),
|
||||
transport: transport,
|
||||
logger: logger,
|
||||
handlers: make(map[string]Handler),
|
||||
pending: make(map[string]chan contracts.RPCResponse),
|
||||
options: Options{
|
||||
CallTimeout: 3 * time.Second,
|
||||
MaxPending: 1024,
|
||||
MaxPayloadBytes: 1024 * 1024,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Name 返回生命周期组件名。
|
||||
func (e *Endpoint) Name() string {
|
||||
return "rpc-endpoint"
|
||||
}
|
||||
|
||||
// SetResolver 设置轻量服务到节点解析函数。
|
||||
func (e *Endpoint) SetResolver(resolver Resolver) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.resolver = resolver
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetDiscoveryResolver 设置带健康反馈的 discovery resolver。
|
||||
func (e *Endpoint) SetDiscoveryResolver(resolver *discovery.Resolver) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.discovery = resolver
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetTracer 设置 RPC 调用和处理链路使用的 tracer。
|
||||
func (e *Endpoint) SetTracer(tracer *tracing.Tracer) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.tracer = tracer
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// Deprecated: 使用 SetOptionsExact 明确表达零值配置;该方法保留零值跳过语义以兼容早期热更新调用。
|
||||
func (e *Endpoint) SetOptions(options Options) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
next := e.options
|
||||
if options.CallTimeout != 0 {
|
||||
next.CallTimeout = options.CallTimeout
|
||||
}
|
||||
if options.MaxPending != 0 {
|
||||
next.MaxPending = options.MaxPending
|
||||
}
|
||||
if options.MaxPayloadBytes != 0 {
|
||||
next.MaxPayloadBytes = options.MaxPayloadBytes
|
||||
}
|
||||
e.options = normalizeOptions(next)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetOptionsExact 设置完整配置,零值也会按规范化规则生效。
|
||||
func (e *Endpoint) SetOptionsExact(options Options) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.options = normalizeOptions(options)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
// Snapshot 返回 endpoint 的稳定状态快照。
|
||||
func (e *Endpoint) Snapshot() Snapshot {
|
||||
if e == nil {
|
||||
return Snapshot{}
|
||||
}
|
||||
e.mu.RLock()
|
||||
handlers := make([]string, 0, len(e.handlers))
|
||||
for route := range e.handlers {
|
||||
handlers = append(handlers, route)
|
||||
}
|
||||
snapshot := Snapshot{
|
||||
ServiceName: e.serviceName,
|
||||
NodeID: e.nodeID,
|
||||
Started: e.started,
|
||||
Pending: len(e.pending),
|
||||
Handlers: handlers,
|
||||
CallTimeoutSeconds: e.options.CallTimeout.Seconds(),
|
||||
MaxPending: e.options.MaxPending,
|
||||
MaxPayloadBytes: e.options.MaxPayloadBytes,
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
sort.Strings(snapshot.Handlers)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// Register 注册或移除指定 RPC 路由的 handler。
|
||||
func (e *Endpoint) Register(route string, handler Handler) error {
|
||||
if e == nil {
|
||||
return ErrInvalidEndpoint
|
||||
}
|
||||
route = strings.TrimSpace(route)
|
||||
if route == "" {
|
||||
return ErrInvalidRoute
|
||||
}
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
e.ensureReadyLocked()
|
||||
if handler == nil {
|
||||
delete(e.handlers, route)
|
||||
return nil
|
||||
}
|
||||
e.handlers[route] = handler
|
||||
return nil
|
||||
}
|
||||
|
||||
// Start 订阅请求和响应主题,使 endpoint 开始接收 RPC 流量。
|
||||
func (e *Endpoint) Start(ctx context.Context) error {
|
||||
if e == nil {
|
||||
return ErrInvalidEndpoint
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(e.nodeID) == "" {
|
||||
return ErrInvalidNodeID
|
||||
}
|
||||
if e.transport == nil {
|
||||
return bus.ErrClosed
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
e.ensureReadyLocked()
|
||||
if e.started {
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
e.mu.Unlock()
|
||||
|
||||
requestSub, err := e.transport.SubscribeRequests(ctx, e.nodeID, e.handleRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
responseSub, err := e.transport.SubscribeResponses(ctx, e.nodeID, e.handleResponse)
|
||||
if err != nil {
|
||||
_ = requestSub.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
if e.started {
|
||||
e.mu.Unlock()
|
||||
_ = requestSub.Close()
|
||||
_ = responseSub.Close()
|
||||
return nil
|
||||
}
|
||||
e.subs = []TransportSubscription{requestSub, responseSub}
|
||||
e.started = true
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop 关闭订阅并唤醒所有等待中的调用。
|
||||
func (e *Endpoint) Stop(context.Context) error {
|
||||
if e == nil {
|
||||
return ErrInvalidEndpoint
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.ensureReadyLocked()
|
||||
subs := append([]TransportSubscription(nil), e.subs...)
|
||||
pending := e.pending
|
||||
e.subs = nil
|
||||
e.pending = make(map[string]chan contracts.RPCResponse)
|
||||
e.started = false
|
||||
e.mu.Unlock()
|
||||
|
||||
var firstErr error
|
||||
for _, sub := range subs {
|
||||
if sub == nil {
|
||||
continue
|
||||
}
|
||||
if err := sub.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
// 停机时唤醒所有等待响应的调用方,避免业务 goroutine 卡到超时才退出。
|
||||
for requestID, ch := range pending {
|
||||
select {
|
||||
case ch <- contracts.RPCResponse{
|
||||
RequestID: requestID,
|
||||
TargetNodeID: e.nodeID,
|
||||
Error: ErrNotStarted.Error(),
|
||||
SentAt: time.Now().UTC(),
|
||||
}:
|
||||
default:
|
||||
}
|
||||
close(ch)
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// CallNode 向指定节点发起一次 RPC 调用并等待响应。
|
||||
func (e *Endpoint) CallNode(ctx context.Context, targetNodeID, route string, payload []byte, metadata map[string]string) (contracts.RPCResponse, error) {
|
||||
if e == nil {
|
||||
return contracts.RPCResponse{}, ErrInvalidEndpoint
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
ctx, cancel := e.withCallTimeout(ctx)
|
||||
defer cancel()
|
||||
targetNodeID = strings.TrimSpace(targetNodeID)
|
||||
route = strings.TrimSpace(route)
|
||||
var callErr error
|
||||
defer func() {
|
||||
e.recordCallMetric(route, callErr)
|
||||
}()
|
||||
if err := ctx.Err(); err != nil {
|
||||
callErr = err
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
if targetNodeID == "" {
|
||||
callErr = ErrInvalidTarget
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
if route == "" {
|
||||
callErr = ErrInvalidRoute
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
if err := e.checkPayload(payload); err != nil {
|
||||
callErr = err
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
|
||||
ctx, finish := e.startTrace(ctx, "rpc.call_node", map[string]string{
|
||||
"target_node_id": targetNodeID,
|
||||
"route": route,
|
||||
})
|
||||
defer func() { finish(callErr) }()
|
||||
|
||||
requestID := e.nextRequestID()
|
||||
responseCh := make(chan contracts.RPCResponse, 1)
|
||||
requestMetadata := tracing.InjectCarrier(ctx, cloneMetadata(metadata))
|
||||
request := contracts.RPCRequest{
|
||||
RequestID: requestID,
|
||||
SourceService: e.serviceName,
|
||||
SourceNodeID: e.nodeID,
|
||||
TargetNodeID: targetNodeID,
|
||||
Route: route,
|
||||
Metadata: requestMetadata,
|
||||
Payload: append([]byte(nil), payload...),
|
||||
SentAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
e.ensureReadyLocked()
|
||||
if !e.started {
|
||||
e.mu.Unlock()
|
||||
callErr = ErrNotStarted
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
if e.options.MaxPending > 0 && len(e.pending) >= e.options.MaxPending {
|
||||
e.mu.Unlock()
|
||||
callErr = ErrTooManyPending
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
// pending 先登记再发布请求,保证极快返回的响应不会先到而找不到等待者。
|
||||
e.pending[requestID] = responseCh
|
||||
e.mu.Unlock()
|
||||
|
||||
if err := e.transport.PublishRequest(ctx, targetNodeID, request); err != nil {
|
||||
e.removePending(requestID)
|
||||
callErr = err
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
e.removePending(requestID)
|
||||
callErr = ctx.Err()
|
||||
return contracts.RPCResponse{}, callErr
|
||||
case resp, ok := <-responseCh:
|
||||
if !ok {
|
||||
callErr = ErrNotStarted
|
||||
return contracts.RPCResponse{}, callErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
callErr = RemoteError{
|
||||
RequestID: resp.RequestID,
|
||||
TargetNodeID: resp.SourceNodeID,
|
||||
Route: route,
|
||||
Message: resp.Error,
|
||||
}
|
||||
return resp, callErr
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
|
||||
// CallService 先解析服务目标节点,再发起一次 RPC 调用。
|
||||
func (e *Endpoint) CallService(ctx context.Context, service, key, route string, payload []byte, metadata map[string]string) (contracts.RPCResponse, error) {
|
||||
if e == nil {
|
||||
return contracts.RPCResponse{}, ErrInvalidEndpoint
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
service = strings.TrimSpace(service)
|
||||
key = strings.TrimSpace(key)
|
||||
route = strings.TrimSpace(route)
|
||||
if service == "" {
|
||||
err := ErrInvalidService
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
if route == "" {
|
||||
err := ErrInvalidRoute
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
discoveryResolver := e.discovery
|
||||
resolver := e.resolver
|
||||
e.mu.RUnlock()
|
||||
if discoveryResolver != nil {
|
||||
// 优先使用带健康反馈的 discovery resolver;失败会回写节点健康,影响后续选路。
|
||||
node, ok, err := discoveryResolver.ResolveRoute(ctx, service, route, key)
|
||||
if err != nil {
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
targetNodeID := strings.TrimSpace(node.NodeID)
|
||||
if !ok || targetNodeID == "" {
|
||||
err := ErrNoTargetNodes
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
resp, err := e.CallNode(ctx, targetNodeID, route, payload, metadata)
|
||||
if err != nil {
|
||||
if isDiscoveryFail(err) {
|
||||
discoveryResolver.MarkFailure(targetNodeID)
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
discoveryResolver.MarkSuccess(targetNodeID)
|
||||
return resp, nil
|
||||
}
|
||||
if resolver == nil {
|
||||
err := ErrNoTargetNodes
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
targetNodeID, ok, err := resolver(ctx, service, key)
|
||||
if err != nil {
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
if !ok || strings.TrimSpace(targetNodeID) == "" {
|
||||
err := ErrNoTargetNodes
|
||||
e.recordCallMetric(route, err)
|
||||
return contracts.RPCResponse{}, err
|
||||
}
|
||||
return e.CallNode(ctx, targetNodeID, route, payload, metadata)
|
||||
}
|
||||
|
||||
// CallServiceReliable 会重试临时 timeout/pending 失败,但底层仍通过配置的 Bus 投递。
|
||||
// 在当前 memory、Redis Pub/Sub 和 NATS core 适配器下,这不是持久化 RPC:
|
||||
// 请求不会落盘,响应也没有消费者确认。
|
||||
func (e *Endpoint) CallServiceReliable(ctx context.Context, service, key, route string, payload []byte, metadata map[string]string, options RetryOptions) (contracts.RPCResponse, error) {
|
||||
if e == nil {
|
||||
return contracts.RPCResponse{}, ErrInvalidEndpoint
|
||||
}
|
||||
options = normRetryOpts(options)
|
||||
var lastResp contracts.RPCResponse
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= options.Attempts; attempt++ {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lastResp, err
|
||||
}
|
||||
attemptMetadata := retryMetadata(metadata, options, attempt)
|
||||
resp, err := e.CallService(ctx, service, key, route, payload, attemptMetadata)
|
||||
lastResp, lastErr = resp, err
|
||||
if err == nil || !shouldRetryCall(err, options) || attempt == options.Attempts {
|
||||
return resp, err
|
||||
}
|
||||
if err := sleepRetryBackoff(ctx, options, attempt); err != nil {
|
||||
return lastResp, err
|
||||
}
|
||||
}
|
||||
return lastResp, lastErr
|
||||
}
|
||||
|
||||
// CallNodeReliable 会重试临时 timeout/pending 失败;只有底层 Bus 适配器具备持久化能力时,
|
||||
// 它才代表持久投递保证。
|
||||
func (e *Endpoint) CallNodeReliable(ctx context.Context, targetNodeID, route string, payload []byte, metadata map[string]string, options RetryOptions) (contracts.RPCResponse, error) {
|
||||
if e == nil {
|
||||
return contracts.RPCResponse{}, ErrInvalidEndpoint
|
||||
}
|
||||
options = normRetryOpts(options)
|
||||
var lastResp contracts.RPCResponse
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= options.Attempts; attempt++ {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return lastResp, err
|
||||
}
|
||||
attemptMetadata := retryMetadata(metadata, options, attempt)
|
||||
resp, err := e.CallNode(ctx, targetNodeID, route, payload, attemptMetadata)
|
||||
lastResp, lastErr = resp, err
|
||||
if err == nil || !shouldRetryCall(err, options) || attempt == options.Attempts {
|
||||
return resp, err
|
||||
}
|
||||
if err := sleepRetryBackoff(ctx, options, attempt); err != nil {
|
||||
return lastResp, err
|
||||
}
|
||||
}
|
||||
return lastResp, lastErr
|
||||
}
|
||||
|
||||
func (e *Endpoint) handleReqEnvelope(ctx context.Context, env bus.Envelope) error {
|
||||
req, ok := rpcReqPayload(env.Payload)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return e.handleRequest(ctx, req)
|
||||
}
|
||||
|
||||
func (e *Endpoint) handleRequest(ctx context.Context, req contracts.RPCRequest) error {
|
||||
if req.TargetNodeID != "" && req.TargetNodeID != e.nodeID {
|
||||
return nil
|
||||
}
|
||||
route := strings.TrimSpace(req.Route)
|
||||
var handlerErr error
|
||||
defer func() {
|
||||
e.recordHandlerMetric(route, handlerErr)
|
||||
}()
|
||||
if strings.TrimSpace(req.RequestID) == "" {
|
||||
handlerErr = ErrInvalidRequestID
|
||||
return handlerErr
|
||||
}
|
||||
if strings.TrimSpace(req.SourceNodeID) == "" {
|
||||
handlerErr = ErrInvalidNodeID
|
||||
return handlerErr
|
||||
}
|
||||
|
||||
ctx = tracing.ExtractCarrier(ctx, req.Metadata)
|
||||
ctx, finish := e.startTrace(ctx, "rpc.handle", map[string]string{
|
||||
"source_node_id": req.SourceNodeID,
|
||||
"route": req.Route,
|
||||
})
|
||||
defer func() { finish(handlerErr) }()
|
||||
|
||||
e.mu.RLock()
|
||||
handler := e.handlers[route]
|
||||
e.mu.RUnlock()
|
||||
|
||||
var resp contracts.RPCResponse
|
||||
var err error
|
||||
// 请求和响应都做 payload 上限校验,防止 RPC 被当成大对象传输通道。
|
||||
if payloadErr := e.checkPayload(req.Payload); payloadErr != nil {
|
||||
err = payloadErr
|
||||
} else if handler == nil {
|
||||
err = fmt.Errorf("no rpc handler registered for route %q", req.Route)
|
||||
} else {
|
||||
resp, err = handler(ctx, req)
|
||||
}
|
||||
handlerErr = err
|
||||
if err != nil && resp.Error == "" {
|
||||
resp.Error = err.Error()
|
||||
}
|
||||
if payloadErr := e.checkPayload(resp.Payload); payloadErr != nil {
|
||||
handlerErr = payloadErr
|
||||
resp = contracts.RPCResponse{Error: payloadErr.Error()}
|
||||
}
|
||||
resp.RequestID = req.RequestID
|
||||
resp.SourceNodeID = e.nodeID
|
||||
resp.TargetNodeID = req.SourceNodeID
|
||||
if resp.Metadata == nil {
|
||||
resp.Metadata = cloneMetadata(req.Metadata)
|
||||
} else {
|
||||
resp.Metadata = cloneMetadata(resp.Metadata)
|
||||
}
|
||||
resp.Metadata = tracing.InjectCarrier(ctx, resp.Metadata)
|
||||
resp.SentAt = time.Now().UTC()
|
||||
|
||||
topic := contracts.RPCResponseTopic(req.SourceNodeID)
|
||||
if topic == "" {
|
||||
return ErrInvalidNodeID
|
||||
}
|
||||
if publishErr := e.transport.PublishResponse(ctx, req.SourceNodeID, resp); publishErr != nil {
|
||||
handlerErr = publishErr
|
||||
return publishErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Endpoint) handleResponse(_ context.Context, resp contracts.RPCResponse) error {
|
||||
if resp.TargetNodeID != "" && resp.TargetNodeID != e.nodeID {
|
||||
return nil
|
||||
}
|
||||
if payloadErr := e.checkPayload(resp.Payload); payloadErr != nil {
|
||||
resp.Payload = nil
|
||||
if resp.Error == "" {
|
||||
resp.Error = payloadErr.Error()
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
ch := e.pending[resp.RequestID]
|
||||
delete(e.pending, resp.RequestID)
|
||||
e.mu.Unlock()
|
||||
// 找不到 pending 通常说明调用方已经超时或节点正在停机;响应直接丢弃。
|
||||
if ch == nil {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case ch <- resp:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Endpoint) removePending(requestID string) {
|
||||
e.mu.Lock()
|
||||
e.ensureReadyLocked()
|
||||
delete(e.pending, requestID)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Endpoint) ensureReadyLocked() {
|
||||
if e.handlers == nil {
|
||||
e.handlers = make(map[string]Handler)
|
||||
}
|
||||
if e.pending == nil {
|
||||
e.pending = make(map[string]chan contracts.RPCResponse)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Endpoint) withCallTimeout(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return ctx, func() {}
|
||||
}
|
||||
e.mu.RLock()
|
||||
timeout := e.options.CallTimeout
|
||||
e.mu.RUnlock()
|
||||
if timeout <= 0 {
|
||||
return ctx, func() {}
|
||||
}
|
||||
return context.WithTimeout(ctx, timeout)
|
||||
}
|
||||
|
||||
func (e *Endpoint) checkPayload(payload []byte) error {
|
||||
e.mu.RLock()
|
||||
limit := e.options.MaxPayloadBytes
|
||||
e.mu.RUnlock()
|
||||
if limit > 0 && len(payload) > limit {
|
||||
return ErrPayloadTooLarge
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Endpoint) nextRequestID() string {
|
||||
seq := atomic.AddUint64(&e.seq, 1)
|
||||
nodeID := strings.TrimSpace(e.nodeID)
|
||||
if nodeID == "" {
|
||||
nodeID = "unknown"
|
||||
}
|
||||
return fmt.Sprintf("%s-%d-%d", nodeID, time.Now().UTC().UnixNano(), seq)
|
||||
}
|
||||
|
||||
func (e *Endpoint) startTrace(ctx context.Context, name string, attrs map[string]string) (context.Context, func(error)) {
|
||||
e.mu.RLock()
|
||||
tracer := e.tracer
|
||||
e.mu.RUnlock()
|
||||
if tracer == nil {
|
||||
return ctx, func(error) {}
|
||||
}
|
||||
return tracer.Start(ctx, name, attrs)
|
||||
}
|
||||
|
||||
// RegistryResolver 把服务注册表包装成 endpoint 可用的节点解析函数。
|
||||
func RegistryResolver(reg registry.Registry, rejectMaintaining bool) Resolver {
|
||||
resolver := discovery.NewResolver(reg, time.Second, 5*time.Second, rejectMaintaining)
|
||||
return func(ctx context.Context, service, key string) (string, bool, error) {
|
||||
if reg == nil {
|
||||
return "", false, ErrNoTargetNodes
|
||||
}
|
||||
service = strings.TrimSpace(service)
|
||||
if service == "" {
|
||||
return "", false, ErrInvalidService
|
||||
}
|
||||
node, ok, err := resolver.Resolve(ctx, service, key)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
if !ok {
|
||||
return "", false, nil
|
||||
}
|
||||
return node.NodeID, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func isDiscoveryFail(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) ||
|
||||
errors.Is(err, ErrTooManyPending) ||
|
||||
errors.Is(err, ErrPayloadTooLarge) ||
|
||||
errors.Is(err, ErrNotStarted) {
|
||||
return false
|
||||
}
|
||||
var remoteErr RemoteError
|
||||
return !errors.As(err, &remoteErr)
|
||||
}
|
||||
|
||||
func normRetryOpts(options RetryOptions) RetryOptions {
|
||||
if options.Attempts <= 0 {
|
||||
options.Attempts = 3
|
||||
}
|
||||
if options.Backoff < 0 {
|
||||
options.Backoff = 0
|
||||
}
|
||||
if options.MaxBackoff < 0 {
|
||||
options.MaxBackoff = 0
|
||||
}
|
||||
options.IdempotencyKey = strings.TrimSpace(options.IdempotencyKey)
|
||||
return options
|
||||
}
|
||||
|
||||
func retryMetadata(metadata map[string]string, options RetryOptions, attempt int) map[string]string {
|
||||
out := cloneMetadata(metadata)
|
||||
if out == nil {
|
||||
out = make(map[string]string, 4)
|
||||
}
|
||||
out["rpc_attempt"] = fmt.Sprint(attempt)
|
||||
out["rpc_max_attempts"] = fmt.Sprint(options.Attempts)
|
||||
if options.IdempotencyKey != "" {
|
||||
out["rpc_idempotency_key"] = options.IdempotencyKey
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func shouldRetryCall(err error, options RetryOptions) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) ||
|
||||
errors.Is(err, ErrInvalidNodeID) ||
|
||||
errors.Is(err, ErrInvalidTarget) ||
|
||||
errors.Is(err, ErrInvalidService) ||
|
||||
errors.Is(err, ErrInvalidRoute) ||
|
||||
errors.Is(err, ErrInvalidRequestID) ||
|
||||
errors.Is(err, ErrNoTargetNodes) ||
|
||||
errors.Is(err, ErrTooManyPending) ||
|
||||
errors.Is(err, ErrPayloadTooLarge) ||
|
||||
errors.Is(err, ErrNotStarted) ||
|
||||
errors.Is(err, discovery.ErrNoRegistry) ||
|
||||
errors.Is(err, discovery.ErrNoRoute) ||
|
||||
errors.Is(err, discovery.ErrRouteRequiresKey) {
|
||||
return false
|
||||
}
|
||||
var remoteErr RemoteError
|
||||
if errors.As(err, &remoteErr) {
|
||||
return options.RetryRemoteErrors
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func sleepRetryBackoff(ctx context.Context, options RetryOptions, attempt int) error {
|
||||
backoff := retryBackoff(options, attempt)
|
||||
if backoff <= 0 {
|
||||
return nil
|
||||
}
|
||||
timer := time.NewTimer(backoff)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func retryBackoff(options RetryOptions, attempt int) time.Duration {
|
||||
backoff := options.Backoff
|
||||
if backoff <= 0 {
|
||||
return 0
|
||||
}
|
||||
for i := 1; i < attempt; i++ {
|
||||
backoff *= 2
|
||||
if options.MaxBackoff > 0 && backoff > options.MaxBackoff {
|
||||
return options.MaxBackoff
|
||||
}
|
||||
}
|
||||
if options.MaxBackoff > 0 && backoff > options.MaxBackoff {
|
||||
return options.MaxBackoff
|
||||
}
|
||||
return backoff
|
||||
}
|
||||
|
||||
func normalizeOptions(options Options) Options {
|
||||
if options.CallTimeout < 0 {
|
||||
options.CallTimeout = 0
|
||||
}
|
||||
if options.MaxPending < 0 {
|
||||
options.MaxPending = 0
|
||||
}
|
||||
if options.MaxPayloadBytes < 0 {
|
||||
options.MaxPayloadBytes = 0
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]string) map[string]string {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rpcReqPayload(payload any) (contracts.RPCRequest, bool) {
|
||||
switch typed := payload.(type) {
|
||||
case contracts.RPCRequest:
|
||||
return typed, true
|
||||
case *contracts.RPCRequest:
|
||||
if typed == nil {
|
||||
return contracts.RPCRequest{}, false
|
||||
}
|
||||
return *typed, true
|
||||
default:
|
||||
return contracts.RPCRequest{}, false
|
||||
}
|
||||
}
|
||||
|
||||
func rpcRespPayload(payload any) (contracts.RPCResponse, bool) {
|
||||
switch typed := payload.(type) {
|
||||
case contracts.RPCResponse:
|
||||
return typed, true
|
||||
case *contracts.RPCResponse:
|
||||
if typed == nil {
|
||||
return contracts.RPCResponse{}, false
|
||||
}
|
||||
return *typed, true
|
||||
default:
|
||||
return contracts.RPCResponse{}, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"longheng.io/server/internal/platform/runtimeguard"
|
||||
"longheng.io/server/pkg/contracts"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// ErrTransportClosed 表示 gRPC transport 已关闭或未正确装配。
|
||||
var ErrTransportClosed = errors.New("rpc transport is closed")
|
||||
|
||||
// ErrTransportCredReq 用于 errors.Is 判断生产强制 TLS/mTLS 时仍缺少 gRPC 传输凭据的配置错误。
|
||||
var ErrTransportCredReq = errors.New("rpc transport credentials are required")
|
||||
|
||||
// GRPCTransportOptions 是 gRPC transport 的监听、节点表和安全凭据配置。
|
||||
type GRPCTransportOptions struct {
|
||||
ListenAddress string
|
||||
Environment string
|
||||
Peers map[string]string
|
||||
ServerCredentials credentials.TransportCredentials
|
||||
DialCredentials credentials.TransportCredentials
|
||||
RequireTransportSecurity bool
|
||||
ServerOptions []grpc.ServerOption
|
||||
DialOptions []grpc.DialOption
|
||||
}
|
||||
|
||||
// GRPCTransport 使用 gRPC 在不同节点之间传递 RPC 请求和响应。
|
||||
type GRPCTransport struct {
|
||||
mu sync.RWMutex
|
||||
listener net.Listener
|
||||
server *grpc.Server
|
||||
listenAddress string
|
||||
peers map[string]string
|
||||
requestHandlers map[string]RequestHandler
|
||||
responseHandlers map[string]ResponseHandler
|
||||
conns map[string]*grpc.ClientConn
|
||||
dialCredentials credentials.TransportCredentials
|
||||
dialOptions []grpc.DialOption
|
||||
closed bool
|
||||
}
|
||||
|
||||
// NewGRPCTransport 创建并启动一个 gRPC transport。
|
||||
func NewGRPCTransport(options GRPCTransportOptions) (*GRPCTransport, error) {
|
||||
address := strings.TrimSpace(options.ListenAddress)
|
||||
if address == "" {
|
||||
address = "127.0.0.1:0"
|
||||
}
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
needSecureTransport := options.RequireTransportSecurity || runtimeguard.IsProductionEnvironment(options.Environment)
|
||||
if needSecureTransport && (options.ServerCredentials == nil || options.DialCredentials == nil) {
|
||||
_ = listener.Close()
|
||||
return nil, ErrTransportCredReq
|
||||
}
|
||||
serverOptions := []grpc.ServerOption{grpc.ForceServerCodec(rpcJSONCodec{})}
|
||||
if options.ServerCredentials != nil {
|
||||
serverOptions = append(serverOptions, grpc.Creds(options.ServerCredentials.Clone()))
|
||||
}
|
||||
serverOptions = append(serverOptions, options.ServerOptions...)
|
||||
transport := &GRPCTransport{
|
||||
listener: listener,
|
||||
server: grpc.NewServer(serverOptions...),
|
||||
listenAddress: listener.Addr().String(),
|
||||
peers: cloneStringMap(options.Peers),
|
||||
requestHandlers: make(map[string]RequestHandler),
|
||||
responseHandlers: make(map[string]ResponseHandler),
|
||||
conns: make(map[string]*grpc.ClientConn),
|
||||
dialCredentials: cloneTransportCreds(options.DialCredentials),
|
||||
dialOptions: append([]grpc.DialOption(nil), options.DialOptions...),
|
||||
}
|
||||
regGRPCWire(transport.server, transport)
|
||||
go func() {
|
||||
_ = transport.server.Serve(listener)
|
||||
}()
|
||||
return transport, nil
|
||||
}
|
||||
|
||||
// ListenAddress 返回实际监听地址,端口为 0 时可用于发现系统分配端口。
|
||||
func (t *GRPCTransport) ListenAddress() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
t.mu.RLock()
|
||||
defer t.mu.RUnlock()
|
||||
return t.listenAddress
|
||||
}
|
||||
|
||||
// SetPeer 设置或删除目标节点到 gRPC 地址的映射。
|
||||
func (t *GRPCTransport) SetPeer(nodeID, address string) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
nodeID = strings.TrimSpace(nodeID)
|
||||
address = strings.TrimSpace(address)
|
||||
if nodeID == "" {
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
if t.peers == nil {
|
||||
t.peers = make(map[string]string)
|
||||
}
|
||||
if address == "" {
|
||||
delete(t.peers, nodeID)
|
||||
} else {
|
||||
t.peers[nodeID] = address
|
||||
}
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
// SubscribeRequests 注册当前节点的请求处理函数。
|
||||
func (t *GRPCTransport) SubscribeRequests(_ context.Context, nodeID string, handler RequestHandler) (TransportSubscription, error) {
|
||||
nodeID = strings.TrimSpace(nodeID)
|
||||
if nodeID == "" {
|
||||
return nil, ErrInvalidNodeID
|
||||
}
|
||||
if handler == nil {
|
||||
return nil, ErrInvalidRoute
|
||||
}
|
||||
if t == nil {
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.closed {
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
t.requestHandlers[nodeID] = handler
|
||||
return transportSub(func() {
|
||||
t.mu.Lock()
|
||||
delete(t.requestHandlers, nodeID)
|
||||
t.mu.Unlock()
|
||||
}), nil
|
||||
}
|
||||
|
||||
// SubscribeResponses 注册当前节点的响应处理函数。
|
||||
func (t *GRPCTransport) SubscribeResponses(_ context.Context, nodeID string, handler ResponseHandler) (TransportSubscription, error) {
|
||||
nodeID = strings.TrimSpace(nodeID)
|
||||
if nodeID == "" {
|
||||
return nil, ErrInvalidNodeID
|
||||
}
|
||||
if handler == nil {
|
||||
return nil, ErrInvalidRoute
|
||||
}
|
||||
if t == nil {
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.closed {
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
t.responseHandlers[nodeID] = handler
|
||||
return transportSub(func() {
|
||||
t.mu.Lock()
|
||||
delete(t.responseHandlers, nodeID)
|
||||
t.mu.Unlock()
|
||||
}), nil
|
||||
}
|
||||
|
||||
// PublishRequest 通过 gRPC 把请求发送到目标节点。
|
||||
func (t *GRPCTransport) PublishRequest(ctx context.Context, targetNodeID string, request contracts.RPCRequest) error {
|
||||
request.TargetNodeID = strings.TrimSpace(targetNodeID)
|
||||
return t.invoke(ctx, request.TargetNodeID, "/longheng.rpc.Transport/Request", &request, &grpcAck{})
|
||||
}
|
||||
|
||||
// PublishResponse 通过 gRPC 把响应发送到目标节点。
|
||||
func (t *GRPCTransport) PublishResponse(ctx context.Context, targetNodeID string, response contracts.RPCResponse) error {
|
||||
response.TargetNodeID = strings.TrimSpace(targetNodeID)
|
||||
return t.invoke(ctx, response.TargetNodeID, "/longheng.rpc.Transport/Response", &response, &grpcAck{})
|
||||
}
|
||||
|
||||
// Close 停止 gRPC server 并关闭所有已缓存连接。
|
||||
func (t *GRPCTransport) Close() error {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
if t.closed {
|
||||
t.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
t.closed = true
|
||||
server := t.server
|
||||
conns := t.conns
|
||||
t.conns = nil
|
||||
t.requestHandlers = nil
|
||||
t.responseHandlers = nil
|
||||
t.mu.Unlock()
|
||||
if server != nil {
|
||||
server.Stop()
|
||||
}
|
||||
var firstErr error
|
||||
for _, conn := range conns {
|
||||
if conn == nil {
|
||||
continue
|
||||
}
|
||||
if err := conn.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (t *GRPCTransport) request(ctx context.Context, request *contracts.RPCRequest) (*grpcAck, error) {
|
||||
if request == nil {
|
||||
return nil, ErrInvalidRequestID
|
||||
}
|
||||
t.mu.RLock()
|
||||
handler := t.requestHandlers[strings.TrimSpace(request.TargetNodeID)]
|
||||
t.mu.RUnlock()
|
||||
if handler == nil {
|
||||
return nil, ErrNoTargetNodes
|
||||
}
|
||||
return &grpcAck{}, handler(ctx, *request)
|
||||
}
|
||||
|
||||
func (t *GRPCTransport) response(ctx context.Context, response *contracts.RPCResponse) (*grpcAck, error) {
|
||||
if response == nil {
|
||||
return nil, ErrInvalidRequestID
|
||||
}
|
||||
t.mu.RLock()
|
||||
handler := t.responseHandlers[strings.TrimSpace(response.TargetNodeID)]
|
||||
t.mu.RUnlock()
|
||||
if handler == nil {
|
||||
return nil, ErrNoTargetNodes
|
||||
}
|
||||
return &grpcAck{}, handler(ctx, *response)
|
||||
}
|
||||
|
||||
func (t *GRPCTransport) invoke(ctx context.Context, nodeID, method string, in any, out any) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if strings.TrimSpace(nodeID) == "" {
|
||||
return ErrInvalidTarget
|
||||
}
|
||||
conn, err := t.clientConn(nodeID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Invoke(ctx, method, in, out, grpc.ForceCodec(rpcJSONCodec{}))
|
||||
}
|
||||
|
||||
func (t *GRPCTransport) clientConn(nodeID string) (*grpc.ClientConn, error) {
|
||||
if t == nil {
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
t.mu.RLock()
|
||||
if t.closed {
|
||||
t.mu.RUnlock()
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
address := strings.TrimSpace(t.peers[strings.TrimSpace(nodeID)])
|
||||
if address == "" {
|
||||
t.mu.RUnlock()
|
||||
return nil, ErrNoTargetNodes
|
||||
}
|
||||
if conn := t.conns[address]; conn != nil {
|
||||
t.mu.RUnlock()
|
||||
return conn, nil
|
||||
}
|
||||
t.mu.RUnlock()
|
||||
|
||||
options := []grpc.DialOption{
|
||||
grpc.WithDefaultCallOptions(grpc.ForceCodec(rpcJSONCodec{})),
|
||||
}
|
||||
if t.dialCredentials != nil {
|
||||
options = append(options, grpc.WithTransportCredentials(t.dialCredentials.Clone()))
|
||||
} else {
|
||||
options = append(options, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
}
|
||||
t.mu.RLock()
|
||||
options = append(options, t.dialOptions...)
|
||||
t.mu.RUnlock()
|
||||
conn, err := grpc.NewClient(grpcTarget(address), options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.mu.Lock()
|
||||
if t.closed {
|
||||
t.mu.Unlock()
|
||||
_ = conn.Close()
|
||||
return nil, ErrTransportClosed
|
||||
}
|
||||
if existing := t.conns[address]; existing != nil {
|
||||
t.mu.Unlock()
|
||||
_ = conn.Close()
|
||||
return existing, nil
|
||||
}
|
||||
t.conns[address] = conn
|
||||
t.mu.Unlock()
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func cloneTransportCreds(creds credentials.TransportCredentials) credentials.TransportCredentials {
|
||||
if creds == nil {
|
||||
return nil
|
||||
}
|
||||
return creds.Clone()
|
||||
}
|
||||
|
||||
type transportSub func()
|
||||
|
||||
// Close 注销 transport 订阅。
|
||||
func (s transportSub) Close() error {
|
||||
if s != nil {
|
||||
s()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type rpcJSONCodec struct{}
|
||||
|
||||
// Name 返回 gRPC codec 名称。
|
||||
func (rpcJSONCodec) Name() string {
|
||||
return "json"
|
||||
}
|
||||
|
||||
// Marshal 把 RPC 结构编码为 JSON 字节。
|
||||
func (rpcJSONCodec) Marshal(v any) ([]byte, error) {
|
||||
return json.Marshal(v)
|
||||
}
|
||||
|
||||
// Unmarshal 把 JSON 字节解码为 RPC 结构。
|
||||
func (rpcJSONCodec) Unmarshal(data []byte, v any) error {
|
||||
return json.Unmarshal(data, v)
|
||||
}
|
||||
|
||||
type grpcAck struct{}
|
||||
|
||||
type grpcTransportService interface {
|
||||
request(context.Context, *contracts.RPCRequest) (*grpcAck, error)
|
||||
response(context.Context, *contracts.RPCResponse) (*grpcAck, error)
|
||||
}
|
||||
|
||||
var grpcServiceDesc = grpc.ServiceDesc{
|
||||
ServiceName: "longheng.rpc.Transport",
|
||||
HandlerType: (*grpcTransportService)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{MethodName: "Request", Handler: grpcRequestHandler},
|
||||
{MethodName: "Response", Handler: grpcResponseHandler},
|
||||
},
|
||||
}
|
||||
|
||||
func grpcRequestHandler(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) {
|
||||
in := new(contracts.RPCRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(grpcTransportService).request(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{Server: srv, FullMethod: "/longheng.rpc.Transport/Request"}
|
||||
handler := func(ctx context.Context, req any) (any, error) {
|
||||
return srv.(grpcTransportService).request(ctx, req.(*contracts.RPCRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func grpcResponseHandler(srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor) (any, error) {
|
||||
in := new(contracts.RPCResponse)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(grpcTransportService).response(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{Server: srv, FullMethod: "/longheng.rpc.Transport/Response"}
|
||||
handler := func(ctx context.Context, req any) (any, error) {
|
||||
return srv.(grpcTransportService).response(ctx, req.(*contracts.RPCResponse))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func regGRPCWire(server *grpc.Server, transport *GRPCTransport) {
|
||||
server.RegisterService(&grpcServiceDesc, transport)
|
||||
}
|
||||
|
||||
func grpcTarget(address string) string {
|
||||
address = strings.TrimSpace(address)
|
||||
if strings.Contains(address, "://") {
|
||||
return address
|
||||
}
|
||||
return "passthrough:///" + address
|
||||
}
|
||||
|
||||
func cloneStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.TrimSpace(value)
|
||||
if key != "" && value != "" {
|
||||
out[key] = value
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"longheng.io/server/internal/platform/metrics"
|
||||
)
|
||||
|
||||
const (
|
||||
// MetricRPCPending 记录当前等待响应的 RPC 调用数量。
|
||||
MetricRPCPending = "rpc_pending"
|
||||
// MetricRPCHandlers 记录当前已注册的 RPC handler 数量。
|
||||
MetricRPCHandlers = "rpc_handlers_count"
|
||||
// MetricRPCStarted 记录 endpoint 是否处于启动状态。
|
||||
MetricRPCStarted = "rpc_started"
|
||||
// MetricRPCCallsTotal 记录 RPC 客户端调用总数。
|
||||
MetricRPCCallsTotal = "rpc_calls_total"
|
||||
// MetricRPCHandlerCallsTotal 记录 RPC 服务端 handler 调用总数。
|
||||
MetricRPCHandlerCallsTotal = "rpc_handler_calls_total"
|
||||
)
|
||||
|
||||
// MetricsObserver 沿用 presence 的模式:bus / presence / rpc 都只暴露一个
|
||||
// ObserveMetrics 方法,由平台 wiring 注册到 metrics.Registry,供 Prometheus scrape 统一 fanout。
|
||||
type MetricsObserver interface {
|
||||
ObserveMetrics(reg *metrics.Registry, labels map[string]string)
|
||||
}
|
||||
|
||||
// ObserveMetrics 发布 rpc_pending(进行中的客户端调用 gauge)、rpc_handlers_count
|
||||
// (已注册服务端 handler gauge)和 rpc_started(0/1)。
|
||||
// 它只获取 endpoint RLock,可安全在 /metrics scrape goroutine 中调用。
|
||||
func (e *Endpoint) ObserveMetrics(reg *metrics.Registry, labels map[string]string) {
|
||||
if e == nil || reg == nil {
|
||||
return
|
||||
}
|
||||
e.mu.RLock()
|
||||
pending := int64(len(e.pending))
|
||||
handlers := int64(len(e.handlers))
|
||||
started := int64(0)
|
||||
if e.started {
|
||||
started = 1
|
||||
}
|
||||
e.mu.RUnlock()
|
||||
|
||||
tags := normObserverLabels(labels)
|
||||
reg.SetGauge(MetricRPCPending, tags, pending)
|
||||
reg.SetGauge(MetricRPCHandlers, tags, handlers)
|
||||
reg.SetGauge(MetricRPCStarted, tags, started)
|
||||
}
|
||||
|
||||
// SetMetrics 设置 endpoint 调用指标使用的 registry 和基础标签。
|
||||
func (e *Endpoint) SetMetrics(reg *metrics.Registry, labels map[string]string) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.metrics = reg
|
||||
e.metricLabels = normObserverLabels(labels)
|
||||
e.mu.Unlock()
|
||||
}
|
||||
|
||||
func (e *Endpoint) recordCallMetric(route string, err error) {
|
||||
e.recordRPCMetric(MetricRPCCallsTotal, route, err)
|
||||
}
|
||||
|
||||
func (e *Endpoint) recordHandlerMetric(route string, err error) {
|
||||
e.recordRPCMetric(MetricRPCHandlerCallsTotal, route, err)
|
||||
}
|
||||
|
||||
func (e *Endpoint) recordRPCMetric(name, route string, err error) {
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
e.mu.RLock()
|
||||
reg := e.metrics
|
||||
labels := normObserverLabels(e.metricLabels)
|
||||
e.mu.RUnlock()
|
||||
if reg == nil {
|
||||
return
|
||||
}
|
||||
route = strings.TrimSpace(route)
|
||||
if route == "" {
|
||||
route = "unknown"
|
||||
}
|
||||
labels = withRPCMetricLabel(labels, "route", route)
|
||||
labels = withRPCMetricLabel(labels, "result", rpcMetricResult(err))
|
||||
reg.Inc(name, labels)
|
||||
}
|
||||
|
||||
func rpcMetricResult(err error) string {
|
||||
if err == nil {
|
||||
return "ok"
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return "canceled"
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return "deadline"
|
||||
}
|
||||
if errors.Is(err, ErrInvalidTarget) {
|
||||
return "invalid_target"
|
||||
}
|
||||
if errors.Is(err, ErrInvalidService) {
|
||||
return "invalid_service"
|
||||
}
|
||||
if errors.Is(err, ErrInvalidRoute) {
|
||||
return "invalid_route"
|
||||
}
|
||||
if errors.Is(err, ErrInvalidRequestID) {
|
||||
return "invalid_request"
|
||||
}
|
||||
if errors.Is(err, ErrInvalidNodeID) {
|
||||
return "invalid_node"
|
||||
}
|
||||
if errors.Is(err, ErrNoTargetNodes) {
|
||||
return "no_target"
|
||||
}
|
||||
if errors.Is(err, ErrTooManyPending) {
|
||||
return "too_many_pending"
|
||||
}
|
||||
if errors.Is(err, ErrPayloadTooLarge) {
|
||||
return "payload_too_large"
|
||||
}
|
||||
if errors.Is(err, ErrNotStarted) {
|
||||
return "not_started"
|
||||
}
|
||||
var remoteErr RemoteError
|
||||
if errors.As(err, &remoteErr) {
|
||||
return "remote_error"
|
||||
}
|
||||
if strings.Contains(err.Error(), "no rpc handler registered") {
|
||||
return "missing_handler"
|
||||
}
|
||||
return "error"
|
||||
}
|
||||
|
||||
func withRPCMetricLabel(labels map[string]string, key, value string) map[string]string {
|
||||
next := normObserverLabels(labels)
|
||||
if next == nil {
|
||||
next = make(map[string]string, 1)
|
||||
}
|
||||
next[key] = value
|
||||
return next
|
||||
}
|
||||
|
||||
func normObserverLabels(labels map[string]string) map[string]string {
|
||||
if len(labels) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(labels))
|
||||
for key, value := range labels {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"longheng.io/server/internal/platform/bus"
|
||||
"longheng.io/server/pkg/contracts"
|
||||
)
|
||||
|
||||
// RequestHandler 是 transport 收到 RPC 请求后的回调。
|
||||
type RequestHandler func(context.Context, contracts.RPCRequest) error
|
||||
|
||||
// ResponseHandler 是 transport 收到 RPC 响应后的回调。
|
||||
type ResponseHandler func(context.Context, contracts.RPCResponse) error
|
||||
|
||||
// TransportSubscription 表示可关闭的 transport 订阅。
|
||||
type TransportSubscription interface {
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Transport 抽象 RPC 请求和响应的订阅与发布能力。
|
||||
type Transport interface {
|
||||
SubscribeRequests(context.Context, string, RequestHandler) (TransportSubscription, error)
|
||||
SubscribeResponses(context.Context, string, ResponseHandler) (TransportSubscription, error)
|
||||
PublishRequest(context.Context, string, contracts.RPCRequest) error
|
||||
PublishResponse(context.Context, string, contracts.RPCResponse) error
|
||||
}
|
||||
|
||||
// BusTransport 通过框架 bus 投递 RPC 请求和响应。
|
||||
type BusTransport struct {
|
||||
bus bus.Bus
|
||||
}
|
||||
|
||||
// NewBusTransport 创建基于 bus 的 RPC transport。
|
||||
func NewBusTransport(eventBus bus.Bus) *BusTransport {
|
||||
return &BusTransport{bus: eventBus}
|
||||
}
|
||||
|
||||
// SubscribeRequests 订阅指定节点的 RPC 请求主题。
|
||||
func (t *BusTransport) SubscribeRequests(_ context.Context, nodeID string, handler RequestHandler) (TransportSubscription, error) {
|
||||
if t == nil || t.bus == nil {
|
||||
return nil, bus.ErrClosed
|
||||
}
|
||||
return t.bus.Subscribe(contracts.RPCNodeRequestTopic(nodeID), func(ctx context.Context, env bus.Envelope) error {
|
||||
req, ok := rpcReqPayload(env.Payload)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return handler(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
// SubscribeResponses 订阅指定节点的 RPC 响应主题。
|
||||
func (t *BusTransport) SubscribeResponses(_ context.Context, nodeID string, handler ResponseHandler) (TransportSubscription, error) {
|
||||
if t == nil || t.bus == nil {
|
||||
return nil, bus.ErrClosed
|
||||
}
|
||||
return t.bus.Subscribe(contracts.RPCResponseTopic(nodeID), func(ctx context.Context, env bus.Envelope) error {
|
||||
resp, ok := rpcRespPayload(env.Payload)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return handler(ctx, resp)
|
||||
})
|
||||
}
|
||||
|
||||
// PublishRequest 把 RPC 请求发布到目标节点主题。
|
||||
func (t *BusTransport) PublishRequest(ctx context.Context, targetNodeID string, request contracts.RPCRequest) error {
|
||||
if t == nil || t.bus == nil {
|
||||
return bus.ErrClosed
|
||||
}
|
||||
return t.bus.Publish(ctx, contracts.RPCNodeRequestTopic(targetNodeID), request)
|
||||
}
|
||||
|
||||
// PublishResponse 把 RPC 响应发布到目标节点响应主题。
|
||||
func (t *BusTransport) PublishResponse(ctx context.Context, targetNodeID string, response contracts.RPCResponse) error {
|
||||
if t == nil || t.bus == nil {
|
||||
return bus.ErrClosed
|
||||
}
|
||||
return t.bus.Publish(ctx, contracts.RPCResponseTopic(targetNodeID), response)
|
||||
}
|
||||
Reference in new issue
Block a user