DNF90 服务端与启动器完整源码（含 GM 工具功能增强）
服务端（Go）： - GM 塞背包/带属性装备发放：在线角色推送 op14 增量物品行，背包实时刷新，无需回角色选择界面 - GM 改金币/改等级：去掉在线拒绝，落库后推送 op14 钱包/op37 等级状态，在线实时刷新 - 等级端点统一切换到新版 GMSetCharacterLevel（自带在线刷新） - 无限/锁死疲劳模式 + 燃烧疲劳：每房间消耗 2 点、打怪与通关经验双倍 - 燃烧疲劳开关时给所有在线角色推送系统提示（op12 mode 0） - 深渊史诗掉率：GM 全局倍率 + 城镇增益药剂窗口倍率，真实接入掉落结算 启动器（Python/JS）： - 标题栏自绘、窗口 1204x717、禁止拖拽改大小、仅最小化/关闭 - GM 工具：疲劳与燃烧、深渊史诗掉率、拍卖行上架（单一定价对齐） - 拍卖行上架前端对齐服务端单一价格语义，修复 400 报错
This commit is contained in:
hanfuit committed
commit
645b64fe21
4388 files changed
+1405457
No files matched your search
@@ -0,0 +1,347 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type asyncDeadSinkOpts[T any, F comparable] struct {
|
||||
Limit int
|
||||
Store DeadLetterStore[T, F]
|
||||
StoreTimeout time.Duration
|
||||
RecordKey KeyFunc[T]
|
||||
NormalizeKey func(string) string
|
||||
Clone CloneFunc[T]
|
||||
NormalizeFields func([]F) []F
|
||||
}
|
||||
|
||||
type asyncDeadLetterSink[T any, F comparable] struct {
|
||||
mu sync.Mutex
|
||||
storeMu sync.Mutex
|
||||
limit int
|
||||
store DeadLetterStore[T, F]
|
||||
storeTimeout time.Duration
|
||||
storeError string
|
||||
deadLetters map[string]AsyncStoreDeadLetter[T, F]
|
||||
deadOrder []string
|
||||
recordKey KeyFunc[T]
|
||||
normalizeKey func(string) string
|
||||
clone CloneFunc[T]
|
||||
normalizeFields func([]F) []F
|
||||
}
|
||||
|
||||
func newAsyncDeadSink[T any, F comparable](options asyncDeadSinkOpts[T, F]) *asyncDeadLetterSink[T, F] {
|
||||
if options.Clone == nil {
|
||||
options.Clone = IdentityClone[T]
|
||||
}
|
||||
if options.StoreTimeout <= 0 {
|
||||
options.StoreTimeout = defAsyncFlush
|
||||
}
|
||||
return &asyncDeadLetterSink[T, F]{
|
||||
limit: options.Limit,
|
||||
store: options.Store,
|
||||
storeTimeout: options.StoreTimeout,
|
||||
deadLetters: make(map[string]AsyncStoreDeadLetter[T, F]),
|
||||
recordKey: options.RecordKey,
|
||||
normalizeKey: options.NormalizeKey,
|
||||
clone: options.Clone,
|
||||
normalizeFields: options.NormalizeFields,
|
||||
}
|
||||
}
|
||||
|
||||
// Load 从持久化死信存储恢复死信记录。
|
||||
func (s *asyncDeadLetterSink[T, F]) Load() {
|
||||
if s.store == nil {
|
||||
return
|
||||
}
|
||||
s.storeMu.Lock()
|
||||
defer s.storeMu.Unlock()
|
||||
ctx, cancel := s.storeContext()
|
||||
deadLetters, err := s.store.List(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
s.setStoreError(err)
|
||||
return
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
for _, dead := range deadLetters {
|
||||
dead = s.cloneDeadLetter(dead)
|
||||
dead.AccountID = s.NormalizeRecordKey(dead.AccountID)
|
||||
if dead.AccountID == "" {
|
||||
accountID, err := s.RecordKey(dead.Profile)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
dead.AccountID = accountID
|
||||
}
|
||||
if dead.AccountID == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := s.deadLetters[dead.AccountID]; !exists {
|
||||
s.deadOrder = append(s.deadOrder, dead.AccountID)
|
||||
}
|
||||
s.deadLetters[dead.AccountID] = dead
|
||||
}
|
||||
evicted := s.enforceLimitLocked()
|
||||
s.mu.Unlock()
|
||||
s.deletePersistedMany(evicted)
|
||||
}
|
||||
|
||||
// Get 返回指定账号的死信副本。
|
||||
func (s *asyncDeadLetterSink[T, F]) Get(accountID string) (AsyncStoreDeadLetter[T, F], bool) {
|
||||
accountID = s.NormalizeRecordKey(accountID)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
dead, ok := s.deadLetters[accountID]
|
||||
if !ok {
|
||||
return AsyncStoreDeadLetter[T, F]{}, false
|
||||
}
|
||||
return s.cloneDeadLetter(dead), true
|
||||
}
|
||||
|
||||
// Add 写入一条死信并持久化；持久化失败时回滚内存死信，调用方应继续保留 pending。
|
||||
func (s *asyncDeadLetterSink[T, F]) Add(save pendingSave[T, F]) error {
|
||||
accountID, err := s.RecordKey(save.profile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dead := AsyncStoreDeadLetter[T, F]{
|
||||
AccountID: accountID,
|
||||
Profile: s.clone(save.profile),
|
||||
Fields: s.NormalizeFields(save.fields),
|
||||
Attempts: save.attempts,
|
||||
Error: save.lastError,
|
||||
FirstFailedAt: save.firstFailedAt,
|
||||
LastFailedAt: save.lastFailedAt,
|
||||
}
|
||||
|
||||
s.storeMu.Lock()
|
||||
defer s.storeMu.Unlock()
|
||||
s.mu.Lock()
|
||||
previous, existed := s.deadLetters[accountID]
|
||||
if _, exists := s.deadLetters[accountID]; !exists {
|
||||
s.deadOrder = append(s.deadOrder, accountID)
|
||||
}
|
||||
s.deadLetters[accountID] = dead
|
||||
s.mu.Unlock()
|
||||
|
||||
if err := s.persist(dead); err != nil {
|
||||
s.rollbackAdd(accountID, previous, existed)
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
evicted := s.enforceLimitLocked()
|
||||
s.mu.Unlock()
|
||||
s.deletePersistedMany(evicted)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Take 取出并删除指定账号死信。
|
||||
func (s *asyncDeadLetterSink[T, F]) Take(accountID string) (AsyncStoreDeadLetter[T, F], bool) {
|
||||
accountID = s.NormalizeRecordKey(accountID)
|
||||
s.storeMu.Lock()
|
||||
defer s.storeMu.Unlock()
|
||||
s.mu.Lock()
|
||||
|
||||
dead, ok := s.deadLetters[accountID]
|
||||
if !ok {
|
||||
s.mu.Unlock()
|
||||
return AsyncStoreDeadLetter[T, F]{}, false
|
||||
}
|
||||
delete(s.deadLetters, accountID)
|
||||
s.removeOrderLocked(accountID)
|
||||
out := s.cloneDeadLetter(dead)
|
||||
s.mu.Unlock()
|
||||
s.deletePersisted(accountID)
|
||||
return out, true
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) restoreMemory(dead AsyncStoreDeadLetter[T, F]) {
|
||||
dead = s.cloneDeadLetter(dead)
|
||||
dead.AccountID = s.NormalizeRecordKey(dead.AccountID)
|
||||
if dead.AccountID == "" {
|
||||
accountID, err := s.RecordKey(dead.Profile)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
dead.AccountID = accountID
|
||||
}
|
||||
if dead.AccountID == "" {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
if _, exists := s.deadLetters[dead.AccountID]; !exists {
|
||||
s.deadOrder = append(s.deadOrder, dead.AccountID)
|
||||
}
|
||||
s.deadLetters[dead.AccountID] = dead
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// List 返回按失败时间排序的死信快照。
|
||||
func (s *asyncDeadLetterSink[T, F]) List() []AsyncStoreDeadLetter[T, F] {
|
||||
s.mu.Lock()
|
||||
out := make([]AsyncStoreDeadLetter[T, F], 0, len(s.deadLetters))
|
||||
for _, dead := range s.deadLetters {
|
||||
out = append(out, s.cloneDeadLetter(dead))
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].LastFailedAt.Equal(out[j].LastFailedAt) {
|
||||
return out[i].AccountID < out[j].AccountID
|
||||
}
|
||||
return out[i].LastFailedAt.Before(out[j].LastFailedAt)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// Count 返回当前死信数量。
|
||||
func (s *asyncDeadLetterSink[T, F]) Count() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.deadLetters)
|
||||
}
|
||||
|
||||
// Limit 返回死信内存上限。
|
||||
func (s *asyncDeadLetterSink[T, F]) Limit() int {
|
||||
return s.limit
|
||||
}
|
||||
|
||||
// StoreName 返回持久化死信存储类型名。
|
||||
func (s *asyncDeadLetterSink[T, F]) StoreName() string {
|
||||
if s.store == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%T", s.store)
|
||||
}
|
||||
|
||||
// LastError 返回最近一次死信持久化错误。
|
||||
func (s *asyncDeadLetterSink[T, F]) LastError() string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.storeError
|
||||
}
|
||||
|
||||
// CheckStore 检查死信存储可用性。
|
||||
func (s *asyncDeadLetterSink[T, F]) CheckStore(ctx context.Context) error {
|
||||
return Check(ctx, s.store)
|
||||
}
|
||||
|
||||
// RecordKey 计算并归一化死信记录主键。
|
||||
func (s *asyncDeadLetterSink[T, F]) RecordKey(record T) (string, error) {
|
||||
key, err := RecordKey(s.recordKey, record)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return s.NormalizeRecordKey(key), nil
|
||||
}
|
||||
|
||||
// NormalizeRecordKey 归一化死信主键。
|
||||
func (s *asyncDeadLetterSink[T, F]) NormalizeRecordKey(key string) string {
|
||||
if s.normalizeKey == nil {
|
||||
return key
|
||||
}
|
||||
return s.normalizeKey(key)
|
||||
}
|
||||
|
||||
// NormalizeFields 归一化死信字段列表。
|
||||
func (s *asyncDeadLetterSink[T, F]) NormalizeFields(fields []F) []F {
|
||||
if s.normalizeFields == nil {
|
||||
return append([]F(nil), fields...)
|
||||
}
|
||||
return s.normalizeFields(fields)
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) cloneDeadLetter(dead AsyncStoreDeadLetter[T, F]) AsyncStoreDeadLetter[T, F] {
|
||||
dead.Profile = s.clone(dead.Profile)
|
||||
dead.Fields = s.NormalizeFields(dead.Fields)
|
||||
return dead
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) enforceLimitLocked() []string {
|
||||
var evicted []string
|
||||
for s.limit > 0 && len(s.deadLetters) > s.limit && len(s.deadOrder) > 0 {
|
||||
evict := s.deadOrder[0]
|
||||
s.deadOrder = s.deadOrder[1:]
|
||||
delete(s.deadLetters, evict)
|
||||
evicted = append(evicted, evict)
|
||||
}
|
||||
return evicted
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) removeOrderLocked(accountID string) {
|
||||
for i, id := range s.deadOrder {
|
||||
if id == accountID {
|
||||
s.deadOrder = append(s.deadOrder[:i], s.deadOrder[i+1:]...)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) persist(dead AsyncStoreDeadLetter[T, F]) error {
|
||||
if s.store == nil {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := s.storeContext()
|
||||
err := s.store.Save(ctx, s.cloneDeadLetter(dead))
|
||||
cancel()
|
||||
if err != nil {
|
||||
s.setStoreError(err)
|
||||
return err
|
||||
}
|
||||
s.setStoreError(nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) deletePersisted(accountID string) {
|
||||
if s.store == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := s.storeContext()
|
||||
err := s.store.Delete(ctx, accountID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
s.setStoreError(err)
|
||||
return
|
||||
}
|
||||
s.setStoreError(nil)
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) deletePersistedMany(accountIDs []string) {
|
||||
for _, accountID := range accountIDs {
|
||||
s.deletePersisted(accountID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) storeContext() (context.Context, context.CancelFunc) {
|
||||
timeout := defAsyncFlush
|
||||
if s != nil && s.storeTimeout > 0 {
|
||||
timeout = s.storeTimeout
|
||||
}
|
||||
return context.WithTimeout(context.Background(), timeout)
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) rollbackAdd(accountID string, previous AsyncStoreDeadLetter[T, F], existed bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if existed {
|
||||
s.deadLetters[accountID] = previous
|
||||
return
|
||||
}
|
||||
delete(s.deadLetters, accountID)
|
||||
s.removeOrderLocked(accountID)
|
||||
}
|
||||
|
||||
func (s *asyncDeadLetterSink[T, F]) setStoreError(err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err == nil {
|
||||
s.storeError = ""
|
||||
return
|
||||
}
|
||||
s.storeError = err.Error()
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type asyncFlusherOptions[T any, F comparable] struct {
|
||||
Base Store[T]
|
||||
Queue *asyncPendingQueue[T, F]
|
||||
Retry asyncRetryPolicy[T, F]
|
||||
DeadLetters *asyncDeadLetterSink[T, F]
|
||||
Clone CloneFunc[T]
|
||||
NormalizeFields func([]F) []F
|
||||
SaveFields SaveFieldsFunc[T, F]
|
||||
SaveBatch SaveFieldBatchFunc[T, F]
|
||||
AutoExpireTTL time.Duration
|
||||
Expire ExpireFunc[T]
|
||||
}
|
||||
|
||||
type asyncFlusher[T any, F comparable] struct {
|
||||
mu sync.Mutex
|
||||
base Store[T]
|
||||
queue *asyncPendingQueue[T, F]
|
||||
retry asyncRetryPolicy[T, F]
|
||||
deadLetters *asyncDeadLetterSink[T, F]
|
||||
clone CloneFunc[T]
|
||||
normalizeFields func([]F) []F
|
||||
saveFields SaveFieldsFunc[T, F]
|
||||
saveBatch SaveFieldBatchFunc[T, F]
|
||||
autoExpireTTL time.Duration
|
||||
expire ExpireFunc[T]
|
||||
}
|
||||
|
||||
func newAsyncFlusher[T any, F comparable](options asyncFlusherOptions[T, F]) *asyncFlusher[T, F] {
|
||||
if options.Clone == nil {
|
||||
options.Clone = IdentityClone[T]
|
||||
}
|
||||
return &asyncFlusher[T, F]{
|
||||
base: options.Base,
|
||||
queue: options.Queue,
|
||||
retry: options.Retry,
|
||||
deadLetters: options.DeadLetters,
|
||||
clone: options.Clone,
|
||||
normalizeFields: options.NormalizeFields,
|
||||
saveFields: options.SaveFields,
|
||||
saveBatch: options.SaveBatch,
|
||||
autoExpireTTL: options.AutoExpireTTL,
|
||||
expire: options.Expire,
|
||||
}
|
||||
}
|
||||
|
||||
// Flush 刷出 pending 队列，并把失败项转入重试或死信。
|
||||
func (f *asyncFlusher[T, F]) Flush(ctx context.Context, force bool) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
for {
|
||||
batch := f.queue.Take(force, time.Now().UTC())
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Take 会乐观地从 pending 队列移除记录。下面每条失败路径都必须记录失败项
|
||||
// 供重试/死信处理，或把尚未尝试的项放回队列；否则后端抖动会静默丢掉已接受的玩家状态。
|
||||
if f.saveBatch != nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
f.queue.Restore(batch)
|
||||
return err
|
||||
}
|
||||
if err := f.saveBatch(ctx, f.base, fieldSavesPending(batch, f.clone, f.normalizeFields)); err != nil {
|
||||
errs := make([]error, 0, len(batch)+1)
|
||||
errs = append(errs, err)
|
||||
for _, save := range batch {
|
||||
if recordErr := f.queue.RecordFailure(save, f.retry, f.deadLetters, err); recordErr != nil {
|
||||
errs = append(errs, recordErr)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
if err := f.expireBatch(ctx, batch); err != nil {
|
||||
errs := make([]error, 0, len(batch)+1)
|
||||
errs = append(errs, err)
|
||||
for _, save := range batch {
|
||||
if recordErr := f.queue.RecordFailure(save, f.retry, f.deadLetters, err); recordErr != nil {
|
||||
errs = append(errs, recordErr)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for i, save := range batch {
|
||||
if err := ctx.Err(); err != nil {
|
||||
f.queue.Restore(batch[i:])
|
||||
return err
|
||||
}
|
||||
// 底层 SaveFields 可能有编码或归一化副作用，单条刷盘必须使用独立快照。
|
||||
flushRecord := f.clone(save.profile)
|
||||
if err := f.saveFields(ctx, f.base, flushRecord, save.fields...); err != nil {
|
||||
recordErr := f.queue.RecordFailure(save, f.retry, f.deadLetters, err)
|
||||
f.queue.Restore(batch[i+1:])
|
||||
return errors.Join(err, recordErr)
|
||||
}
|
||||
if err := f.expireSave(ctx, save); err != nil {
|
||||
recordErr := f.queue.RecordFailure(save, f.retry, f.deadLetters, err)
|
||||
f.queue.Restore(batch[i+1:])
|
||||
return errors.Join(err, recordErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AutoExpireTTL 返回自动续期 TTL。
|
||||
func (f *asyncFlusher[T, F]) AutoExpireTTL() time.Duration {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
return f.autoExpireTTL
|
||||
}
|
||||
|
||||
func (f *asyncFlusher[T, F]) expireBatch(ctx context.Context, batch []pendingSave[T, F]) error {
|
||||
if f.autoExpireTTL <= 0 || f.expire == nil {
|
||||
return nil
|
||||
}
|
||||
for _, save := range batch {
|
||||
if err := f.expireSave(ctx, save); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *asyncFlusher[T, F]) expireSave(ctx context.Context, save pendingSave[T, F]) error {
|
||||
if f.autoExpireTTL <= 0 || f.expire == nil {
|
||||
return nil
|
||||
}
|
||||
key, err := f.queue.RecordKey(save.profile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return f.expire(ctx, f.base, key, f.autoExpireTTL)
|
||||
}
|
||||
|
||||
func fieldSavesPending[T any, F comparable](batch []pendingSave[T, F], clone CloneFunc[T], normalize func([]F) []F) []FieldSave[T, F] {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]FieldSave[T, F], 0, len(batch))
|
||||
for _, save := range batch {
|
||||
save = save.clone(clone, normalize)
|
||||
out = append(out, FieldSave[T, F]{
|
||||
Record: save.profile,
|
||||
Fields: save.fields,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type pendingSave[T any, F comparable] struct {
|
||||
profile T
|
||||
fields []F
|
||||
attempts int
|
||||
nextAttemptAt time.Time
|
||||
firstFailedAt time.Time
|
||||
lastFailedAt time.Time
|
||||
lastError string
|
||||
sequence uint64
|
||||
}
|
||||
|
||||
type asyncPendingQueue[T any, F comparable] struct {
|
||||
mu sync.Mutex
|
||||
pending map[string]pendingSave[T, F]
|
||||
recordKey KeyFunc[T]
|
||||
normalizeKey func(string) string
|
||||
clone CloneFunc[T]
|
||||
normalizeFields func([]F) []F
|
||||
nextSequence uint64
|
||||
}
|
||||
|
||||
func newAsyncPendingQueue[T any, F comparable](
|
||||
recordKey KeyFunc[T],
|
||||
normalizeKey func(string) string,
|
||||
clone CloneFunc[T],
|
||||
normalizeFields func([]F) []F,
|
||||
) *asyncPendingQueue[T, F] {
|
||||
if clone == nil {
|
||||
clone = IdentityClone[T]
|
||||
}
|
||||
return &asyncPendingQueue[T, F]{
|
||||
pending: make(map[string]pendingSave[T, F]),
|
||||
recordKey: recordKey,
|
||||
normalizeKey: normalizeKey,
|
||||
clone: clone,
|
||||
normalizeFields: normalizeFields,
|
||||
}
|
||||
}
|
||||
|
||||
// Get 返回 pending 队列里的记录副本。
|
||||
func (q *asyncPendingQueue[T, F]) Get(accountID string) (pendingSave[T, F], bool) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
save, ok := q.pending[accountID]
|
||||
if !ok {
|
||||
return pendingSave[T, F]{}, false
|
||||
}
|
||||
return save.clone(q.clone, q.normalizeFields), true
|
||||
}
|
||||
|
||||
// Add 添加或合并 pending 保存请求。
|
||||
func (q *asyncPendingQueue[T, F]) Add(accountID string, save pendingSave[T, F]) int {
|
||||
accountID = q.NormalizeRecordKey(accountID)
|
||||
save = save.clone(q.clone, q.normalizeFields)
|
||||
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if existing, ok := q.pending[accountID]; ok {
|
||||
merged := existing.clone(q.clone, q.normalizeFields)
|
||||
merged.profile = q.clone(save.profile)
|
||||
merged.fields = q.Merge(merged.fields, save.fields)
|
||||
save = merged
|
||||
}
|
||||
if save.sequence == 0 {
|
||||
q.nextSequence++
|
||||
save.sequence = q.nextSequence
|
||||
}
|
||||
q.pending[accountID] = save
|
||||
return len(q.pending)
|
||||
}
|
||||
|
||||
// Requeue 把失败保存重新放回 pending 队列。
|
||||
func (q *asyncPendingQueue[T, F]) Requeue(accountID string, save pendingSave[T, F]) int {
|
||||
accountID = q.NormalizeRecordKey(accountID)
|
||||
save = save.clone(q.clone, q.normalizeFields)
|
||||
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if existing, ok := q.pending[accountID]; ok {
|
||||
merged := existing.clone(q.clone, q.normalizeFields)
|
||||
merged.fields = q.Merge(merged.fields, save.fields)
|
||||
if merged.sequence == 0 {
|
||||
q.nextSequence++
|
||||
merged.sequence = q.nextSequence
|
||||
}
|
||||
q.pending[accountID] = merged
|
||||
return len(q.pending)
|
||||
}
|
||||
if save.sequence == 0 {
|
||||
q.nextSequence++
|
||||
save.sequence = q.nextSequence
|
||||
}
|
||||
q.pending[accountID] = save
|
||||
return len(q.pending)
|
||||
}
|
||||
|
||||
// Take 取出当前到期的 pending 保存请求。
|
||||
func (q *asyncPendingQueue[T, F]) Take(force bool, now time.Time) []pendingSave[T, F] {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
if len(q.pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]keyedPendingSave[T, F], 0, len(q.pending))
|
||||
for accountID, save := range q.pending {
|
||||
if !force && !save.nextAttemptAt.IsZero() && now.Before(save.nextAttemptAt) {
|
||||
continue
|
||||
}
|
||||
out = append(out, keyedPendingSave[T, F]{
|
||||
key: accountID,
|
||||
save: save.clone(q.clone, q.normalizeFields),
|
||||
})
|
||||
delete(q.pending, accountID)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].save.sequence == out[j].save.sequence {
|
||||
return out[i].key < out[j].key
|
||||
}
|
||||
return out[i].save.sequence < out[j].save.sequence
|
||||
})
|
||||
|
||||
saves := make([]pendingSave[T, F], 0, len(out))
|
||||
for _, item := range out {
|
||||
saves = append(saves, item.save)
|
||||
}
|
||||
return saves
|
||||
}
|
||||
|
||||
// Restore 把未尝试的保存请求恢复到 pending 队列。
|
||||
func (q *asyncPendingQueue[T, F]) Restore(saves []pendingSave[T, F]) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
for _, save := range saves {
|
||||
accountID, err := q.RecordKey(save.profile)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if existing, exists := q.pending[accountID]; exists {
|
||||
merged := existing.clone(q.clone, q.normalizeFields)
|
||||
merged.fields = q.Merge(merged.fields, save.fields)
|
||||
q.pending[accountID] = merged
|
||||
continue
|
||||
}
|
||||
if save.sequence == 0 {
|
||||
q.nextSequence++
|
||||
save.sequence = q.nextSequence
|
||||
}
|
||||
q.pending[accountID] = save.clone(q.clone, q.normalizeFields)
|
||||
}
|
||||
}
|
||||
|
||||
// RecordFailure 记录保存失败，并按重试策略放回队列或转入死信。
|
||||
func (q *asyncPendingQueue[T, F]) RecordFailure(save pendingSave[T, F], retry asyncRetryPolicy[T, F], sink *asyncDeadLetterSink[T, F], err error) error {
|
||||
save = retry.RecordFailure(save, err, time.Now().UTC())
|
||||
accountID, keyErr := q.RecordKey(save.profile)
|
||||
if keyErr != nil {
|
||||
return keyErr
|
||||
}
|
||||
|
||||
q.mu.Lock()
|
||||
if existing, exists := q.pending[accountID]; exists {
|
||||
merged := existing.clone(q.clone, q.normalizeFields)
|
||||
merged.fields = q.Merge(merged.fields, save.fields)
|
||||
if save.sequence == 0 || merged.sequence <= save.sequence {
|
||||
merged.attempts = save.attempts
|
||||
merged.firstFailedAt = save.firstFailedAt
|
||||
merged.lastFailedAt = save.lastFailedAt
|
||||
merged.lastError = save.lastError
|
||||
merged.nextAttemptAt = save.nextAttemptAt
|
||||
}
|
||||
if merged.sequence == 0 {
|
||||
q.nextSequence++
|
||||
merged.sequence = q.nextSequence
|
||||
}
|
||||
if retry.Exhausted(merged) {
|
||||
delete(q.pending, accountID)
|
||||
q.mu.Unlock()
|
||||
if err := sink.Add(merged); err != nil {
|
||||
q.Requeue(accountID, merged)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
q.pending[accountID] = merged
|
||||
q.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
if retry.Exhausted(save) {
|
||||
q.mu.Unlock()
|
||||
if err := sink.Add(save); err != nil {
|
||||
q.Requeue(accountID, save)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if save.sequence == 0 {
|
||||
q.nextSequence++
|
||||
save.sequence = q.nextSequence
|
||||
}
|
||||
q.pending[accountID] = save.clone(q.clone, q.normalizeFields)
|
||||
q.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stats 返回 pending 总数和当前可重试数量。
|
||||
func (q *asyncPendingQueue[T, F]) Stats(now time.Time) (int, int) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
due := 0
|
||||
for _, save := range q.pending {
|
||||
if save.nextAttemptAt.IsZero() || !now.Before(save.nextAttemptAt) {
|
||||
due++
|
||||
}
|
||||
}
|
||||
return len(q.pending), due
|
||||
}
|
||||
|
||||
// RecordKey 计算并归一化记录主键。
|
||||
func (q *asyncPendingQueue[T, F]) RecordKey(record T) (string, error) {
|
||||
key, err := RecordKey(q.recordKey, record)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return q.NormalizeRecordKey(key), nil
|
||||
}
|
||||
|
||||
// NormalizeRecordKey 归一化记录主键。
|
||||
func (q *asyncPendingQueue[T, F]) NormalizeRecordKey(key string) string {
|
||||
if q.normalizeKey == nil {
|
||||
return key
|
||||
}
|
||||
return q.normalizeKey(key)
|
||||
}
|
||||
|
||||
// NormalizeFields 归一化字段列表。
|
||||
func (q *asyncPendingQueue[T, F]) NormalizeFields(fields []F) []F {
|
||||
if q.normalizeFields == nil {
|
||||
return append([]F(nil), fields...)
|
||||
}
|
||||
return q.normalizeFields(fields)
|
||||
}
|
||||
|
||||
// Merge 合并两组字段并归一化。
|
||||
func (q *asyncPendingQueue[T, F]) Merge(left, right []F) []F {
|
||||
merged := append(append([]F(nil), left...), right...)
|
||||
return q.NormalizeFields(merged)
|
||||
}
|
||||
|
||||
// Clone 拷贝记录。
|
||||
func (q *asyncPendingQueue[T, F]) Clone(record T) T {
|
||||
return q.clone(record)
|
||||
}
|
||||
|
||||
type keyedPendingSave[T any, F comparable] struct {
|
||||
key string
|
||||
save pendingSave[T, F]
|
||||
}
|
||||
|
||||
func (s pendingSave[T, F]) clone(clone CloneFunc[T], normalize func([]F) []F) pendingSave[T, F] {
|
||||
if clone == nil {
|
||||
clone = IdentityClone[T]
|
||||
}
|
||||
s.profile = clone(s.profile)
|
||||
if normalize != nil {
|
||||
s.fields = normalize(s.fields)
|
||||
} else {
|
||||
s.fields = append([]F(nil), s.fields...)
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package db
|
||||
|
||||
import "time"
|
||||
|
||||
type asyncRetryPolicy[T any, F comparable] struct {
|
||||
backoff time.Duration
|
||||
maxRetries int
|
||||
}
|
||||
|
||||
// RecordFailure 更新失败次数、失败时间和下一次重试时间。
|
||||
func (p asyncRetryPolicy[T, F]) RecordFailure(save pendingSave[T, F], err error, now time.Time) pendingSave[T, F] {
|
||||
save.attempts++
|
||||
if save.firstFailedAt.IsZero() {
|
||||
save.firstFailedAt = now
|
||||
}
|
||||
save.lastFailedAt = now
|
||||
if err != nil {
|
||||
save.lastError = err.Error()
|
||||
}
|
||||
save.nextAttemptAt = now.Add(p.backoff * time.Duration(save.attempts))
|
||||
return save
|
||||
}
|
||||
|
||||
// Exhausted 判断保存请求是否已耗尽重试次数。
|
||||
func (p asyncRetryPolicy[T, F]) Exhausted(save pendingSave[T, F]) bool {
|
||||
return p.maxRetries > 0 && save.attempts >= p.maxRetries
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AsyncStoreOptions 配置异步写后队列、重试和死信存储。
|
||||
type AsyncStoreOptions[T any, F comparable] struct {
|
||||
FlushInterval time.Duration
|
||||
MaxPending int
|
||||
RetryBackoff time.Duration
|
||||
MaxRetries int
|
||||
DeadLetterLimit int
|
||||
DeadLetterStore DeadLetterStore[T, F]
|
||||
|
||||
RecordKey KeyFunc[T]
|
||||
NormalizeKey func(string) string
|
||||
Clone CloneFunc[T]
|
||||
AllFields func() []F
|
||||
NormalizeFields func([]F) []F
|
||||
SaveFields SaveFieldsFunc[T, F]
|
||||
SaveBatch SaveFieldBatchFunc[T, F]
|
||||
AutoExpire bool
|
||||
AutoExpireTTL time.Duration
|
||||
Expire ExpireFunc[T]
|
||||
ClosedError error
|
||||
}
|
||||
|
||||
// AsyncStoreStats 描述异步存储当前运行状态。
|
||||
type AsyncStoreStats struct {
|
||||
Pending int `json:"pending"`
|
||||
PendingDue int `json:"pending_due"`
|
||||
DeadLetters int `json:"dead_letters"`
|
||||
Closed bool `json:"closed"`
|
||||
FlushInterval string `json:"flush_interval"`
|
||||
RetryBackoff string `json:"retry_backoff"`
|
||||
MaxPending int `json:"max_pending"`
|
||||
MaxRetries int `json:"max_retries"`
|
||||
AutoExpireTTL string `json:"auto_expire_ttl,omitempty"`
|
||||
DeadLetterLimit int `json:"dead_letter_limit"`
|
||||
DeadLetterStore string `json:"dead_letter_store,omitempty"`
|
||||
DeadLetterStoreError string `json:"dead_letter_store_error,omitempty"`
|
||||
}
|
||||
|
||||
const defAsyncFlush = 5 * time.Second
|
||||
|
||||
// AsyncStore 用内存队列包装同步 Store，实现异步刷盘和死信兜底。
|
||||
type AsyncStore[T any, F comparable] struct {
|
||||
base Store[T]
|
||||
flushInterval time.Duration
|
||||
maxPending int
|
||||
allFields func() []F
|
||||
closedError error
|
||||
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
|
||||
queue *asyncPendingQueue[T, F]
|
||||
retry asyncRetryPolicy[T, F]
|
||||
deadLetters *asyncDeadLetterSink[T, F]
|
||||
flusher *asyncFlusher[T, F]
|
||||
|
||||
done chan struct{}
|
||||
flushNow chan struct{}
|
||||
stopped chan struct{}
|
||||
stopOnce sync.Once
|
||||
}
|
||||
|
||||
// NewAsyncStore 用写后队列包装同步 Store。
|
||||
// Load 会先读 pending 和 dead-letter，再读 base store；这样即使后台刷盘还没落到持久层，
|
||||
// 调用方也能看到自己已被接受的最新 mutation。
|
||||
func NewAsyncStore[T any, F comparable](base Store[T], options AsyncStoreOptions[T, F]) (*AsyncStore[T, F], error) {
|
||||
if base == nil {
|
||||
return nil, fmt.Errorf("db.AsyncStore requires a base store")
|
||||
}
|
||||
if options.AutoExpire && options.AutoExpireTTL <= 0 {
|
||||
return nil, fmt.Errorf("db.AsyncStore auto expire ttl must be positive")
|
||||
}
|
||||
return newAsyncStore(base, options), nil
|
||||
}
|
||||
|
||||
// MustNewAsyncStore 创建异步存储，失败时 panic，适合启动期固定配置。
|
||||
func MustNewAsyncStore[T any, F comparable](base Store[T], options AsyncStoreOptions[T, F]) *AsyncStore[T, F] {
|
||||
store, err := NewAsyncStore(base, options)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
func newAsyncStore[T any, F comparable](base Store[T], options AsyncStoreOptions[T, F]) *AsyncStore[T, F] {
|
||||
if options.RetryBackoff < 0 {
|
||||
options.RetryBackoff = time.Second
|
||||
}
|
||||
if options.DeadLetterLimit <= 0 {
|
||||
options.DeadLetterLimit = 128
|
||||
}
|
||||
if options.Clone == nil {
|
||||
options.Clone = IdentityClone[T]
|
||||
}
|
||||
if options.SaveFields == nil {
|
||||
options.SaveFields = func(ctx context.Context, store Store[T], record T, fields ...F) error {
|
||||
return SaveFields(ctx, store, record, options.NormalizeFields, fields...)
|
||||
}
|
||||
}
|
||||
if options.SaveBatch == nil {
|
||||
if _, ok := base.(BatchFieldStore[T, F]); ok {
|
||||
options.SaveBatch = func(ctx context.Context, store Store[T], saves []FieldSave[T, F]) error {
|
||||
return SaveFieldBatch(ctx, store, options.NormalizeFields, saves)
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.AutoExpire && options.Expire == nil {
|
||||
options.Expire = func(ctx context.Context, store Store[T], key string, ttl time.Duration) error {
|
||||
return Expire(ctx, store, key, ttl)
|
||||
}
|
||||
}
|
||||
if options.ClosedError == nil {
|
||||
options.ClosedError = ErrAsyncStoreClosed
|
||||
}
|
||||
|
||||
queue := newAsyncPendingQueue(options.RecordKey, options.NormalizeKey, options.Clone, options.NormalizeFields)
|
||||
retry := asyncRetryPolicy[T, F]{backoff: options.RetryBackoff, maxRetries: options.MaxRetries}
|
||||
deadLetters := newAsyncDeadSink(asyncDeadSinkOpts[T, F]{
|
||||
Limit: options.DeadLetterLimit,
|
||||
Store: options.DeadLetterStore,
|
||||
RecordKey: options.RecordKey,
|
||||
NormalizeKey: options.NormalizeKey,
|
||||
Clone: options.Clone,
|
||||
NormalizeFields: options.NormalizeFields,
|
||||
})
|
||||
flusher := newAsyncFlusher(asyncFlusherOptions[T, F]{
|
||||
Base: base,
|
||||
Queue: queue,
|
||||
Retry: retry,
|
||||
DeadLetters: deadLetters,
|
||||
Clone: options.Clone,
|
||||
NormalizeFields: options.NormalizeFields,
|
||||
SaveFields: options.SaveFields,
|
||||
SaveBatch: options.SaveBatch,
|
||||
AutoExpireTTL: options.AutoExpireTTL,
|
||||
Expire: options.Expire,
|
||||
})
|
||||
|
||||
store := &AsyncStore[T, F]{
|
||||
base: base,
|
||||
flushInterval: options.FlushInterval,
|
||||
maxPending: options.MaxPending,
|
||||
allFields: options.AllFields,
|
||||
closedError: options.ClosedError,
|
||||
queue: queue,
|
||||
retry: retry,
|
||||
deadLetters: deadLetters,
|
||||
flusher: flusher,
|
||||
done: make(chan struct{}),
|
||||
flushNow: make(chan struct{}, 1),
|
||||
stopped: make(chan struct{}),
|
||||
}
|
||||
store.deadLetters.Load()
|
||||
go store.run()
|
||||
return store
|
||||
}
|
||||
|
||||
// Load 优先读取 pending 和 dead-letter，再读取底层存储。
|
||||
func (s *AsyncStore[T, F]) Load(ctx context.Context, accountID string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
accountID = s.queue.NormalizeRecordKey(accountID)
|
||||
if pending, ok := s.queue.Get(accountID); ok {
|
||||
return s.queue.Clone(pending.profile), true, nil
|
||||
}
|
||||
if dead, ok := s.deadLetters.Get(accountID); ok {
|
||||
return s.queue.Clone(dead.Profile), true, nil
|
||||
}
|
||||
return s.base.Load(ctx, accountID)
|
||||
}
|
||||
|
||||
// Check 检查异步存储、底层存储和死信存储状态。
|
||||
func (s *AsyncStore[T, F]) Check(ctx context.Context) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
s.mu.Lock()
|
||||
closed := s.closed
|
||||
base := s.base
|
||||
closedError := s.closedError
|
||||
s.mu.Unlock()
|
||||
|
||||
if closed {
|
||||
return closedError
|
||||
}
|
||||
if deadLetterStoreError := s.deadLetters.LastError(); deadLetterStoreError != "" {
|
||||
return fmt.Errorf("dead letter store: %s", deadLetterStoreError)
|
||||
}
|
||||
if err := Check(ctx, base); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.deadLetters.CheckStore(ctx)
|
||||
}
|
||||
|
||||
// Save 保存完整记录字段。
|
||||
func (s *AsyncStore[T, F]) Save(ctx context.Context, profile T) error {
|
||||
fields := s.allFieldsForSave()
|
||||
if len(fields) == 0 {
|
||||
return ErrAllFieldsRequired
|
||||
}
|
||||
return s.SaveFields(ctx, profile, fields...)
|
||||
}
|
||||
|
||||
// SaveFields 把局部字段写入 pending 队列。
|
||||
func (s *AsyncStore[T, F]) SaveFields(ctx context.Context, profile T, fields ...F) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
accountID, err := s.queue.RecordKey(profile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fields = s.queue.NormalizeFields(fields)
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
save := pendingSave[T, F]{
|
||||
profile: s.queue.Clone(profile),
|
||||
fields: fields,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return s.closedError
|
||||
}
|
||||
pending := s.queue.Add(accountID, save)
|
||||
s.mu.Unlock()
|
||||
|
||||
if s.maxPending > 0 && pending >= s.maxPending {
|
||||
s.requestFlush()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush 强制刷出当前可写 pending 队列。
|
||||
func (s *AsyncStore[T, F]) Flush(ctx context.Context) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
return s.flusher.Flush(ctx, true)
|
||||
}
|
||||
|
||||
// Stats 返回异步队列、死信和重试状态。
|
||||
func (s *AsyncStore[T, F]) Stats() AsyncStoreStats {
|
||||
now := time.Now().UTC()
|
||||
pending, pendingDue := s.queue.Stats(now)
|
||||
|
||||
s.mu.Lock()
|
||||
closed := s.closed
|
||||
s.mu.Unlock()
|
||||
|
||||
return AsyncStoreStats{
|
||||
Pending: pending,
|
||||
PendingDue: pendingDue,
|
||||
DeadLetters: s.deadLetters.Count(),
|
||||
Closed: closed,
|
||||
FlushInterval: s.flushInterval.String(),
|
||||
RetryBackoff: s.retry.backoff.String(),
|
||||
MaxPending: s.maxPending,
|
||||
MaxRetries: s.retry.maxRetries,
|
||||
AutoExpireTTL: asyncDurationString(s.flusher.AutoExpireTTL()),
|
||||
DeadLetterLimit: s.deadLetters.Limit(),
|
||||
DeadLetterStore: s.deadLetters.StoreName(),
|
||||
DeadLetterStoreError: s.deadLetters.LastError(),
|
||||
}
|
||||
}
|
||||
|
||||
// DeadLetters 返回当前死信快照。
|
||||
func (s *AsyncStore[T, F]) DeadLetters() []AsyncStoreDeadLetter[T, F] {
|
||||
return s.deadLetters.List()
|
||||
}
|
||||
|
||||
// RequeueDeadLetter 把指定死信重新放回 pending 队列。
|
||||
func (s *AsyncStore[T, F]) RequeueDeadLetter(accountID string) bool {
|
||||
accountID = s.queue.NormalizeRecordKey(accountID)
|
||||
s.mu.Lock()
|
||||
if s.closed {
|
||||
s.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
dead, ok := s.deadLetters.Take(accountID)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
fields := s.queue.NormalizeFields(dead.Fields)
|
||||
if len(fields) == 0 {
|
||||
fields = s.allFieldsForSave()
|
||||
}
|
||||
save := pendingSave[T, F]{
|
||||
profile: s.queue.Clone(dead.Profile),
|
||||
fields: fields,
|
||||
attempts: dead.Attempts,
|
||||
lastError: dead.Error,
|
||||
firstFailedAt: dead.FirstFailedAt,
|
||||
lastFailedAt: dead.LastFailedAt,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.closed {
|
||||
if err := s.deadLetters.Add(save); err != nil {
|
||||
s.deadLetters.restoreMemory(dead)
|
||||
}
|
||||
return false
|
||||
}
|
||||
s.queue.Requeue(accountID, save)
|
||||
return true
|
||||
}
|
||||
|
||||
// Close 停止后台刷盘并做最后一次 Flush。
|
||||
func (s *AsyncStore[T, F]) Close(ctx context.Context) error {
|
||||
ctx, cancel := asyncCloseCtx(ctx)
|
||||
defer cancel()
|
||||
s.mu.Lock()
|
||||
s.closed = true
|
||||
s.mu.Unlock()
|
||||
|
||||
s.stopOnce.Do(func() {
|
||||
close(s.done)
|
||||
})
|
||||
select {
|
||||
case <-s.stopped:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
return errors.Join(s.Flush(ctx), CloseOrFlush(ctx, s.base))
|
||||
}
|
||||
|
||||
func asyncCloseCtx(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return ctx, func() {}
|
||||
}
|
||||
return context.WithTimeout(ctx, defAsyncFlush)
|
||||
}
|
||||
|
||||
func (s *AsyncStore[T, F]) run() {
|
||||
defer close(s.stopped)
|
||||
if s.flushInterval <= 0 {
|
||||
for {
|
||||
select {
|
||||
case <-s.flushNow:
|
||||
s.flushBackground()
|
||||
case <-s.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.flushInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
s.flushBackground()
|
||||
case <-s.flushNow:
|
||||
s.flushBackground()
|
||||
case <-s.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AsyncStore[T, F]) requestFlush() {
|
||||
select {
|
||||
case s.flushNow <- struct{}{}:
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *AsyncStore[T, F]) flushBackground() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defAsyncFlush)
|
||||
_ = s.flusher.Flush(ctx, false)
|
||||
cancel()
|
||||
}
|
||||
|
||||
func (s *AsyncStore[T, F]) allFieldsForSave() []F {
|
||||
if s.allFields == nil {
|
||||
return nil
|
||||
}
|
||||
return s.queue.NormalizeFields(s.allFields())
|
||||
}
|
||||
|
||||
func asyncDurationString(duration time.Duration) string {
|
||||
if duration <= 0 {
|
||||
return ""
|
||||
}
|
||||
return duration.String()
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// WriteFileAtomically 通过同目录临时文件、fsync 和原子替换持久化写入单个文件。
|
||||
func WriteFileAtomically(ctx context.Context, path string, data []byte, perm fs.FileMode) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if path == "" {
|
||||
return fmt.Errorf("atomic file path is required")
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
keepTmp := true
|
||||
defer func() {
|
||||
if keepTmp {
|
||||
_ = os.Remove(tmpName)
|
||||
}
|
||||
}()
|
||||
if err := writeAndSyncTempFile(tmp, data, perm); err != nil {
|
||||
_ = tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := replaceFile(tmpName, path); err != nil {
|
||||
return err
|
||||
}
|
||||
keepTmp = false
|
||||
return syncDir(dir)
|
||||
}
|
||||
|
||||
func writeAndSyncTempFile(file *os.File, data []byte, perm fs.FileMode) error {
|
||||
written, err := file.Write(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written != len(data) {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
if err := file.Chmod(perm); err != nil {
|
||||
return err
|
||||
}
|
||||
return file.Sync()
|
||||
}
|
||||
|
||||
func replaceFile(tmpName, path string) error {
|
||||
if err := os.Rename(tmpName, path); err == nil {
|
||||
return nil
|
||||
} else if _, statErr := os.Stat(path); statErr != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
backupPath := tmpName + ".bak"
|
||||
if err := os.Rename(path, backupPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmpName, path); err != nil {
|
||||
restoreErr := os.Rename(backupPath, path)
|
||||
return errors.Join(err, restoreErr)
|
||||
}
|
||||
if err := os.Remove(backupPath); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func syncDir(dir string) error {
|
||||
file, err := os.Open(dir) //nolint:gosec // G304：目录来自调用点限定的持久化路径。
|
||||
if err != nil {
|
||||
if runtime.GOOS == "windows" {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
if err := file.Sync(); err != nil && runtime.GOOS != "windows" {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// BatchLoader 支持按主键批量加载记录。
|
||||
type BatchLoader[T any] interface {
|
||||
LoadBatch(context.Context, []string) (map[string]T, error)
|
||||
}
|
||||
|
||||
// LoadBatch 批量加载记录；底层不支持批量时逐条加载。
|
||||
func LoadBatch[T any](ctx context.Context, store Store[T], keys []string, normalizeKey func(string) string) (map[string]T, error) {
|
||||
if store == nil {
|
||||
return nil, errors.New("store is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
keys = normalizeBatchKeys(keys, normalizeKey)
|
||||
if len(keys) == 0 {
|
||||
return map[string]T{}, ctx.Err()
|
||||
}
|
||||
if loader, ok := store.(BatchLoader[T]); ok {
|
||||
return loader.LoadBatch(ctx, keys)
|
||||
}
|
||||
out := make(map[string]T, len(keys))
|
||||
for _, key := range keys {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
record, ok, err := store.Load(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
out[key] = record
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func normalizeBatchKeys(keys []string, normalizeKey func(string) string) []string {
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(keys))
|
||||
for _, key := range keys {
|
||||
key = normalizeLookupKey(key, normalizeKey)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for key := range seen {
|
||||
out = append(out, key)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeLookupKey(key string, normalizeKey func(string) string) string {
|
||||
if normalizeKey != nil {
|
||||
key = normalizeKey(key)
|
||||
}
|
||||
return strings.TrimSpace(key)
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AsyncStoreDeadLetter 描述异步存储刷盘失败后的死信记录。
|
||||
type AsyncStoreDeadLetter[T any, F comparable] struct {
|
||||
AccountID string `json:"account_id"`
|
||||
Profile T `json:"profile"`
|
||||
Fields []F `json:"fields,omitempty"`
|
||||
Attempts int `json:"attempts"`
|
||||
Error string `json:"error"`
|
||||
FirstFailedAt time.Time `json:"first_failed_at"`
|
||||
LastFailedAt time.Time `json:"last_failed_at"`
|
||||
}
|
||||
|
||||
// DeadLetterStore 是死信持久化接口。
|
||||
type DeadLetterStore[T any, F comparable] interface {
|
||||
Save(context.Context, AsyncStoreDeadLetter[T, F]) error
|
||||
Delete(context.Context, string) error
|
||||
List(context.Context) ([]AsyncStoreDeadLetter[T, F], error)
|
||||
}
|
||||
|
||||
// FileDeadLetterStoreOptions 配置文件死信存储目录。
|
||||
type FileDeadLetterStoreOptions struct {
|
||||
Directory string
|
||||
DefaultDirectory string
|
||||
}
|
||||
|
||||
// FileDeadLetterStore 用本地 JSON 文件保存死信。
|
||||
type FileDeadLetterStore[T any, F comparable] struct {
|
||||
mu sync.RWMutex
|
||||
dir string
|
||||
}
|
||||
|
||||
// NewFileDeadLetterStore 创建文件死信存储。
|
||||
func NewFileDeadLetterStore[T any, F comparable](options FileDeadLetterStoreOptions) *FileDeadLetterStore[T, F] {
|
||||
dir := strings.TrimSpace(options.Directory)
|
||||
if dir == "" {
|
||||
dir = strings.TrimSpace(options.DefaultDirectory)
|
||||
}
|
||||
if dir == "" {
|
||||
dir = "data/dead_letters"
|
||||
}
|
||||
return &FileDeadLetterStore[T, F]{dir: dir}
|
||||
}
|
||||
|
||||
// Save 原子写入一条死信文件。
|
||||
func (s *FileDeadLetterStore[T, F]) Save(ctx context.Context, dead AsyncStoreDeadLetter[T, F]) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := s.deadLetterPath(dead.AccountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(dead, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return WriteFileAtomically(ctx, path, data, 0o600)
|
||||
}
|
||||
|
||||
// Check 检查死信目录是否可写。
|
||||
func (s *FileDeadLetterStore[T, F]) Check(ctx context.Context) error {
|
||||
return checkWritableDir(ctx, s.dir)
|
||||
}
|
||||
|
||||
// Delete 删除指定账号死信文件。
|
||||
func (s *FileDeadLetterStore[T, F]) Delete(ctx context.Context, accountID string) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := s.deadLetterPath(accountID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List 读取并按失败时间排序全部死信。
|
||||
func (s *FileDeadLetterStore[T, F]) List(ctx context.Context) ([]AsyncStoreDeadLetter[T, F], error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
entries, err := os.ReadDir(s.dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]AsyncStoreDeadLetter[T, F], 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
|
||||
continue
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(s.dir, entry.Name()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var dead AsyncStoreDeadLetter[T, F]
|
||||
if err := json.Unmarshal(data, &dead); err != nil {
|
||||
return nil, fmt.Errorf("decode dead letter %s: %w", entry.Name(), err)
|
||||
}
|
||||
out = append(out, dead)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].LastFailedAt.Equal(out[j].LastFailedAt) {
|
||||
return out[i].AccountID < out[j].AccountID
|
||||
}
|
||||
return out[i].LastFailedAt.Before(out[j].LastFailedAt)
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *FileDeadLetterStore[T, F]) deadLetterPath(accountID string) (string, error) {
|
||||
accountID = strings.TrimSpace(accountID)
|
||||
if accountID == "" {
|
||||
return "", ErrRecordKeyRequired
|
||||
}
|
||||
name := base64.RawURLEncoding.EncodeToString([]byte(accountID)) + ".json"
|
||||
return filepath.Join(s.dir, name), nil
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DirtyRecordOptions 配置脏字段记录的初始值和字段集合。
|
||||
type DirtyRecordOptions[T any, F comparable] struct {
|
||||
Record T
|
||||
Fields FieldRegistry[F]
|
||||
Clone CloneFunc[T]
|
||||
Dirty []F
|
||||
MarkAllDirty bool
|
||||
}
|
||||
|
||||
// DirtySaveResult 描述一次脏字段保存结果。
|
||||
type DirtySaveResult[F comparable] struct {
|
||||
Saved bool
|
||||
Fields []F
|
||||
Remaining []F
|
||||
}
|
||||
|
||||
// DirtyRecord 在内存中维护记录快照和脏字段集合。
|
||||
type DirtyRecord[T any, F comparable] struct {
|
||||
mu sync.RWMutex
|
||||
record T
|
||||
fields FieldRegistry[F]
|
||||
clone CloneFunc[T]
|
||||
dirty map[F]uint64
|
||||
revision uint64
|
||||
}
|
||||
|
||||
// NewDirtyRecord 创建脏字段记录。
|
||||
func NewDirtyRecord[T any, F comparable](options DirtyRecordOptions[T, F]) *DirtyRecord[T, F] {
|
||||
clone := options.Clone
|
||||
if clone == nil {
|
||||
clone = IdentityClone[T]
|
||||
}
|
||||
r := &DirtyRecord[T, F]{
|
||||
record: clone(options.Record),
|
||||
fields: options.Fields,
|
||||
clone: clone,
|
||||
dirty: make(map[F]uint64),
|
||||
}
|
||||
if options.MarkAllDirty {
|
||||
r.markAllDirtyLocked()
|
||||
} else {
|
||||
r.markDirtyLocked(options.Dirty...)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// Record 返回当前记录副本。
|
||||
func (r *DirtyRecord[T, F]) Record() T {
|
||||
r.mu.RLock()
|
||||
record := r.clone(r.record)
|
||||
r.mu.RUnlock()
|
||||
return record
|
||||
}
|
||||
|
||||
// Snapshot 返回当前记录副本和脏字段列表。
|
||||
func (r *DirtyRecord[T, F]) Snapshot() (T, []F) {
|
||||
r.mu.RLock()
|
||||
record := r.clone(r.record)
|
||||
fields := r.dirtyFieldsLocked()
|
||||
r.mu.RUnlock()
|
||||
return record, fields
|
||||
}
|
||||
|
||||
// DirtyFields 返回当前脏字段列表。
|
||||
func (r *DirtyRecord[T, F]) DirtyFields() []F {
|
||||
r.mu.RLock()
|
||||
fields := r.dirtyFieldsLocked()
|
||||
r.mu.RUnlock()
|
||||
return fields
|
||||
}
|
||||
|
||||
// MarkDirty 标记指定字段为脏。
|
||||
func (r *DirtyRecord[T, F]) MarkDirty(fields ...F) {
|
||||
r.mu.Lock()
|
||||
r.markDirtyLocked(fields...)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// MarkAllDirty 标记全部字段为脏。
|
||||
func (r *DirtyRecord[T, F]) MarkAllDirty() {
|
||||
r.mu.Lock()
|
||||
r.markAllDirtyLocked()
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// ClearDirty 清除指定字段的脏标记。
|
||||
func (r *DirtyRecord[T, F]) ClearDirty(fields ...F) {
|
||||
r.mu.Lock()
|
||||
for _, field := range r.fields.Normalize(fields) {
|
||||
delete(r.dirty, field)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// ClearAllDirty 清除全部脏标记。
|
||||
func (r *DirtyRecord[T, F]) ClearAllDirty() {
|
||||
r.mu.Lock()
|
||||
clear(r.dirty)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// Replace 替换记录并标记指定字段。
|
||||
func (r *DirtyRecord[T, F]) Replace(record T, fields ...F) {
|
||||
r.mu.Lock()
|
||||
r.record = r.clone(record)
|
||||
r.markDirtyLocked(fields...)
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// Mutate 在锁内修改记录，并返回修改前后副本。
|
||||
func (r *DirtyRecord[T, F]) Mutate(mutate func(*T) ([]F, error)) (T, T, error) {
|
||||
var zero T
|
||||
if mutate == nil {
|
||||
return zero, zero, errors.New("dirty record mutation is required")
|
||||
}
|
||||
r.mu.Lock()
|
||||
before := r.clone(r.record)
|
||||
next := r.clone(r.record)
|
||||
fields, err := mutate(&next)
|
||||
if err != nil {
|
||||
r.mu.Unlock()
|
||||
return before, before, err
|
||||
}
|
||||
r.record = r.clone(next)
|
||||
r.markDirtyLocked(fields...)
|
||||
after := r.clone(r.record)
|
||||
r.mu.Unlock()
|
||||
return before, after, nil
|
||||
}
|
||||
|
||||
// SaveDirty 保存当前脏字段，成功后只清除本次已保存字段。
|
||||
func (r *DirtyRecord[T, F]) SaveDirty(ctx context.Context, store Store[T]) (DirtySaveResult[F], error) {
|
||||
if store == nil {
|
||||
return DirtySaveResult[F]{}, errors.New("store is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
record, fields, revisions := r.saveSnapshot()
|
||||
if len(fields) == 0 {
|
||||
return DirtySaveResult[F]{Saved: false}, ctx.Err()
|
||||
}
|
||||
if err := SaveFields(ctx, store, record, r.fields.Normalize, fields...); err != nil {
|
||||
return DirtySaveResult[F]{Saved: false, Fields: append([]F(nil), fields...), Remaining: r.DirtyFields()}, err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
for _, field := range fields {
|
||||
if current, ok := r.dirty[field]; ok && current == revisions[field] {
|
||||
delete(r.dirty, field)
|
||||
}
|
||||
}
|
||||
remaining := r.dirtyFieldsLocked()
|
||||
r.mu.Unlock()
|
||||
return DirtySaveResult[F]{
|
||||
Saved: true,
|
||||
Fields: append([]F(nil), fields...),
|
||||
Remaining: remaining,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *DirtyRecord[T, F]) saveSnapshot() (T, []F, map[F]uint64) {
|
||||
r.mu.RLock()
|
||||
record := r.clone(r.record)
|
||||
fields := r.dirtyFieldsLocked()
|
||||
revisions := make(map[F]uint64, len(fields))
|
||||
for _, field := range fields {
|
||||
revisions[field] = r.dirty[field]
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
return record, fields, revisions
|
||||
}
|
||||
|
||||
func (r *DirtyRecord[T, F]) markDirtyLocked(fields ...F) {
|
||||
for _, field := range r.fields.Normalize(fields) {
|
||||
r.revision++
|
||||
r.dirty[field] = r.revision
|
||||
}
|
||||
}
|
||||
|
||||
func (r *DirtyRecord[T, F]) markAllDirtyLocked() {
|
||||
r.markDirtyLocked(r.fields.All()...)
|
||||
}
|
||||
|
||||
func (r *DirtyRecord[T, F]) dirtyFieldsLocked() []F {
|
||||
if len(r.dirty) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]F, 0, len(r.dirty))
|
||||
for _, field := range r.fields.All() {
|
||||
if _, ok := r.dirty[field]; ok {
|
||||
out = append(out, field)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Package db 提供 Profile、读模型、事件和 runtime 状态模块共用的存储契约。
|
||||
//
|
||||
// 这里刻意拆开克隆、字段级脏标记、异步刷盘、重试和死信处理，让玩法模块在不改
|
||||
// mutation 代码的前提下切换 memory、Redis、SQL 或 Mongo 后端。
|
||||
package db
|
||||
@@ -0,0 +1,155 @@
|
||||
package db
|
||||
|
||||
// FieldRegistry 保存字段的合法集合和稳定顺序。
|
||||
type FieldRegistry[F comparable] struct {
|
||||
order []F
|
||||
known map[F]struct{}
|
||||
}
|
||||
|
||||
// NewFieldRegistry 创建字段注册表。
|
||||
func NewFieldRegistry[F comparable](order []F) FieldRegistry[F] {
|
||||
registry := FieldRegistry[F]{
|
||||
order: append([]F(nil), order...),
|
||||
known: make(map[F]struct{}, len(order)),
|
||||
}
|
||||
for _, field := range order {
|
||||
registry.known[field] = struct{}{}
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
// All 返回全部字段并保持注册顺序。
|
||||
func (r FieldRegistry[F]) All() []F {
|
||||
return append([]F(nil), r.order...)
|
||||
}
|
||||
|
||||
// IsKnown 判断字段是否已注册。
|
||||
func (r FieldRegistry[F]) IsKnown(field F) bool {
|
||||
_, ok := r.known[field]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Normalize 去重、过滤未知字段，并按注册顺序返回。
|
||||
func (r FieldRegistry[F]) Normalize(fields []F) []F {
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[F]struct{}, len(fields))
|
||||
for _, field := range fields {
|
||||
if r.IsKnown(field) {
|
||||
seen[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]F, 0, len(seen))
|
||||
for _, field := range r.order {
|
||||
if _, ok := seen[field]; ok {
|
||||
out = append(out, field)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Merge 合并两组字段并做归一化。
|
||||
func (r FieldRegistry[F]) Merge(left, right []F) []F {
|
||||
if len(left) == 0 {
|
||||
return r.Normalize(right)
|
||||
}
|
||||
if len(right) == 0 {
|
||||
return r.Normalize(left)
|
||||
}
|
||||
merged := make([]F, 0, len(left)+len(right))
|
||||
merged = append(merged, left...)
|
||||
merged = append(merged, right...)
|
||||
return r.Normalize(merged)
|
||||
}
|
||||
|
||||
// NewSet 创建字段集合。
|
||||
func (r FieldRegistry[F]) NewSet(fields ...F) FieldSet[F] {
|
||||
set := FieldSet[F]{
|
||||
registry: r,
|
||||
values: make(map[F]struct{}),
|
||||
}
|
||||
set.Add(fields...)
|
||||
return set
|
||||
}
|
||||
|
||||
// AllSet 创建包含所有字段的集合。
|
||||
func (r FieldRegistry[F]) AllSet() FieldSet[F] {
|
||||
set := r.NewSet()
|
||||
set.AddAll()
|
||||
return set
|
||||
}
|
||||
|
||||
// FieldSet 表示一组已注册字段。
|
||||
type FieldSet[F comparable] struct {
|
||||
registry FieldRegistry[F]
|
||||
values map[F]struct{}
|
||||
}
|
||||
|
||||
// IsZero 判断字段集合是否为空。
|
||||
func (s FieldSet[F]) IsZero() bool {
|
||||
return len(s.values) == 0
|
||||
}
|
||||
|
||||
// Add 向集合加入字段并忽略未知字段。
|
||||
func (s FieldSet[F]) Add(fields ...F) {
|
||||
if s.values == nil {
|
||||
return
|
||||
}
|
||||
for _, field := range fields {
|
||||
if s.registry.IsKnown(field) {
|
||||
s.values[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddAll 把注册表里的所有字段加入集合。
|
||||
func (s FieldSet[F]) AddAll() {
|
||||
if s.values == nil {
|
||||
return
|
||||
}
|
||||
for _, field := range s.registry.order {
|
||||
s.values[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge 合并另一个字段集合。
|
||||
func (s FieldSet[F]) Merge(other FieldSet[F]) {
|
||||
if s.values == nil {
|
||||
return
|
||||
}
|
||||
for field := range other.values {
|
||||
if s.registry.IsKnown(field) {
|
||||
s.values[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// List 按注册顺序返回字段列表。
|
||||
func (s FieldSet[F]) List() []F {
|
||||
if len(s.values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]F, 0, len(s.values))
|
||||
for _, field := range s.registry.order {
|
||||
if _, ok := s.values[field]; ok {
|
||||
out = append(out, field)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Clone 拷贝字段集合。
|
||||
func (s FieldSet[F]) Clone() FieldSet[F] {
|
||||
if len(s.values) == 0 {
|
||||
return FieldSet[F]{registry: s.registry}
|
||||
}
|
||||
out := FieldSet[F]{
|
||||
registry: s.registry,
|
||||
values: make(map[F]struct{}, len(s.values)),
|
||||
}
|
||||
for field := range s.values {
|
||||
out.values[field] = struct{}{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrHashBackendRequired 表示 hash 后端缺失或没有完成初始化。
|
||||
var ErrHashBackendRequired = errors.New("hash backend is required")
|
||||
|
||||
// HashBackend 是按记录主键和字段名读写二进制字段的后端接口。
|
||||
type HashBackend interface {
|
||||
LoadHash(context.Context, string, []string) (map[string][]byte, bool, error)
|
||||
SaveHash(context.Context, string, map[string][]byte, time.Duration) error
|
||||
}
|
||||
|
||||
// HashSaveBatch 表示一次 hash 字段批量保存项。
|
||||
type HashSaveBatch struct {
|
||||
Key string
|
||||
Fields map[string][]byte
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// HashBatchBackend 支持批量保存 hash 字段。
|
||||
type HashBatchBackend interface {
|
||||
SaveHashBatch(context.Context, []HashSaveBatch) error
|
||||
}
|
||||
|
||||
// HashLoadRequest 描述一次 hash 批量读取请求。
|
||||
type HashLoadRequest struct {
|
||||
Key string
|
||||
Fields []string
|
||||
}
|
||||
|
||||
// HashLoadResult 描述一次 hash 批量读取结果。
|
||||
type HashLoadResult struct {
|
||||
Fields map[string][]byte
|
||||
Found bool
|
||||
}
|
||||
|
||||
// HashBatchLoaderBackend 支持批量读取 hash 字段。
|
||||
type HashBatchLoaderBackend interface {
|
||||
LoadHashBatch(context.Context, []HashLoadRequest) (map[string]HashLoadResult, error)
|
||||
}
|
||||
|
||||
// MemoryHashBackend 是线程安全的内存 hash 后端。
|
||||
type MemoryHashBackend struct {
|
||||
mu sync.RWMutex
|
||||
records map[string]map[string][]byte
|
||||
ttl map[string]time.Duration
|
||||
}
|
||||
|
||||
// NewMemoryHashBackend 创建内存 hash 后端。
|
||||
func NewMemoryHashBackend() *MemoryHashBackend {
|
||||
return &MemoryHashBackend{
|
||||
records: make(map[string]map[string][]byte),
|
||||
ttl: make(map[string]time.Duration),
|
||||
}
|
||||
}
|
||||
|
||||
// LoadHash 从内存读取指定 hash 字段。
|
||||
func (b *MemoryHashBackend) LoadHash(ctx context.Context, key string, fields []string) (map[string][]byte, bool, error) {
|
||||
if b == nil {
|
||||
return nil, false, ErrHashBackendRequired
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return nil, false, ErrRecordKeyRequired
|
||||
}
|
||||
fields = normHashFields(fields)
|
||||
|
||||
b.mu.RLock()
|
||||
record, ok := b.records[key]
|
||||
if !ok {
|
||||
b.mu.RUnlock()
|
||||
return nil, false, nil
|
||||
}
|
||||
out := make(map[string][]byte, len(record))
|
||||
if len(fields) == 0 {
|
||||
for name, data := range record {
|
||||
out[name] = append([]byte(nil), data...)
|
||||
}
|
||||
} else {
|
||||
for _, name := range fields {
|
||||
if data, exists := record[name]; exists {
|
||||
out[name] = append([]byte(nil), data...)
|
||||
}
|
||||
}
|
||||
}
|
||||
b.mu.RUnlock()
|
||||
return out, true, nil
|
||||
}
|
||||
|
||||
// LoadHashBatch 批量读取 hash 字段。
|
||||
func (b *MemoryHashBackend) LoadHashBatch(ctx context.Context, requests []HashLoadRequest) (map[string]HashLoadResult, error) {
|
||||
if b == nil {
|
||||
return nil, ErrHashBackendRequired
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]HashLoadResult, len(requests))
|
||||
for _, request := range requests {
|
||||
fields, found, err := b.LoadHash(ctx, request.Key, request.Fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[strings.TrimSpace(request.Key)] = HashLoadResult{Fields: fields, Found: found}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// SaveHash 保存指定 hash 字段。
|
||||
func (b *MemoryHashBackend) SaveHash(ctx context.Context, key string, fields map[string][]byte, ttl time.Duration) error {
|
||||
if b == nil {
|
||||
return ErrHashBackendRequired
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return ErrRecordKeyRequired
|
||||
}
|
||||
fields, err := cloneHashFields(fields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.ensureReadyLocked()
|
||||
record := b.records[key]
|
||||
if record == nil {
|
||||
record = make(map[string][]byte, len(fields))
|
||||
b.records[key] = record
|
||||
}
|
||||
for name, data := range fields {
|
||||
record[name] = append([]byte(nil), data...)
|
||||
}
|
||||
if ttl > 0 {
|
||||
b.ttl[key] = ttl
|
||||
}
|
||||
b.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveHashBatch 批量保存 hash 字段。
|
||||
func (b *MemoryHashBackend) SaveHashBatch(ctx context.Context, batches []HashSaveBatch) error {
|
||||
if b == nil {
|
||||
return ErrHashBackendRequired
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
merged, err := mergeHashSaveBatches(batches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, batch := range merged {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.SaveHash(ctx, batch.Key, batch.Fields, batch.TTL); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeHashSaveBatches(batches []HashSaveBatch) ([]HashSaveBatch, error) {
|
||||
if len(batches) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
merged := make(map[string]HashSaveBatch, len(batches))
|
||||
for _, batch := range batches {
|
||||
if len(batch.Fields) == 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(batch.Key)
|
||||
if key == "" {
|
||||
return nil, ErrRecordKeyRequired
|
||||
}
|
||||
fields, err := cloneHashFields(batch.Fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
current := merged[key]
|
||||
if current.Fields == nil {
|
||||
current = HashSaveBatch{
|
||||
Key: key,
|
||||
Fields: make(map[string][]byte, len(fields)),
|
||||
}
|
||||
}
|
||||
for name, data := range fields {
|
||||
current.Fields[name] = append([]byte(nil), data...)
|
||||
}
|
||||
current.TTL = batch.TTL
|
||||
merged[key] = current
|
||||
}
|
||||
keys := make([]string, 0, len(merged))
|
||||
for key := range merged {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]HashSaveBatch, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
batch := merged[key]
|
||||
batch.Fields = cloneBytesMap(batch.Fields)
|
||||
out = append(out, batch)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Check 检查上下文是否已取消。
|
||||
func (b *MemoryHashBackend) Check(ctx context.Context) error {
|
||||
if b == nil {
|
||||
return ErrHashBackendRequired
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Snapshot 返回全部 hash 记录副本。
|
||||
func (b *MemoryHashBackend) Snapshot() map[string]map[string][]byte {
|
||||
if b == nil {
|
||||
return map[string]map[string][]byte{}
|
||||
}
|
||||
b.mu.RLock()
|
||||
out := make(map[string]map[string][]byte, len(b.records))
|
||||
for key, fields := range b.records {
|
||||
out[key] = cloneBytesMap(fields)
|
||||
}
|
||||
b.mu.RUnlock()
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *MemoryHashBackend) ensureReadyLocked() {
|
||||
if b.records == nil {
|
||||
b.records = make(map[string]map[string][]byte)
|
||||
}
|
||||
if b.ttl == nil {
|
||||
b.ttl = make(map[string]time.Duration)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneHashFields(fields map[string][]byte) (map[string][]byte, error) {
|
||||
if len(fields) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make(map[string][]byte, len(fields))
|
||||
names := make([]string, 0, len(fields))
|
||||
for rawName := range fields {
|
||||
names = append(names, rawName)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, rawName := range names {
|
||||
name := strings.TrimSpace(rawName)
|
||||
if name == "" {
|
||||
return nil, ErrRecordKeyRequired
|
||||
}
|
||||
if _, exists := out[name]; exists {
|
||||
continue
|
||||
}
|
||||
data := fields[rawName]
|
||||
if data == nil {
|
||||
return nil, errors.New("hash field value is nil")
|
||||
}
|
||||
out[name] = append([]byte(nil), data...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cloneBytesMap(in map[string][]byte) map[string][]byte {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]byte, len(in))
|
||||
for key, data := range in {
|
||||
out[key] = append([]byte(nil), data...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normHashFields(fields []string) []string {
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(fields))
|
||||
for _, field := range fields {
|
||||
field = strings.TrimSpace(field)
|
||||
if field != "" {
|
||||
seen[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for field := range seen {
|
||||
out = append(out, field)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// JSONFileStoreOptions 配置 JSON 文件记录存储。
|
||||
type JSONFileStoreOptions[T any] struct {
|
||||
Directory string
|
||||
DefaultDirectory string
|
||||
Key KeyFunc[T]
|
||||
Clone CloneFunc[T]
|
||||
WithLoadedKey func(T, string) T
|
||||
}
|
||||
|
||||
// JSONFileStore 用单记录 JSON 文件实现 Store。
|
||||
type JSONFileStore[T any] struct {
|
||||
mu sync.RWMutex
|
||||
dir string
|
||||
keyFn KeyFunc[T]
|
||||
cloneFn CloneFunc[T]
|
||||
withLoadedKey func(T, string) T
|
||||
}
|
||||
|
||||
// NewJSONFileStore 创建 JSON 文件记录存储。
|
||||
func NewJSONFileStore[T any](options JSONFileStoreOptions[T]) *JSONFileStore[T] {
|
||||
dir := strings.TrimSpace(options.Directory)
|
||||
if dir == "" {
|
||||
dir = strings.TrimSpace(options.DefaultDirectory)
|
||||
}
|
||||
if dir == "" {
|
||||
dir = "data/records"
|
||||
}
|
||||
cloneFn := options.Clone
|
||||
if cloneFn == nil {
|
||||
cloneFn = IdentityClone[T]
|
||||
}
|
||||
return &JSONFileStore[T]{
|
||||
dir: dir,
|
||||
keyFn: options.Key,
|
||||
cloneFn: cloneFn,
|
||||
withLoadedKey: options.WithLoadedKey,
|
||||
}
|
||||
}
|
||||
|
||||
// Load 从 JSON 文件读取记录。
|
||||
func (s *JSONFileStore[T]) Load(ctx context.Context, key string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
path, err := s.recordPath(key)
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
s.mu.RLock()
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
|
||||
s.mu.RUnlock()
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return zero, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
var record T
|
||||
if err := json.Unmarshal(data, &record); err != nil {
|
||||
return zero, false, fmt.Errorf("decode record %s: %w", key, err)
|
||||
}
|
||||
if s.withLoadedKey != nil {
|
||||
record = s.withLoadedKey(record, key)
|
||||
}
|
||||
return s.cloneFn(record), true, nil
|
||||
}
|
||||
|
||||
// Check 检查记录目录是否可写。
|
||||
func (s *JSONFileStore[T]) Check(ctx context.Context) error {
|
||||
return checkWritableDir(ctx, s.dir)
|
||||
}
|
||||
|
||||
// Save 原子写入记录 JSON 文件。
|
||||
func (s *JSONFileStore[T]) Save(ctx context.Context, record T) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
key, err := RecordKey(s.keyFn, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
path, err := s.recordPath(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(record, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return WriteFileAtomically(ctx, path, data, 0o600)
|
||||
}
|
||||
|
||||
func (s *JSONFileStore[T]) recordPath(key string) (string, error) {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return "", ErrRecordKeyRequired
|
||||
}
|
||||
name := base64.RawURLEncoding.EncodeToString([]byte(key)) + ".json"
|
||||
return filepath.Join(s.dir, name), nil
|
||||
}
|
||||
|
||||
func checkWritableDir(ctx context.Context, dir string) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(dir, ".preflight-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
if err := tmp.Close(); err != nil {
|
||||
_ = os.Remove(tmpName)
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(tmpName); err != nil {
|
||||
return err
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// MemoryStore 是线程安全的内存记录存储。
|
||||
type MemoryStore[T any] struct {
|
||||
mu sync.RWMutex
|
||||
records map[string]T
|
||||
keyFn KeyFunc[T]
|
||||
cloneFn CloneFunc[T]
|
||||
}
|
||||
|
||||
// NewMemoryStore 创建内存记录存储。
|
||||
func NewMemoryStore[T any](keyFn KeyFunc[T], cloneFn CloneFunc[T]) *MemoryStore[T] {
|
||||
if cloneFn == nil {
|
||||
cloneFn = IdentityClone[T]
|
||||
}
|
||||
return &MemoryStore[T]{
|
||||
records: make(map[string]T),
|
||||
keyFn: keyFn,
|
||||
cloneFn: cloneFn,
|
||||
}
|
||||
}
|
||||
|
||||
// Load 从内存按主键读取记录。
|
||||
func (s *MemoryStore[T]) Load(_ context.Context, key string) (T, bool, error) {
|
||||
s.mu.RLock()
|
||||
record, ok := s.records[key]
|
||||
s.mu.RUnlock()
|
||||
if !ok {
|
||||
var zero T
|
||||
return zero, false, nil
|
||||
}
|
||||
return s.cloneFn(record), true, nil
|
||||
}
|
||||
|
||||
// Check 检查上下文是否已取消。
|
||||
func (s *MemoryStore[T]) Check(ctx context.Context) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// Save 保存记录副本到内存。
|
||||
func (s *MemoryStore[T]) Save(_ context.Context, record T) error {
|
||||
key, err := RecordKey(s.keyFn, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.records[key] = s.cloneFn(record)
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes one record. It is primarily used by in-memory transactional
|
||||
// tests to restore an aggregate that did not exist before a failed commit.
|
||||
func (s *MemoryStore[T]) Delete(ctx context.Context, key string) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return ErrRecordKeyRequired
|
||||
}
|
||||
s.mu.Lock()
|
||||
delete(s.records, key)
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Snapshot 返回全部记录副本，可按 less 排序。
|
||||
func (s *MemoryStore[T]) Snapshot(less func(T, T) bool) []T {
|
||||
s.mu.RLock()
|
||||
out := make([]T, 0, len(s.records))
|
||||
for _, record := range s.records {
|
||||
out = append(out, s.cloneFn(record))
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
if less != nil {
|
||||
sortRecords(out, less)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortRecords[T any](records []T, less func(T, T) bool) {
|
||||
for i := 1; i < len(records); i++ {
|
||||
for j := i; j > 0 && less(records[j], records[j-1]); j-- {
|
||||
records[j], records[j-1] = records[j-1], records[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrObjectCodecReq 表示对象字段编解码器缺失或不合法。
|
||||
var ErrObjectCodecReq = errors.New("object field codec is required")
|
||||
|
||||
// ErrHashObjectStoreReq 表示 hash 对象存储本身缺失。
|
||||
var ErrHashObjectStoreReq = errors.New("hash object store is required")
|
||||
|
||||
// ErrObjectRecordFactoryReq 表示对象记录构造函数缺失。
|
||||
var ErrObjectRecordFactoryReq = errors.New("hash object new record function is required")
|
||||
|
||||
// ObjectFieldCodec 描述对象字段和 hash 字段之间的编解码规则。
|
||||
type ObjectFieldCodec[T any, F comparable] struct {
|
||||
Field F
|
||||
Name string
|
||||
Marshal func(T) ([]byte, error)
|
||||
Apply func(*T, []byte) error
|
||||
}
|
||||
|
||||
// HashObjectStoreOptions 配置 hash 后端对象存储。
|
||||
type HashObjectStoreOptions[T any, F comparable] struct {
|
||||
Backend HashBackend
|
||||
Fields FieldRegistry[F]
|
||||
Codecs []ObjectFieldCodec[T, F]
|
||||
Key KeyFunc[T]
|
||||
Clone CloneFunc[T]
|
||||
NewRecord func(string) T
|
||||
NormalizeKey func(string) string
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// HashObjectStore 把对象字段拆分保存到 HashBackend。
|
||||
type HashObjectStore[T any, F comparable] struct {
|
||||
backend HashBackend
|
||||
fields FieldRegistry[F]
|
||||
codecs map[F]objectFieldCodec[T]
|
||||
keyFn KeyFunc[T]
|
||||
cloneFn CloneFunc[T]
|
||||
newRecord func(string) T
|
||||
normalizeKey func(string) string
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
type objectFieldCodec[T any] struct {
|
||||
name string
|
||||
marshal func(T) ([]byte, error)
|
||||
apply func(*T, []byte) error
|
||||
}
|
||||
|
||||
// NewHashObjectStore 创建 hash 对象存储。
|
||||
func NewHashObjectStore[T any, F comparable](options HashObjectStoreOptions[T, F]) (*HashObjectStore[T, F], error) {
|
||||
if options.Backend == nil {
|
||||
return nil, ErrHashBackendRequired
|
||||
}
|
||||
if options.Key == nil {
|
||||
return nil, fmt.Errorf("%w: key function is nil", ErrRecordKeyRequired)
|
||||
}
|
||||
if options.NewRecord == nil {
|
||||
return nil, ErrObjectRecordFactoryReq
|
||||
}
|
||||
if options.Clone == nil {
|
||||
options.Clone = IdentityClone[T]
|
||||
}
|
||||
fields := options.Fields
|
||||
if len(fields.All()) == 0 && len(options.Codecs) > 0 {
|
||||
order := make([]F, 0, len(options.Codecs))
|
||||
for _, codec := range options.Codecs {
|
||||
order = append(order, codec.Field)
|
||||
}
|
||||
fields = NewFieldRegistry(order)
|
||||
}
|
||||
codecs, err := normObjCodecs(fields, options.Codecs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &HashObjectStore[T, F]{
|
||||
backend: options.Backend,
|
||||
fields: fields,
|
||||
codecs: codecs,
|
||||
keyFn: options.Key,
|
||||
cloneFn: options.Clone,
|
||||
newRecord: options.NewRecord,
|
||||
normalizeKey: options.NormalizeKey,
|
||||
ttl: options.TTL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Load 从 hash 后端读取并组装对象。
|
||||
func (s *HashObjectStore[T, F]) Load(ctx context.Context, key string) (T, bool, error) {
|
||||
var zero T
|
||||
if err := s.ensureLoadReady(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
key = normalizeLookupKey(key, s.normalizeKey)
|
||||
if key == "" {
|
||||
return zero, false, ErrRecordKeyRequired
|
||||
}
|
||||
raw, found, err := s.backend.LoadHash(ctx, key, s.fieldNames(s.fields.All()))
|
||||
if err != nil || !found {
|
||||
return zero, found, err
|
||||
}
|
||||
record, err := s.decodeRecord(key, raw)
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
return s.cloneFn(record), true, nil
|
||||
}
|
||||
|
||||
// LoadBatch 批量读取并组装对象。
|
||||
func (s *HashObjectStore[T, F]) LoadBatch(ctx context.Context, keys []string) (map[string]T, error) {
|
||||
if err := s.ensureLoadReady(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
keys = normalizeBatchKeys(keys, s.normalizeKey)
|
||||
if len(keys) == 0 {
|
||||
return map[string]T{}, ctx.Err()
|
||||
}
|
||||
if batcher, ok := s.backend.(HashBatchLoaderBackend); ok {
|
||||
fieldNames := s.fieldNames(s.fields.All())
|
||||
requests := make([]HashLoadRequest, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
requests = append(requests, HashLoadRequest{
|
||||
Key: key,
|
||||
Fields: fieldNames,
|
||||
})
|
||||
}
|
||||
results, err := batcher.LoadHashBatch(ctx, requests)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]T, len(results))
|
||||
for _, key := range keys {
|
||||
result := results[key]
|
||||
if !result.Found {
|
||||
continue
|
||||
}
|
||||
record, err := s.decodeRecord(key, result.Fields)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[key] = s.cloneFn(record)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
out := make(map[string]T, len(keys))
|
||||
for _, key := range keys {
|
||||
record, ok, err := s.Load(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
out[key] = record
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Save 保存对象的全部字段。
|
||||
func (s *HashObjectStore[T, F]) Save(ctx context.Context, record T) error {
|
||||
if err := s.ensureSaveReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
fields := s.fields.All()
|
||||
if len(fields) == 0 {
|
||||
return ErrAllFieldsRequired
|
||||
}
|
||||
return s.SaveFields(ctx, record, fields...)
|
||||
}
|
||||
|
||||
// SaveFields 保存对象的指定字段。
|
||||
func (s *HashObjectStore[T, F]) SaveFields(ctx context.Context, record T, fields ...F) error {
|
||||
if err := s.ensureSaveReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
fields = s.fields.Normalize(fields)
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
key, err := s.recordKey(record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashFields, err := s.encodeFields(record, fields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.backend.SaveHash(ctx, key, hashFields, s.ttl)
|
||||
}
|
||||
|
||||
// SaveFieldBatch 批量保存对象字段。
|
||||
func (s *HashObjectStore[T, F]) SaveFieldBatch(ctx context.Context, saves []FieldSave[T, F]) error {
|
||||
if err := s.ensureSaveReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
batches := make([]HashSaveBatch, 0, len(saves))
|
||||
for _, save := range normalizeFieldSaves(s.fields.Normalize, saves) {
|
||||
key, err := s.recordKey(save.Record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fields, err := s.encodeFields(save.Record, save.Fields)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
batches = append(batches, HashSaveBatch{Key: key, Fields: fields, TTL: s.ttl})
|
||||
}
|
||||
var err error
|
||||
batches, err = mergeHashSaveBatches(batches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(batches) == 0 {
|
||||
return nil
|
||||
}
|
||||
if batcher, ok := s.backend.(HashBatchBackend); ok {
|
||||
return batcher.SaveHashBatch(ctx, batches)
|
||||
}
|
||||
for _, batch := range batches {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.backend.SaveHash(ctx, batch.Key, batch.Fields, batch.TTL); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check 检查 hash 后端可用性。
|
||||
func (s *HashObjectStore[T, F]) Check(ctx context.Context) error {
|
||||
if err := s.ensureBaseReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
return Check(ctx, s.backend)
|
||||
}
|
||||
|
||||
// Fields 返回对象存储支持的字段列表。
|
||||
func (s *HashObjectStore[T, F]) Fields() []F {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
return s.fields.All()
|
||||
}
|
||||
|
||||
// FieldNames 返回字段对应的 hash 字段名。
|
||||
func (s *HashObjectStore[T, F]) FieldNames(fields ...F) []string {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return s.fieldNames(s.fields.All())
|
||||
}
|
||||
return s.fieldNames(s.fields.Normalize(fields))
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) recordKey(record T) (string, error) {
|
||||
key, err := RecordKey(s.keyFn, record)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key = normalizeLookupKey(key, s.normalizeKey)
|
||||
if key == "" {
|
||||
return "", ErrRecordKeyRequired
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) ensureBaseReady() error {
|
||||
if s == nil {
|
||||
return ErrHashObjectStoreReq
|
||||
}
|
||||
if s.backend == nil {
|
||||
return ErrHashBackendRequired
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) ensureLoadReady() error {
|
||||
if err := s.ensureBaseReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.newRecord == nil {
|
||||
return ErrObjectRecordFactoryReq
|
||||
}
|
||||
s.ensureClone()
|
||||
fields := s.fields.All()
|
||||
if len(fields) == 0 {
|
||||
return ErrAllFieldsRequired
|
||||
}
|
||||
return s.ensureCodecs(fields, false, true)
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) ensureSaveReady() error {
|
||||
if err := s.ensureBaseReady(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.keyFn == nil {
|
||||
return fmt.Errorf("%w: key function is nil", ErrRecordKeyRequired)
|
||||
}
|
||||
s.ensureClone()
|
||||
fields := s.fields.All()
|
||||
if len(fields) == 0 {
|
||||
return ErrAllFieldsRequired
|
||||
}
|
||||
return s.ensureCodecs(fields, true, false)
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) ensureClone() {
|
||||
if s.cloneFn == nil {
|
||||
s.cloneFn = IdentityClone[T]
|
||||
}
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) ensureCodecs(fields []F, requireMarshal bool, requireApply bool) error {
|
||||
for _, field := range fields {
|
||||
codec, ok := s.codecs[field]
|
||||
if !ok || strings.TrimSpace(codec.name) == "" {
|
||||
return fmt.Errorf("%w: %v", ErrObjectCodecReq, field)
|
||||
}
|
||||
if requireMarshal && codec.marshal == nil {
|
||||
return fmt.Errorf("%w: %v", ErrObjectCodecReq, field)
|
||||
}
|
||||
if requireApply && codec.apply == nil {
|
||||
return fmt.Errorf("%w: %v", ErrObjectCodecReq, field)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) encodeFields(record T, fields []F) (map[string][]byte, error) {
|
||||
out := make(map[string][]byte, len(fields))
|
||||
for _, field := range fields {
|
||||
codec, ok := s.codecs[field]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: %v", ErrObjectCodecReq, field)
|
||||
}
|
||||
data, err := codec.marshal(s.cloneFn(record))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal object field %q: %w", codec.name, err)
|
||||
}
|
||||
if data == nil {
|
||||
return nil, fmt.Errorf("marshal object field %q returned nil", codec.name)
|
||||
}
|
||||
out[codec.name] = append([]byte(nil), data...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) decodeRecord(key string, raw map[string][]byte) (T, error) {
|
||||
record := s.newRecord(key)
|
||||
for _, field := range s.fields.All() {
|
||||
codec := s.codecs[field]
|
||||
data, ok := raw[codec.name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if err := codec.apply(&record, append([]byte(nil), data...)); err != nil {
|
||||
return record, fmt.Errorf("apply object field %q: %w", codec.name, err)
|
||||
}
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (s *HashObjectStore[T, F]) fieldNames(fields []F) []string {
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
if codec, ok := s.codecs[field]; ok {
|
||||
out = append(out, codec.name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normObjCodecs[T any, F comparable](fields FieldRegistry[F], codecs []ObjectFieldCodec[T, F]) (map[F]objectFieldCodec[T], error) {
|
||||
if len(fields.All()) == 0 {
|
||||
return nil, ErrAllFieldsRequired
|
||||
}
|
||||
out := make(map[F]objectFieldCodec[T], len(codecs))
|
||||
names := make(map[string]F, len(codecs))
|
||||
seenFields := make(map[F]struct{}, len(codecs))
|
||||
for _, codec := range codecs {
|
||||
if !fields.IsKnown(codec.Field) {
|
||||
return nil, fmt.Errorf("%w: unknown field %v", ErrObjectCodecReq, codec.Field)
|
||||
}
|
||||
if _, exists := seenFields[codec.Field]; exists {
|
||||
return nil, fmt.Errorf("duplicate object field codec: %v", codec.Field)
|
||||
}
|
||||
seenFields[codec.Field] = struct{}{}
|
||||
name := strings.TrimSpace(codec.Name)
|
||||
if name == "" || codec.Marshal == nil || codec.Apply == nil {
|
||||
return nil, fmt.Errorf("%w: %v", ErrObjectCodecReq, codec.Field)
|
||||
}
|
||||
if existing, exists := names[name]; exists && existing != codec.Field {
|
||||
return nil, fmt.Errorf("duplicate object field name %q", name)
|
||||
}
|
||||
names[name] = codec.Field
|
||||
out[codec.Field] = objectFieldCodec[T]{
|
||||
name: name,
|
||||
marshal: codec.Marshal,
|
||||
apply: codec.Apply,
|
||||
}
|
||||
}
|
||||
for _, field := range fields.All() {
|
||||
if _, ok := out[field]; !ok {
|
||||
return nil, fmt.Errorf("%w: %v", ErrObjectCodecReq, field)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/gomodule/redigo/redis"
|
||||
)
|
||||
|
||||
// ErrRedisExecutorClosed 表示 Redis 执行器已关闭或未初始化。
|
||||
var ErrRedisExecutorClosed = errors.New("redis executor is closed")
|
||||
|
||||
// closeRedisConnErr 在 Redis 操作本身成功时保留连接关闭错误，避免池连接异常被静默吞掉。
|
||||
func closeRedisConnErr(conn redis.Conn, err *error) {
|
||||
if conn == nil || err == nil {
|
||||
return
|
||||
}
|
||||
if closeErr := conn.Close(); closeErr != nil && *err == nil {
|
||||
*err = closeErr
|
||||
}
|
||||
}
|
||||
|
||||
// RedigoOptions 配置 redigo 连接池。
|
||||
type RedigoOptions struct {
|
||||
Address string
|
||||
Password string
|
||||
DB int
|
||||
PoolSize int
|
||||
MaxActive int
|
||||
IdleTimeout time.Duration
|
||||
Timeout time.Duration
|
||||
ConnectTimeout time.Duration
|
||||
ReadTimeout time.Duration
|
||||
WriteTimeout time.Duration
|
||||
TestOnBorrowAfter time.Duration
|
||||
}
|
||||
|
||||
// RedisPoolStats 描述 Redis 连接池状态。
|
||||
type RedisPoolStats struct {
|
||||
Address string `json:"address"`
|
||||
DB int `json:"db"`
|
||||
MaxIdle int `json:"max_idle"`
|
||||
MaxActive int `json:"max_active"`
|
||||
IdleTimeout time.Duration `json:"idle_timeout"`
|
||||
ActiveCount int `json:"active_count"`
|
||||
IdleCount int `json:"idle_count"`
|
||||
WaitCount int64 `json:"wait_count"`
|
||||
WaitDuration time.Duration `json:"wait_duration"`
|
||||
}
|
||||
|
||||
// RedigoExecutor 用 redigo 连接池实现 RedisExecutor。
|
||||
type RedigoExecutor struct {
|
||||
options RedigoOptions
|
||||
pool *redis.Pool
|
||||
}
|
||||
|
||||
// NormalizeRedigoOptions 补齐 redigo 默认连接参数。
|
||||
func NormalizeRedigoOptions(options RedigoOptions) RedigoOptions {
|
||||
if options.Address == "" {
|
||||
options.Address = "127.0.0.1:6379"
|
||||
}
|
||||
if options.PoolSize <= 0 {
|
||||
options.PoolSize = 8
|
||||
}
|
||||
if options.MaxActive <= 0 {
|
||||
options.MaxActive = options.PoolSize * 2
|
||||
}
|
||||
if options.IdleTimeout <= 0 {
|
||||
options.IdleTimeout = time.Minute
|
||||
}
|
||||
if options.Timeout <= 0 {
|
||||
options.Timeout = 2 * time.Second
|
||||
}
|
||||
if options.ConnectTimeout <= 0 {
|
||||
options.ConnectTimeout = options.Timeout
|
||||
}
|
||||
if options.ReadTimeout <= 0 {
|
||||
options.ReadTimeout = options.Timeout
|
||||
}
|
||||
if options.WriteTimeout <= 0 {
|
||||
options.WriteTimeout = options.Timeout
|
||||
}
|
||||
if options.TestOnBorrowAfter < 0 {
|
||||
options.TestOnBorrowAfter = 0
|
||||
}
|
||||
if options.TestOnBorrowAfter == 0 {
|
||||
options.TestOnBorrowAfter = time.Minute
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
// NewRedigoExecutor 创建 redigo Redis 执行器。
|
||||
func NewRedigoExecutor(options RedigoOptions) *RedigoExecutor {
|
||||
options = NormalizeRedigoOptions(options)
|
||||
return &RedigoExecutor{options: options, pool: &redis.Pool{
|
||||
MaxIdle: options.PoolSize,
|
||||
MaxActive: options.MaxActive,
|
||||
IdleTimeout: options.IdleTimeout,
|
||||
Wait: true,
|
||||
Dial: func() (redis.Conn, error) {
|
||||
dialOptions := []redis.DialOption{
|
||||
redis.DialConnectTimeout(options.ConnectTimeout),
|
||||
redis.DialReadTimeout(options.ReadTimeout),
|
||||
redis.DialWriteTimeout(options.WriteTimeout),
|
||||
redis.DialDatabase(options.DB),
|
||||
}
|
||||
if options.Password != "" {
|
||||
dialOptions = append(dialOptions, redis.DialPassword(options.Password))
|
||||
}
|
||||
return redis.Dial("tcp", options.Address, dialOptions...)
|
||||
},
|
||||
TestOnBorrow: func(conn redis.Conn, lastUsed time.Time) error {
|
||||
if time.Since(lastUsed) < options.TestOnBorrowAfter {
|
||||
return nil
|
||||
}
|
||||
_, err := conn.Do("PING")
|
||||
return err
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// Do 执行单条 Redis 命令。
|
||||
func (e *RedigoExecutor) Do(ctx context.Context, command string, args ...any) (reply any, err error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e == nil || e.pool == nil {
|
||||
return nil, ErrRedisExecutorClosed
|
||||
}
|
||||
conn, err := e.pool.GetContext(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeRedisConnErr(conn, &err)
|
||||
return conn.Do(command, args...)
|
||||
}
|
||||
|
||||
// DoBatch 使用同一连接批量发送 Redis 命令。
|
||||
func (e *RedigoExecutor) DoBatch(ctx context.Context, commands []RedisCommand) (err error) {
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if e == nil || e.pool == nil {
|
||||
return ErrRedisExecutorClosed
|
||||
}
|
||||
conn, err := e.pool.GetContext(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeRedisConnErr(conn, &err)
|
||||
return RunRedigoBatch(ctx, conn, commands)
|
||||
}
|
||||
|
||||
// RunRedigoBatch 在已获取的 redigo 连接上发送批量命令，并在 Flush 后完整接收响应。
|
||||
func RunRedigoBatch(ctx context.Context, conn redis.Conn, commands []RedisCommand) error {
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
if conn == nil {
|
||||
return ErrRedisExecutorClosed
|
||||
}
|
||||
sent := 0
|
||||
for _, command := range commands {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if command.Name == "" {
|
||||
continue
|
||||
}
|
||||
if err := conn.Send(command.Name, command.Args...); err != nil {
|
||||
return err
|
||||
}
|
||||
sent++
|
||||
}
|
||||
if sent == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := conn.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Flush 成功后命令可能已经被 Redis 执行；此时必须排空响应，不能再让调用方 ctx
|
||||
// 中途打断 Receive，否则异步刷盘会把已提交写误判成失败并重复入队。
|
||||
for idx := 0; idx < sent; idx++ {
|
||||
if _, err := conn.Receive(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PoolStats 返回连接池统计信息。
|
||||
func (e *RedigoExecutor) PoolStats() RedisPoolStats {
|
||||
if e == nil || e.pool == nil {
|
||||
return RedisPoolStats{}
|
||||
}
|
||||
stats := e.pool.Stats()
|
||||
return RedisPoolStats{
|
||||
Address: e.options.Address,
|
||||
DB: e.options.DB,
|
||||
MaxIdle: e.options.PoolSize,
|
||||
MaxActive: e.options.MaxActive,
|
||||
IdleTimeout: e.options.IdleTimeout,
|
||||
ActiveCount: stats.ActiveCount,
|
||||
IdleCount: stats.IdleCount,
|
||||
WaitCount: stats.WaitCount,
|
||||
WaitDuration: stats.WaitDuration,
|
||||
}
|
||||
}
|
||||
|
||||
// Close 关闭 Redis 连接池。
|
||||
func (e *RedigoExecutor) Close() error {
|
||||
if e == nil || e.pool == nil {
|
||||
return nil
|
||||
}
|
||||
return e.pool.Close()
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RedisExecutor 是最小 Redis 命令执行接口。
|
||||
type RedisExecutor interface {
|
||||
Do(context.Context, string, ...any) (any, error)
|
||||
}
|
||||
|
||||
// RedisBatchExecutor 支持批量发送 Redis 命令。
|
||||
type RedisBatchExecutor interface {
|
||||
DoBatch(context.Context, []RedisCommand) error
|
||||
}
|
||||
|
||||
// RedisCommand 描述一条 Redis 命令。
|
||||
type RedisCommand struct {
|
||||
Name string
|
||||
Args []any
|
||||
}
|
||||
|
||||
// RedisHashBatch 描述一次 Redis hash 字段保存项。
|
||||
type RedisHashBatch struct {
|
||||
Key string
|
||||
Fields map[string][]byte
|
||||
TTL time.Duration
|
||||
}
|
||||
|
||||
// SaveRedisHashFields 保存单个 Redis hash 的字段。
|
||||
func SaveRedisHashFields(ctx context.Context, executor RedisExecutor, key string, fields map[string][]byte, ttl time.Duration) error {
|
||||
return SaveRedisHashFieldBatches(ctx, executor, []RedisHashBatch{{
|
||||
Key: key,
|
||||
Fields: fields,
|
||||
TTL: ttl,
|
||||
}})
|
||||
}
|
||||
|
||||
// SaveRedisHashFieldBatches 批量保存 Redis hash 字段。
|
||||
func SaveRedisHashFieldBatches(ctx context.Context, executor RedisExecutor, batches []RedisHashBatch) error {
|
||||
if executor == nil {
|
||||
return errors.New("redis executor is nil")
|
||||
}
|
||||
commands, err := redisHashCmds(batches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
return DoRedisBatch(ctx, executor, commands)
|
||||
}
|
||||
|
||||
// DoRedisBatch 执行一组 Redis 命令，优先使用批量执行器。
|
||||
func DoRedisBatch(ctx context.Context, executor RedisExecutor, commands []RedisCommand) error {
|
||||
if executor == nil {
|
||||
return errors.New("redis executor is nil")
|
||||
}
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if batcher, ok := executor.(RedisBatchExecutor); ok {
|
||||
return batcher.DoBatch(ctx, cloneRedisCommands(commands))
|
||||
}
|
||||
for _, cmd := range commands {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := executor.Do(ctx, cmd.Name, cmd.Args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func redisHashCmds(batches []RedisHashBatch) ([]RedisCommand, error) {
|
||||
if len(batches) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
merged, err := mergeRedisBatches(batches)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
commands := make([]RedisCommand, 0, len(merged)*2)
|
||||
for _, batch := range merged {
|
||||
key := strings.TrimSpace(batch.Key)
|
||||
values := batch.Fields
|
||||
names := make([]string, 0, len(values))
|
||||
for name := range values {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
hsetArgs := make([]any, 0, 1+len(names)*2)
|
||||
hsetArgs = append(hsetArgs, key)
|
||||
for _, name := range names {
|
||||
hsetArgs = append(hsetArgs, name, append([]byte(nil), values[name]...))
|
||||
}
|
||||
commands = append(commands, RedisCommand{Name: "HSET", Args: hsetArgs})
|
||||
ttlSeconds := int(batch.TTL.Seconds())
|
||||
if len(names) > 0 && ttlSeconds > 0 {
|
||||
commands = append(commands, RedisCommand{
|
||||
Name: "EXPIRE",
|
||||
Args: []any{key, ttlSeconds},
|
||||
})
|
||||
}
|
||||
}
|
||||
return commands, nil
|
||||
}
|
||||
|
||||
func mergeRedisBatches(batches []RedisHashBatch) ([]RedisHashBatch, error) {
|
||||
converted := make([]HashSaveBatch, 0, len(batches))
|
||||
for _, batch := range batches {
|
||||
converted = append(converted, HashSaveBatch(batch))
|
||||
}
|
||||
merged, err := mergeHashSaveBatches(converted)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]RedisHashBatch, 0, len(merged))
|
||||
for _, batch := range merged {
|
||||
out = append(out, RedisHashBatch(batch))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cloneRedisCommands(commands []RedisCommand) []RedisCommand {
|
||||
if len(commands) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]RedisCommand, len(commands))
|
||||
for i, command := range commands {
|
||||
args := make([]any, len(command.Args))
|
||||
for j, arg := range command.Args {
|
||||
args[j] = cloneRedisCommandArg(arg)
|
||||
}
|
||||
out[i] = RedisCommand{
|
||||
Name: command.Name,
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneRedisCommandArg(arg any) any {
|
||||
if data, ok := arg.([]byte); ok {
|
||||
return append([]byte(nil), data...)
|
||||
}
|
||||
return arg
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RedisHashBackend 用 Redis hash 实现 HashBackend。
|
||||
type RedisHashBackend struct {
|
||||
Executor RedisExecutor
|
||||
}
|
||||
|
||||
// NewRedisHashBackend 创建 Redis hash 后端。
|
||||
func NewRedisHashBackend(executor RedisExecutor) RedisHashBackend {
|
||||
return RedisHashBackend{Executor: executor}
|
||||
}
|
||||
|
||||
// LoadHash 从 Redis hash 读取字段。
|
||||
func (b RedisHashBackend) LoadHash(ctx context.Context, key string, fields []string) (map[string][]byte, bool, error) {
|
||||
if b.Executor == nil {
|
||||
return nil, false, errors.New("redis executor is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" {
|
||||
return nil, false, ErrRecordKeyRequired
|
||||
}
|
||||
fields = normHashFields(fields)
|
||||
if len(fields) == 0 {
|
||||
reply, err := b.Executor.Do(ctx, "HGETALL", key)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return redisHGetAllFields(reply)
|
||||
}
|
||||
args := make([]any, 0, 1+len(fields))
|
||||
args = append(args, key)
|
||||
for _, field := range fields {
|
||||
args = append(args, field)
|
||||
}
|
||||
reply, err := b.Executor.Do(ctx, "HMGET", args...)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
values, err := redisArray(reply)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(values) != len(fields) {
|
||||
return nil, false, fmt.Errorf("redis HMGET returned %d values for %d fields", len(values), len(fields))
|
||||
}
|
||||
out := make(map[string][]byte, len(fields))
|
||||
for i, value := range values {
|
||||
data, ok, err := redisBulkBytes(value)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if ok {
|
||||
out[fields[i]] = data
|
||||
}
|
||||
}
|
||||
return out, len(out) > 0, nil
|
||||
}
|
||||
|
||||
// SaveHash 保存 Redis hash 字段。
|
||||
func (b RedisHashBackend) SaveHash(ctx context.Context, key string, fields map[string][]byte, ttl time.Duration) error {
|
||||
return SaveRedisHashFields(ctx, b.Executor, key, fields, ttl)
|
||||
}
|
||||
|
||||
// SaveHashBatch 批量保存 Redis hash 字段。
|
||||
func (b RedisHashBackend) SaveHashBatch(ctx context.Context, batches []HashSaveBatch) error {
|
||||
merged, err := mergeHashSaveBatches(batches)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
redisBatches := make([]RedisHashBatch, 0, len(merged))
|
||||
for _, batch := range merged {
|
||||
redisBatches = append(redisBatches, RedisHashBatch(batch))
|
||||
}
|
||||
return SaveRedisHashFieldBatches(ctx, b.Executor, redisBatches)
|
||||
}
|
||||
|
||||
// Check 通过 PING 检查 Redis 连接。
|
||||
func (b RedisHashBackend) Check(ctx context.Context) error {
|
||||
if b.Executor == nil {
|
||||
return errors.New("redis executor is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
_, err := b.Executor.Do(ctx, "PING")
|
||||
return err
|
||||
}
|
||||
|
||||
func redisHGetAllFields(reply any) (map[string][]byte, bool, error) {
|
||||
values, err := redisArray(reply)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if len(values)%2 != 0 {
|
||||
return nil, false, fmt.Errorf("redis HGETALL returned odd value count %d", len(values))
|
||||
}
|
||||
out := make(map[string][]byte, len(values)/2)
|
||||
for i := 0; i < len(values); i += 2 {
|
||||
name, ok, err := redisBulkString(values[i])
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
data, ok, err := redisBulkBytes(values[i+1])
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if ok {
|
||||
out[name] = data
|
||||
}
|
||||
}
|
||||
return out, len(out) > 0, nil
|
||||
}
|
||||
|
||||
func redisArray(reply any) ([]any, error) {
|
||||
switch value := reply.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case []any:
|
||||
return value, nil
|
||||
case [][]byte:
|
||||
out := make([]any, len(value))
|
||||
for i, item := range value {
|
||||
if item == nil {
|
||||
out[i] = nil
|
||||
} else {
|
||||
out[i] = append([]byte(nil), item...)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
case []string:
|
||||
out := make([]any, len(value))
|
||||
for i, item := range value {
|
||||
out[i] = item
|
||||
}
|
||||
return out, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected redis array reply %T", reply)
|
||||
}
|
||||
}
|
||||
|
||||
func redisBulkString(value any) (string, bool, error) {
|
||||
data, ok, err := redisBulkBytes(value)
|
||||
if err != nil || !ok {
|
||||
return "", ok, err
|
||||
}
|
||||
return string(data), true, nil
|
||||
}
|
||||
|
||||
func redisBulkBytes(value any) ([]byte, bool, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return nil, false, nil
|
||||
case []byte:
|
||||
return append([]byte(nil), v...), true, nil
|
||||
case string:
|
||||
return []byte(v), true, nil
|
||||
default:
|
||||
return nil, false, fmt.Errorf("unexpected redis bulk reply %T", value)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrRecordKeyRequired 表示记录缺少可用主键。
|
||||
ErrRecordKeyRequired = errors.New("record key is required")
|
||||
// ErrAsyncStoreClosed 表示异步存储已关闭。
|
||||
ErrAsyncStoreClosed = errors.New("async store is closed")
|
||||
// ErrAllFieldsRequired 表示调用方需要提供完整字段集合。
|
||||
ErrAllFieldsRequired = errors.New("all fields function is required")
|
||||
)
|
||||
|
||||
// KeyFunc 从记录中提取稳定主键。
|
||||
type KeyFunc[T any] func(T) string
|
||||
|
||||
// CloneFunc 拷贝记录，避免缓存和调用方共享可变对象。
|
||||
type CloneFunc[T any] func(T) T
|
||||
|
||||
// Store 是最小记录存取接口。
|
||||
type Store[T any] interface {
|
||||
Load(context.Context, string) (T, bool, error)
|
||||
Save(context.Context, T) error
|
||||
}
|
||||
|
||||
// Checker 是存储预检接口。
|
||||
type Checker interface {
|
||||
Check(context.Context) error
|
||||
}
|
||||
|
||||
// FieldStore 支持按字段局部保存记录。
|
||||
type FieldStore[T any, F comparable] interface {
|
||||
Store[T]
|
||||
SaveFields(context.Context, T, ...F) error
|
||||
}
|
||||
|
||||
// FieldSave 表示一次局部字段保存请求。
|
||||
type FieldSave[T any, F comparable] struct {
|
||||
Record T
|
||||
Fields []F
|
||||
}
|
||||
|
||||
// BatchFieldStore 支持批量局部字段保存。
|
||||
type BatchFieldStore[T any, F comparable] interface {
|
||||
SaveFieldBatch(context.Context, []FieldSave[T, F]) error
|
||||
}
|
||||
|
||||
// SaveFieldsFunc 是可注入的局部保存函数。
|
||||
type SaveFieldsFunc[T any, F comparable] func(context.Context, Store[T], T, ...F) error
|
||||
|
||||
// SaveFieldBatchFunc 是可注入的批量局部保存函数。
|
||||
type SaveFieldBatchFunc[T any, F comparable] func(context.Context, Store[T], []FieldSave[T, F]) error
|
||||
|
||||
// Expirer 是支持设置记录过期时间的存储接口。
|
||||
type Expirer interface {
|
||||
Expire(context.Context, string, time.Duration) error
|
||||
}
|
||||
|
||||
// ExpireFunc 是可注入的过期设置函数。
|
||||
type ExpireFunc[T any] func(context.Context, Store[T], string, time.Duration) error
|
||||
|
||||
// IdentityClone 原样返回记录，适用于不可变值或调用方自行拷贝的场景。
|
||||
func IdentityClone[T any](record T) T {
|
||||
return record
|
||||
}
|
||||
|
||||
// RecordKey 提取并校验记录主键。
|
||||
func RecordKey[T any](keyFn KeyFunc[T], record T) (string, error) {
|
||||
if keyFn == nil {
|
||||
return "", fmt.Errorf("%w: key function is nil", ErrRecordKeyRequired)
|
||||
}
|
||||
key := strings.TrimSpace(keyFn(record))
|
||||
if key == "" {
|
||||
return "", ErrRecordKeyRequired
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// SaveFields 按字段保存；底层不支持字段保存时退化为整条保存。
|
||||
func SaveFields[T any, F comparable](ctx context.Context, store Store[T], record T, normalize func([]F) []F, fields ...F) error {
|
||||
if store == nil {
|
||||
return errors.New("store is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
if normalize != nil {
|
||||
fields = normalize(fields)
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return nil
|
||||
}
|
||||
if fieldStore, ok := store.(FieldStore[T, F]); ok {
|
||||
return fieldStore.SaveFields(ctx, record, fields...)
|
||||
}
|
||||
return store.Save(ctx, record)
|
||||
}
|
||||
|
||||
// SaveFieldBatch 批量保存字段；底层不支持批量时逐条退化。
|
||||
func SaveFieldBatch[T any, F comparable](ctx context.Context, store Store[T], normalize func([]F) []F, saves []FieldSave[T, F]) error {
|
||||
if store == nil {
|
||||
return errors.New("store is nil")
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
batch := normalizeFieldSaves(normalize, saves)
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
if batchStore, ok := store.(BatchFieldStore[T, F]); ok {
|
||||
return batchStore.SaveFieldBatch(ctx, batch)
|
||||
}
|
||||
for _, save := range batch {
|
||||
if err := SaveFields(ctx, store, save.Record, normalize, save.Fields...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Expire 在存储支持过期能力时设置记录 TTL。
|
||||
func Expire(ctx context.Context, store any, key string, ttl time.Duration) error {
|
||||
if ttl <= 0 {
|
||||
return nil
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
expirer, ok := store.(Expirer)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return expirer.Expire(ctx, key, ttl)
|
||||
}
|
||||
|
||||
func normalizeFieldSaves[T any, F comparable](normalize func([]F) []F, saves []FieldSave[T, F]) []FieldSave[T, F] {
|
||||
if len(saves) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]FieldSave[T, F], 0, len(saves))
|
||||
for _, save := range saves {
|
||||
fields := append([]F(nil), save.Fields...)
|
||||
if normalize != nil {
|
||||
fields = normalize(fields)
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, FieldSave[T, F]{
|
||||
Record: save.Record,
|
||||
Fields: fields,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CloseOrFlush 优先关闭存储，否则刷新待写队列。
|
||||
func CloseOrFlush(ctx context.Context, store any) error {
|
||||
if closer, ok := store.(interface {
|
||||
Close(context.Context) error
|
||||
}); ok {
|
||||
return closer.Close(ctx)
|
||||
}
|
||||
if flusher, ok := store.(interface {
|
||||
Flush(context.Context) error
|
||||
}); ok {
|
||||
return flusher.Flush(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check 对实现 Checker 的对象执行预检。
|
||||
func Check(ctx context.Context, target any) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if checker, ok := target.(Checker); ok {
|
||||
return checker.Check(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func contextOrBackground(ctx context.Context) context.Context {
|
||||
if ctx == nil {
|
||||
return context.Background()
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
Reference in new issue
Block a user