DNF90 服务端与启动器完整源码（含 GM 工具功能增强）
服务端（Go）： - GM 塞背包/带属性装备发放：在线角色推送 op14 增量物品行，背包实时刷新，无需回角色选择界面 - GM 改金币/改等级：去掉在线拒绝，落库后推送 op14 钱包/op37 等级状态，在线实时刷新 - 等级端点统一切换到新版 GMSetCharacterLevel（自带在线刷新） - 无限/锁死疲劳模式 + 燃烧疲劳：每房间消耗 2 点、打怪与通关经验双倍 - 燃烧疲劳开关时给所有在线角色推送系统提示（op12 mode 0） - 深渊史诗掉率：GM 全局倍率 + 城镇增益药剂窗口倍率，真实接入掉落结算 启动器（Python/JS）： - 标题栏自绘、窗口 1204x717、禁止拖拽改大小、仅最小化/关闭 - GM 工具：疲劳与燃烧、深渊史诗掉率、拍卖行上架（单一定价对齐） - 拍卖行上架前端对齐服务端单一价格语义，修复 400 报错
This commit is contained in:
hanfuit committed
commit
645b64fe21
4388 files changed
+1405457
No files matched your search
@@ -0,0 +1,101 @@
|
||||
package readmodel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var ErrAdminBadRequest = errors.New("read model admin bad request")
|
||||
|
||||
type AdminError struct {
|
||||
Kind error
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e AdminError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
func (e AdminError) Unwrap() error {
|
||||
return e.Kind
|
||||
}
|
||||
|
||||
func BadRequest(message string) error {
|
||||
return AdminError{Kind: ErrAdminBadRequest, Message: strings.TrimSpace(message)}
|
||||
}
|
||||
|
||||
func IsBadRequest(err error) bool {
|
||||
return errors.Is(err, ErrAdminBadRequest)
|
||||
}
|
||||
|
||||
type AdminSearchFunc func(context.Context, url.Values) (any, error)
|
||||
type AdminRebuildFunc func(context.Context, url.Values) (any, error)
|
||||
|
||||
type AdminModel struct {
|
||||
Name string
|
||||
Search AdminSearchFunc
|
||||
Rebuild AdminRebuildFunc
|
||||
}
|
||||
|
||||
type AdminRegistry struct {
|
||||
mu sync.RWMutex
|
||||
models map[string]AdminModel
|
||||
}
|
||||
|
||||
func NewAdminRegistry() *AdminRegistry {
|
||||
return &AdminRegistry{
|
||||
models: make(map[string]AdminModel),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *AdminRegistry) Register(model AdminModel) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
name := normAdminModel(model.Name)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
model.Name = name
|
||||
r.mu.Lock()
|
||||
r.models[name] = model
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *AdminRegistry) Search(ctx context.Context, name string, values url.Values) (any, bool, error) {
|
||||
model, ok := r.model(name)
|
||||
if !ok || model.Search == nil {
|
||||
return nil, ok, nil
|
||||
}
|
||||
out, err := model.Search(ctx, values)
|
||||
return out, true, err
|
||||
}
|
||||
|
||||
func (r *AdminRegistry) Rebuild(ctx context.Context, name string, values url.Values) (any, bool, error) {
|
||||
model, ok := r.model(name)
|
||||
if !ok || model.Rebuild == nil {
|
||||
return nil, ok, nil
|
||||
}
|
||||
out, err := model.Rebuild(ctx, values)
|
||||
return out, true, err
|
||||
}
|
||||
|
||||
func (r *AdminRegistry) model(name string) (AdminModel, bool) {
|
||||
if r == nil {
|
||||
return AdminModel{}, false
|
||||
}
|
||||
name = normAdminModel(name)
|
||||
r.mu.RLock()
|
||||
model, ok := r.models[name]
|
||||
r.mu.RUnlock()
|
||||
return model, ok
|
||||
}
|
||||
|
||||
func normAdminModel(name string) string {
|
||||
name = strings.TrimSpace(name)
|
||||
name = strings.Trim(name, "/")
|
||||
return name
|
||||
}
|
||||
@@ -0,0 +1,261 @@
|
||||
package readmodel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type RebuildOptions struct {
|
||||
AfterID string `json:"after_id,omitempty"`
|
||||
BatchSize int `json:"batch_size,omitempty"`
|
||||
MaxRecords int `json:"max_records,omitempty"`
|
||||
}
|
||||
|
||||
type RebuildResult struct {
|
||||
Scanned int `json:"scanned"`
|
||||
Rebuilt int `json:"rebuilt"`
|
||||
ActiveRebuilt int `json:"active_rebuilt,omitempty"`
|
||||
LastID string `json:"last_id,omitempty"`
|
||||
Complete bool `json:"complete"`
|
||||
}
|
||||
|
||||
type StreamRebuildResult struct {
|
||||
Scanned int `json:"scanned"`
|
||||
Rebuilt int `json:"rebuilt"`
|
||||
ActiveRebuilt int `json:"active_rebuilt,omitempty"`
|
||||
LastID string `json:"last_id,omitempty"`
|
||||
Complete bool `json:"complete"`
|
||||
Pages int `json:"pages"`
|
||||
}
|
||||
|
||||
type RebuildRunner[T any, V any] struct {
|
||||
Scan func(context.Context, string, int) ([]T, error)
|
||||
Flush func(context.Context) error
|
||||
ActiveSnapshot func() []T
|
||||
RecordID func(T) string
|
||||
Build func(T) V
|
||||
Save func(context.Context, V) error
|
||||
}
|
||||
|
||||
type RebuildProgressFunc func(RebuildResult) error
|
||||
|
||||
func Rebuild[T any, V any](ctx context.Context, options RebuildOptions, runner RebuildRunner[T, V]) (RebuildResult, error) {
|
||||
if err := runner.validate(); err != nil {
|
||||
return RebuildResult{}, err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
options = normRebuildOpts(options)
|
||||
|
||||
after := strings.TrimSpace(options.AfterID)
|
||||
result := RebuildResult{}
|
||||
activeAfterID := after
|
||||
rebuiltActive := make(map[string]struct{})
|
||||
if runner.Flush != nil {
|
||||
if err := runner.Flush(ctx); err != nil {
|
||||
return result, fmt.Errorf("flush source before read model rebuild: %w", err)
|
||||
}
|
||||
}
|
||||
activeRecords, activeByRecordID := snapshotActive(runner)
|
||||
|
||||
for result.Scanned < options.MaxRecords {
|
||||
limit := options.BatchSize
|
||||
if remaining := options.MaxRecords - result.Scanned; remaining < limit {
|
||||
limit = remaining
|
||||
}
|
||||
records, err := runner.Scan(ctx, after, limit)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if len(records) == 0 {
|
||||
result.Complete = true
|
||||
return result, rebuildActive(ctx, runner, &result, activeRecords, activeAfterID, "", rebuiltActive)
|
||||
}
|
||||
for _, record := range records {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
recordID := strings.TrimSpace(runner.RecordID(record))
|
||||
if recordID == "" {
|
||||
return result, errors.New("read model rebuild source returned blank id")
|
||||
}
|
||||
result.Scanned++
|
||||
result.LastID = recordID
|
||||
after = recordID
|
||||
summaryRecord := record
|
||||
if active, ok := activeByRecordID[recordID]; ok {
|
||||
summaryRecord = active
|
||||
result.ActiveRebuilt++
|
||||
rebuiltActive[recordID] = struct{}{}
|
||||
}
|
||||
if err := runner.Save(ctx, runner.Build(summaryRecord)); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Rebuilt++
|
||||
}
|
||||
if len(records) < limit {
|
||||
result.Complete = true
|
||||
return result, rebuildActive(ctx, runner, &result, activeRecords, activeAfterID, "", rebuiltActive)
|
||||
}
|
||||
}
|
||||
return result, rebuildActive(ctx, runner, &result, activeRecords, activeAfterID, result.LastID, rebuiltActive)
|
||||
}
|
||||
|
||||
func StreamRebuild[T any, V any](ctx context.Context, options RebuildOptions, runner RebuildRunner[T, V], progress RebuildProgressFunc) (StreamRebuildResult, error) {
|
||||
if err := runner.validate(); err != nil {
|
||||
return StreamRebuildResult{}, err
|
||||
}
|
||||
ctx = contextOrBackground(ctx)
|
||||
options = normStreamOpts(options)
|
||||
|
||||
after := strings.TrimSpace(options.AfterID)
|
||||
remaining := options.MaxRecords
|
||||
streamRunner := runner
|
||||
if runner.ActiveSnapshot != nil {
|
||||
activeSnapshot := runner.ActiveSnapshot()
|
||||
streamRunner.ActiveSnapshot = func() []T {
|
||||
return append([]T(nil), activeSnapshot...)
|
||||
}
|
||||
}
|
||||
var result StreamRebuildResult
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
pageLimit := options.BatchSize
|
||||
if remaining > 0 && remaining < pageLimit {
|
||||
pageLimit = remaining
|
||||
}
|
||||
page, err := Rebuild(ctx, RebuildOptions{
|
||||
AfterID: after,
|
||||
BatchSize: options.BatchSize,
|
||||
MaxRecords: pageLimit,
|
||||
}, streamRunner)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if page.Scanned == 0 && page.Rebuilt == 0 && page.Complete {
|
||||
result.Complete = true
|
||||
return result, nil
|
||||
}
|
||||
result.Pages++
|
||||
result.Scanned += page.Scanned
|
||||
result.Rebuilt += page.Rebuilt
|
||||
result.ActiveRebuilt += page.ActiveRebuilt
|
||||
if page.LastID != "" {
|
||||
result.LastID = page.LastID
|
||||
after = page.LastID
|
||||
}
|
||||
result.Complete = page.Complete
|
||||
if progress != nil {
|
||||
if err := progress(page); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
if page.Complete {
|
||||
return result, nil
|
||||
}
|
||||
if page.Scanned == 0 || page.LastID == "" {
|
||||
return result, errors.New("read model stream rebuild made no cursor progress")
|
||||
}
|
||||
if remaining > 0 {
|
||||
remaining -= page.Scanned
|
||||
if remaining <= 0 {
|
||||
result.Complete = false
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
streamRunner.Flush = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (r RebuildRunner[T, V]) validate() error {
|
||||
if r.Scan == nil {
|
||||
return errors.New("read model rebuild scan function is required")
|
||||
}
|
||||
if r.RecordID == nil {
|
||||
return errors.New("read model rebuild record id function is required")
|
||||
}
|
||||
if r.Build == nil {
|
||||
return errors.New("read model rebuild build function is required")
|
||||
}
|
||||
if r.Save == nil {
|
||||
return errors.New("read model rebuild save function is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normRebuildOpts(options RebuildOptions) RebuildOptions {
|
||||
options.AfterID = strings.TrimSpace(options.AfterID)
|
||||
if options.BatchSize <= 0 {
|
||||
options.BatchSize = 100
|
||||
}
|
||||
if options.BatchSize > 1000 {
|
||||
options.BatchSize = 1000
|
||||
}
|
||||
if options.MaxRecords <= 0 {
|
||||
options.MaxRecords = options.BatchSize
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
func normStreamOpts(options RebuildOptions) RebuildOptions {
|
||||
options.AfterID = strings.TrimSpace(options.AfterID)
|
||||
if options.BatchSize <= 0 {
|
||||
options.BatchSize = 100
|
||||
}
|
||||
if options.BatchSize > 1000 {
|
||||
options.BatchSize = 1000
|
||||
}
|
||||
if options.MaxRecords < 0 {
|
||||
options.MaxRecords = 0
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
func rebuildActive[T any, V any](ctx context.Context, runner RebuildRunner[T, V], result *RebuildResult, records []T, afterID, throughID string, skip map[string]struct{}) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, record := range records {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
recordID := strings.TrimSpace(runner.RecordID(record))
|
||||
if recordID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := skip[recordID]; ok {
|
||||
continue
|
||||
}
|
||||
if afterID != "" && recordID <= afterID {
|
||||
continue
|
||||
}
|
||||
if throughID != "" && recordID > throughID {
|
||||
continue
|
||||
}
|
||||
if err := runner.Save(ctx, runner.Build(record)); err != nil {
|
||||
return err
|
||||
}
|
||||
result.Rebuilt++
|
||||
result.ActiveRebuilt++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func snapshotActive[T any, V any](runner RebuildRunner[T, V]) ([]T, map[string]T) {
|
||||
byID := make(map[string]T)
|
||||
if runner.ActiveSnapshot == nil {
|
||||
return nil, byID
|
||||
}
|
||||
records := runner.ActiveSnapshot()
|
||||
for _, record := range records {
|
||||
recordID := strings.TrimSpace(runner.RecordID(record))
|
||||
if recordID == "" {
|
||||
continue
|
||||
}
|
||||
byID[recordID] = record
|
||||
}
|
||||
return records, byID
|
||||
}
|
||||
@@ -0,0 +1,476 @@
|
||||
package readmodel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gomodule/redigo/redis"
|
||||
"longheng.io/server/internal/platform/db"
|
||||
)
|
||||
|
||||
const (
|
||||
redisScanCount = 256
|
||||
|
||||
RedisCleanupMissingScript = `
|
||||
if redis.call("HGET", KEYS[2], ARGV[1]) == false then
|
||||
return redis.call("SREM", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0`
|
||||
|
||||
RedisCleanupCorruptScript = `
|
||||
local current = redis.call("HGET", KEYS[2], ARGV[1])
|
||||
if current == ARGV[3] then
|
||||
redis.call("DEL", KEYS[2])
|
||||
return redis.call("SREM", KEYS[1], ARGV[2])
|
||||
end
|
||||
return 0`
|
||||
|
||||
RedisDeleteSecondaryIndexIfPrimaryScript = `
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0`
|
||||
)
|
||||
|
||||
var ErrCorruptRecord = errors.New("corrupt read model record")
|
||||
|
||||
type RedisStoreOptions[T any, Q any] struct {
|
||||
Model ModelOptions[T, Q]
|
||||
Executor db.RedisExecutor
|
||||
KeyPrefix string
|
||||
HashField string
|
||||
PrimarySetName string
|
||||
SecondaryIndexName string
|
||||
TTL time.Duration
|
||||
// ScanCount 控制 Search 无索引兜底扫描时每次 SSCAN 建议返回的主键数。
|
||||
ScanCount int
|
||||
Encode func(T) ([]byte, error)
|
||||
Decode func(string, []byte) (T, error)
|
||||
}
|
||||
|
||||
type RedisStore[T any, Q any] struct {
|
||||
options ModelOptions[T, Q]
|
||||
executor db.RedisExecutor
|
||||
keyspace redisKeyspace
|
||||
ttl time.Duration
|
||||
scanCount int
|
||||
encode func(T) ([]byte, error)
|
||||
decode func(string, []byte) (T, error)
|
||||
}
|
||||
|
||||
type redisKeyspace struct {
|
||||
prefix string
|
||||
hashField string
|
||||
primarySetName string
|
||||
secondaryIndexName string
|
||||
}
|
||||
|
||||
type cleanupCandidate struct {
|
||||
primaryID string
|
||||
data []byte
|
||||
}
|
||||
|
||||
func NewRedisStore[T any, Q any](options RedisStoreOptions[T, Q]) (*RedisStore[T, Q], error) {
|
||||
if err := options.Model.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if options.Executor == nil {
|
||||
return nil, errors.New("redis executor is nil")
|
||||
}
|
||||
keyspace := redisKeyspace{
|
||||
prefix: strings.TrimSpace(options.KeyPrefix),
|
||||
hashField: strings.TrimSpace(options.HashField),
|
||||
primarySetName: strings.TrimSpace(options.PrimarySetName),
|
||||
secondaryIndexName: strings.TrimSpace(options.SecondaryIndexName),
|
||||
}
|
||||
if keyspace.prefix == "" {
|
||||
keyspace.prefix = "readmodel"
|
||||
}
|
||||
if keyspace.hashField == "" {
|
||||
keyspace.hashField = "summary"
|
||||
}
|
||||
if keyspace.primarySetName == "" {
|
||||
keyspace.primarySetName = "ids"
|
||||
}
|
||||
if keyspace.secondaryIndexName == "" {
|
||||
keyspace.secondaryIndexName = "secondary"
|
||||
}
|
||||
scanCount := options.ScanCount
|
||||
if scanCount <= 0 {
|
||||
scanCount = redisScanCount
|
||||
}
|
||||
encode := options.Encode
|
||||
if encode == nil {
|
||||
encode = func(record T) ([]byte, error) {
|
||||
return json.Marshal(record)
|
||||
}
|
||||
}
|
||||
decode := options.Decode
|
||||
if decode == nil {
|
||||
decode = func(_ string, data []byte) (T, error) {
|
||||
var record T
|
||||
if err := json.Unmarshal(data, &record); err != nil {
|
||||
return record, fmt.Errorf("%w: %w", ErrCorruptRecord, err)
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
}
|
||||
return &RedisStore[T, Q]{
|
||||
options: options.Model,
|
||||
executor: options.Executor,
|
||||
keyspace: keyspace,
|
||||
ttl: options.TTL,
|
||||
scanCount: scanCount,
|
||||
encode: encode,
|
||||
decode: decode,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) SetExecutor(executor db.RedisExecutor) {
|
||||
s.executor = executor
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) Save(ctx context.Context, record T) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
record = s.options.normalize(record)
|
||||
primaryID := s.options.primaryID(record)
|
||||
if primaryID == "" {
|
||||
return nil
|
||||
}
|
||||
old, err := s.recordForSave(ctx, primaryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := s.encode(record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
secondaryID := s.options.secondaryID(record)
|
||||
oldSecondaryID := s.options.secondaryID(old)
|
||||
|
||||
commands := []db.RedisCommand{
|
||||
{Name: "HSET", Args: []any{s.SummaryKey(primaryID), s.keyspace.hashField, data}},
|
||||
{Name: "SADD", Args: []any{s.PrimaryIDsKey(), primaryID}},
|
||||
}
|
||||
if secondaryID != "" {
|
||||
commands = append(commands, db.RedisCommand{Name: "SET", Args: []any{s.SecondaryKey(secondaryID), primaryID}})
|
||||
}
|
||||
if oldSecondaryID != "" && oldSecondaryID != secondaryID {
|
||||
commands = append(commands, db.RedisCommand{Name: "EVAL", Args: []any{RedisDeleteSecondaryIndexIfPrimaryScript, 1, s.SecondaryKey(oldSecondaryID), primaryID}})
|
||||
}
|
||||
if ttlSeconds := int(s.ttl.Seconds()); ttlSeconds > 0 {
|
||||
commands = append(commands, db.RedisCommand{Name: "EXPIRE", Args: []any{s.SummaryKey(primaryID), ttlSeconds}})
|
||||
commands = append(commands, db.RedisCommand{Name: "EXPIRE", Args: []any{s.PrimaryIDsKey(), ttlSeconds}})
|
||||
if secondaryID != "" {
|
||||
commands = append(commands, db.RedisCommand{Name: "EXPIRE", Args: []any{s.SecondaryKey(secondaryID), ttlSeconds}})
|
||||
}
|
||||
}
|
||||
return db.DoRedisBatch(ctx, s.executor, commands)
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) Get(ctx context.Context, primaryID string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
primaryID = strings.TrimSpace(primaryID)
|
||||
if primaryID == "" {
|
||||
return zero, false, nil
|
||||
}
|
||||
data, ok, err := s.recordData(ctx, primaryID)
|
||||
if err != nil || !ok {
|
||||
return zero, ok, err
|
||||
}
|
||||
record, err := s.decodeRecord(primaryID, data)
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
return record, true, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) GetBySecondaryID(ctx context.Context, secondaryID string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
secondaryID = strings.TrimSpace(secondaryID)
|
||||
if secondaryID == "" {
|
||||
return zero, false, nil
|
||||
}
|
||||
reply, err := s.executor.Do(ctx, "GET", s.SecondaryKey(secondaryID))
|
||||
primaryID, err := redis.String(reply, err)
|
||||
if errors.Is(err, redis.ErrNil) {
|
||||
return zero, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
record, ok, err := s.Get(ctx, primaryID)
|
||||
if err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
if !ok || s.options.secondaryID(record) != secondaryID {
|
||||
s.deleteStaleSecondary(ctx, secondaryID, primaryID)
|
||||
return zero, false, nil
|
||||
}
|
||||
return record, true, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) ListByPrimaryIDs(ctx context.Context, primaryIDs []string) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
return s.listByPrimaryIDs(ctx, primaryIDs, false)
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) listByPrimaryIDs(ctx context.Context, primaryIDs []string, cleanupMissing bool) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
primaryIDs = normalizeIDs(primaryIDs)
|
||||
out := make([]T, 0, len(primaryIDs))
|
||||
cleanup := make([]cleanupCandidate, 0)
|
||||
for _, primaryID := range primaryIDs {
|
||||
data, ok, err := s.recordData(ctx, primaryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
if cleanupMissing {
|
||||
cleanup = append(cleanup, cleanupCandidate{primaryID: primaryID})
|
||||
}
|
||||
continue
|
||||
}
|
||||
record, err := s.decodeRecord(primaryID, data)
|
||||
if err != nil {
|
||||
if cleanupMissing && errors.Is(err, ErrCorruptRecord) {
|
||||
cleanup = append(cleanup, cleanupCandidate{primaryID: primaryID, data: append([]byte(nil), data...)})
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, record)
|
||||
}
|
||||
if len(cleanup) > 0 {
|
||||
s.removePrimaryIDsBest(ctx, cleanup)
|
||||
}
|
||||
s.options.sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) ListBySecondaryIDs(ctx context.Context, secondaryIDs []string) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
secondaryIDs = normalizeIDs(secondaryIDs)
|
||||
out := make([]T, 0, len(secondaryIDs))
|
||||
for _, secondaryID := range secondaryIDs {
|
||||
record, ok, err := s.GetBySecondaryID(ctx, secondaryID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
out = append(out, record)
|
||||
}
|
||||
}
|
||||
s.options.sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) Search(ctx context.Context, query Q) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if ids := s.options.queryPrimaryIDs(query); len(ids) > 0 {
|
||||
records, err := s.ListByPrimaryIDs(ctx, ids)
|
||||
return s.options.filter(records, query), err
|
||||
}
|
||||
if ids := s.options.querySecondaryIDs(query); len(ids) > 0 {
|
||||
records, err := s.ListBySecondaryIDs(ctx, ids)
|
||||
return s.options.filter(records, query), err
|
||||
}
|
||||
return s.scanSearch(ctx, query)
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) scanSearch(ctx context.Context, query Q) ([]T, error) {
|
||||
limit := s.options.queryLimit(query)
|
||||
stopEarly := limit > 0 && s.options.Less == nil
|
||||
out := make([]T, 0)
|
||||
cursor := "0"
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := s.executor.Do(ctx, "SSCAN", s.PrimaryIDsKey(), cursor, "COUNT", s.scanCount)
|
||||
next, primaryIDs, err := parseSSCAN(reply, err)
|
||||
if errors.Is(err, redis.ErrNil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records, err := s.listByPrimaryIDs(ctx, primaryIDs, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, record := range records {
|
||||
candidate := s.options.clone(record)
|
||||
if !s.options.match(candidate, query) {
|
||||
continue
|
||||
}
|
||||
out = append(out, candidate)
|
||||
if stopEarly && len(out) >= limit {
|
||||
return s.finishSearch(out, query), nil
|
||||
}
|
||||
}
|
||||
if next == "0" {
|
||||
break
|
||||
}
|
||||
cursor = next
|
||||
}
|
||||
return s.finishSearch(out, query), nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) finishSearch(records []T, query Q) []T {
|
||||
s.options.sort(records)
|
||||
if limit := s.options.queryLimit(query); limit > 0 && len(records) > limit {
|
||||
return records[:limit]
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func parseSSCAN(reply any, err error) (string, []string, error) {
|
||||
values, err := redis.Values(reply, err)
|
||||
if errors.Is(err, redis.ErrNil) {
|
||||
return "", nil, err
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if len(values) != 2 {
|
||||
return "", nil, fmt.Errorf("unexpected redis SSCAN response with %d parts", len(values))
|
||||
}
|
||||
cursor, err := redis.String(values[0], nil)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
primaryIDs, err := redis.Strings(values[1], nil)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
cursor = strings.TrimSpace(cursor)
|
||||
if cursor == "" {
|
||||
cursor = "0"
|
||||
}
|
||||
return cursor, normalizeIDs(primaryIDs), nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) Check(ctx context.Context) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
reply, err := s.executor.Do(ctx, "PING")
|
||||
pong, err := redis.String(reply, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.ToUpper(pong) != "PONG" {
|
||||
return fmt.Errorf("unexpected redis ping response %q", pong)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) Close(context.Context) error {
|
||||
if s == nil || s.executor == nil {
|
||||
return nil
|
||||
}
|
||||
if closer, ok := s.executor.(interface{ Close() error }); ok {
|
||||
return closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) SummaryKey(primaryID string) string {
|
||||
return s.keyspace.prefix + ":summary:" + encodeRedisKeyPart(strings.TrimSpace(primaryID))
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) SecondaryKey(secondaryID string) string {
|
||||
return s.keyspace.prefix + ":secondary:" + encodeRedisKeyPart(s.keyspace.secondaryIndexName) + ":" + encodeRedisKeyPart(strings.TrimSpace(secondaryID))
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) PrimaryIDsKey() string {
|
||||
return s.keyspace.prefix + ":" + s.keyspace.primarySetName
|
||||
}
|
||||
|
||||
func encodeRedisKeyPart(value string) string {
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(value))
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) recordForSave(ctx context.Context, primaryID string) (T, error) {
|
||||
var zero T
|
||||
data, ok, err := s.recordData(ctx, primaryID)
|
||||
if err != nil || !ok {
|
||||
return zero, err
|
||||
}
|
||||
record, err := s.decodeRecord(primaryID, data)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) recordData(ctx context.Context, primaryID string) ([]byte, bool, error) {
|
||||
reply, err := s.executor.Do(ctx, "HGET", s.SummaryKey(primaryID), s.keyspace.hashField)
|
||||
data, err := redis.Bytes(reply, err)
|
||||
if errors.Is(err, redis.ErrNil) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return data, true, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) decodeRecord(primaryID string, data []byte) (T, error) {
|
||||
var zero T
|
||||
record, err := s.decode(primaryID, data)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
record = s.options.normalize(record)
|
||||
return s.options.clone(record), nil
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) deleteStaleSecondary(ctx context.Context, secondaryID, primaryID string) {
|
||||
secondaryID = strings.TrimSpace(secondaryID)
|
||||
primaryID = strings.TrimSpace(primaryID)
|
||||
if secondaryID == "" || primaryID == "" {
|
||||
return
|
||||
}
|
||||
_, _ = s.executor.Do(ctx, "EVAL", RedisDeleteSecondaryIndexIfPrimaryScript, 1, s.SecondaryKey(secondaryID), primaryID)
|
||||
}
|
||||
|
||||
func (s *RedisStore[T, Q]) removePrimaryIDsBest(ctx context.Context, candidates []cleanupCandidate) {
|
||||
seen := make(map[string]struct{}, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
primaryID := strings.TrimSpace(candidate.primaryID)
|
||||
if primaryID == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[primaryID]; ok {
|
||||
continue
|
||||
}
|
||||
seen[primaryID] = struct{}{}
|
||||
if len(candidate.data) == 0 {
|
||||
_, _ = s.executor.Do(ctx, "EVAL", RedisCleanupMissingScript, 2, s.PrimaryIDsKey(), s.SummaryKey(primaryID), s.keyspace.hashField, primaryID)
|
||||
continue
|
||||
}
|
||||
_, _ = s.executor.Do(ctx, "EVAL", RedisCleanupCorruptScript, 2, s.PrimaryIDsKey(), s.SummaryKey(primaryID), s.keyspace.hashField, primaryID, candidate.data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
package readmodel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var ErrSecondaryConflict = errors.New("read model secondary id conflict")
|
||||
|
||||
type Store[T any, Q any] interface {
|
||||
Save(context.Context, T) error
|
||||
Get(context.Context, string) (T, bool, error)
|
||||
GetBySecondaryID(context.Context, string) (T, bool, error)
|
||||
ListByPrimaryIDs(context.Context, []string) ([]T, error)
|
||||
ListBySecondaryIDs(context.Context, []string) ([]T, error)
|
||||
Search(context.Context, Q) ([]T, error)
|
||||
}
|
||||
|
||||
type ModelOptions[T any, Q any] struct {
|
||||
PrimaryID func(T) string
|
||||
SecondaryID func(T) string
|
||||
Normalize func(T) T
|
||||
Clone func(T) T
|
||||
Match func(T, Q) bool
|
||||
Less func(T, T) bool
|
||||
QueryPrimaryIDs func(Q) []string
|
||||
QuerySecondaryIDs func(Q) []string
|
||||
QueryLimit func(Q) int
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) validate() error {
|
||||
if o.PrimaryID == nil {
|
||||
return errors.New("read model primary id function is required")
|
||||
}
|
||||
if o.SecondaryID == nil {
|
||||
return errors.New("read model secondary id function is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) normalize(record T) T {
|
||||
if o.Normalize != nil {
|
||||
return o.Normalize(record)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) clone(record T) T {
|
||||
if o.Clone != nil {
|
||||
return o.Clone(record)
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) primaryID(record T) string {
|
||||
return strings.TrimSpace(o.PrimaryID(record))
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) secondaryID(record T) string {
|
||||
return strings.TrimSpace(o.SecondaryID(record))
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) queryPrimaryIDs(query Q) []string {
|
||||
if o.QueryPrimaryIDs == nil {
|
||||
return nil
|
||||
}
|
||||
return normalizeIDs(o.QueryPrimaryIDs(query))
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) querySecondaryIDs(query Q) []string {
|
||||
if o.QuerySecondaryIDs == nil {
|
||||
return nil
|
||||
}
|
||||
return normalizeIDs(o.QuerySecondaryIDs(query))
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) queryLimit(query Q) int {
|
||||
if o.QueryLimit == nil {
|
||||
return 0
|
||||
}
|
||||
limit := o.QueryLimit(query)
|
||||
if limit < 0 {
|
||||
return 0
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) match(record T, query Q) bool {
|
||||
if o.Match == nil {
|
||||
return true
|
||||
}
|
||||
return o.Match(record, query)
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) sort(records []T) {
|
||||
if o.Less == nil {
|
||||
return
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return o.Less(records[i], records[j])
|
||||
})
|
||||
}
|
||||
|
||||
func (o ModelOptions[T, Q]) filter(records []T, query Q) []T {
|
||||
out := make([]T, 0, len(records))
|
||||
for _, record := range records {
|
||||
candidate := o.clone(record)
|
||||
if o.match(candidate, query) {
|
||||
out = append(out, candidate)
|
||||
}
|
||||
}
|
||||
o.sort(out)
|
||||
if limit := o.queryLimit(query); limit > 0 && len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type MemoryStore[T any, Q any] struct {
|
||||
options ModelOptions[T, Q]
|
||||
mu sync.RWMutex
|
||||
byPrimary map[string]T
|
||||
secondary map[string]string
|
||||
}
|
||||
|
||||
func NewMemoryStore[T any, Q any](options ModelOptions[T, Q]) (*MemoryStore[T, Q], error) {
|
||||
if err := options.validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MemoryStore[T, Q]{
|
||||
options: options,
|
||||
byPrimary: make(map[string]T),
|
||||
secondary: make(map[string]string),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) Save(ctx context.Context, record T) error {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
record = s.options.normalize(record)
|
||||
primaryID := s.options.primaryID(record)
|
||||
if primaryID == "" {
|
||||
return nil
|
||||
}
|
||||
secondaryID := s.options.secondaryID(record)
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if secondaryID != "" {
|
||||
if owner, ok := s.secondary[secondaryID]; ok && owner != primaryID {
|
||||
return ErrSecondaryConflict
|
||||
}
|
||||
}
|
||||
if old, ok := s.byPrimary[primaryID]; ok {
|
||||
oldSecondaryID := s.options.secondaryID(old)
|
||||
if oldSecondaryID != "" && oldSecondaryID != secondaryID {
|
||||
delete(s.secondary, oldSecondaryID)
|
||||
}
|
||||
}
|
||||
s.byPrimary[primaryID] = s.options.clone(record)
|
||||
if secondaryID != "" {
|
||||
s.secondary[secondaryID] = primaryID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) Get(ctx context.Context, primaryID string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
primaryID = strings.TrimSpace(primaryID)
|
||||
if primaryID == "" {
|
||||
return zero, false, nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
record, ok := s.byPrimary[primaryID]
|
||||
s.mu.RUnlock()
|
||||
return s.options.clone(record), ok, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) GetBySecondaryID(ctx context.Context, secondaryID string) (T, bool, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
var zero T
|
||||
if err := ctx.Err(); err != nil {
|
||||
return zero, false, err
|
||||
}
|
||||
secondaryID = strings.TrimSpace(secondaryID)
|
||||
if secondaryID == "" {
|
||||
return zero, false, nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
primaryID := s.secondary[secondaryID]
|
||||
record, ok := s.byPrimary[primaryID]
|
||||
s.mu.RUnlock()
|
||||
if !ok || s.options.secondaryID(record) != secondaryID {
|
||||
return zero, false, nil
|
||||
}
|
||||
return s.options.clone(record), true, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) ListByPrimaryIDs(ctx context.Context, primaryIDs []string) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
primaryIDs = normalizeIDs(primaryIDs)
|
||||
s.mu.RLock()
|
||||
out := make([]T, 0, len(primaryIDs))
|
||||
for _, primaryID := range primaryIDs {
|
||||
if record, ok := s.byPrimary[primaryID]; ok {
|
||||
out = append(out, s.options.clone(record))
|
||||
}
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
s.options.sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) ListBySecondaryIDs(ctx context.Context, secondaryIDs []string) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secondaryIDs = normalizeIDs(secondaryIDs)
|
||||
s.mu.RLock()
|
||||
out := make([]T, 0, len(secondaryIDs))
|
||||
for _, secondaryID := range secondaryIDs {
|
||||
if record, ok := s.byPrimary[s.secondary[secondaryID]]; ok && s.options.secondaryID(record) == secondaryID {
|
||||
out = append(out, s.options.clone(record))
|
||||
}
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
s.options.sort(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore[T, Q]) Search(ctx context.Context, query Q) ([]T, error) {
|
||||
ctx = contextOrBackground(ctx)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ids := s.options.queryPrimaryIDs(query); len(ids) > 0 {
|
||||
records, err := s.ListByPrimaryIDs(ctx, ids)
|
||||
return s.options.filter(records, query), err
|
||||
}
|
||||
if ids := s.options.querySecondaryIDs(query); len(ids) > 0 {
|
||||
records, err := s.ListBySecondaryIDs(ctx, ids)
|
||||
return s.options.filter(records, query), err
|
||||
}
|
||||
|
||||
s.mu.RLock()
|
||||
candidates := make([]T, 0, len(s.byPrimary))
|
||||
for _, record := range s.byPrimary {
|
||||
candidates = append(candidates, record)
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
return s.options.filter(candidates, query), nil
|
||||
}
|
||||
|
||||
func normalizeIDs(ids []string) []string {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(ids))
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
for _, id := range ids {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contextOrBackground(ctx context.Context) context.Context {
|
||||
if ctx == nil {
|
||||
return context.Background()
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
Reference in new issue
Block a user