DNF90 服务端与启动器完整源码（含 GM 工具功能增强）
服务端（Go）： - GM 塞背包/带属性装备发放：在线角色推送 op14 增量物品行，背包实时刷新，无需回角色选择界面 - GM 改金币/改等级：去掉在线拒绝，落库后推送 op14 钱包/op37 等级状态，在线实时刷新 - 等级端点统一切换到新版 GMSetCharacterLevel（自带在线刷新） - 无限/锁死疲劳模式 + 燃烧疲劳：每房间消耗 2 点、打怪与通关经验双倍 - 燃烧疲劳开关时给所有在线角色推送系统提示（op12 mode 0） - 深渊史诗掉率：GM 全局倍率 + 城镇增益药剂窗口倍率，真实接入掉落结算 启动器（Python/JS）： - 标题栏自绘、窗口 1204x717、禁止拖拽改大小、仅最小化/关闭 - GM 工具：疲劳与燃烧、深渊史诗掉率、拍卖行上架（单一定价对齐） - 拍卖行上架前端对齐服务端单一价格语义，修复 400 报错
This commit is contained in:
hanfuit committed
commit
645b64fe21
4388 files changed
+1405457
No files matched your search
@@ -0,0 +1,515 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var errXLSXPartNotFound = errors.New("xlsx part not found")
|
||||
|
||||
type GenerateOptions struct {
|
||||
Input string `json:"input"`
|
||||
Output string `json:"output"`
|
||||
Format string `json:"format,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
ManifestOut string `json:"manifest_out,omitempty"`
|
||||
Pretty bool `json:"pretty,omitempty"`
|
||||
}
|
||||
|
||||
type GeneratedTable struct {
|
||||
Name string `json:"name"`
|
||||
Source string `json:"source"`
|
||||
Output string `json:"output"`
|
||||
Rows int `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Checksum string `json:"checksum"`
|
||||
}
|
||||
|
||||
type GenerateReport struct {
|
||||
Input string `json:"input"`
|
||||
Output string `json:"output"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Tables []GeneratedTable `json:"tables"`
|
||||
Manifest Manifest `json:"manifest"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
}
|
||||
|
||||
func GenerateTables(ctx context.Context, options GenerateOptions) (GenerateReport, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
options.Input = strings.TrimSpace(options.Input)
|
||||
options.Output = strings.TrimSpace(options.Output)
|
||||
options.Format = strings.ToLower(strings.TrimSpace(options.Format))
|
||||
options.Version = strings.TrimSpace(options.Version)
|
||||
options.ManifestOut = strings.TrimSpace(options.ManifestOut)
|
||||
if options.Input == "" {
|
||||
return GenerateReport{}, ErrDirectoryRequired
|
||||
}
|
||||
if options.Output == "" {
|
||||
return GenerateReport{}, fmt.Errorf("data table output is required")
|
||||
}
|
||||
info, err := os.Stat(options.Input)
|
||||
if err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
sources, err := generatorSources(options.Input, info)
|
||||
if err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
return GenerateReport{}, fmt.Errorf("no data table source files found in %s", options.Input)
|
||||
}
|
||||
if err := os.MkdirAll(options.Output, 0o750); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
report := GenerateReport{
|
||||
Input: options.Input,
|
||||
Output: options.Output,
|
||||
Version: options.Version,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
}
|
||||
tableSources := make(map[string]string, len(sources))
|
||||
for _, source := range sources {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
tableName := tableNameFromFile(source)
|
||||
tableKey := strings.ToLower(tableName)
|
||||
if previous, ok := tableSources[tableKey]; ok {
|
||||
return GenerateReport{}, fmt.Errorf("duplicate generated data table name %q from %s and %s", tableName, previous, source)
|
||||
}
|
||||
tableSources[tableKey] = source
|
||||
rows, err := loadGeneratedRows(source, options.Format)
|
||||
if err != nil {
|
||||
return GenerateReport{}, fmt.Errorf("generate table %s: %w", source, err)
|
||||
}
|
||||
data, err := marshalGeneratedRows(rows, options.Pretty)
|
||||
if err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
outPath := filepath.Join(options.Output, tableName+".json")
|
||||
if err := os.WriteFile(outPath, data, 0o600); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
checksum := sha256.Sum256(data)
|
||||
report.Tables = append(report.Tables, GeneratedTable{
|
||||
Name: tableName,
|
||||
Source: source,
|
||||
Output: outPath,
|
||||
Rows: len(rows),
|
||||
Bytes: int64(len(data)),
|
||||
Checksum: hex.EncodeToString(checksum[:]),
|
||||
})
|
||||
}
|
||||
sort.Slice(report.Tables, func(i, j int) bool { return report.Tables[i].Name < report.Tables[j].Name })
|
||||
registry := NewRegistry(options.Output, options.Version)
|
||||
if err := registry.Load(ctx); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
report.Manifest = registry.Manifest()
|
||||
if options.ManifestOut != "" {
|
||||
manifestData, err := json.MarshalIndent(report.Manifest, "", " ")
|
||||
if err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(options.ManifestOut), 0o750); err != nil && filepath.Dir(options.ManifestOut) != "." {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
if err := os.WriteFile(options.ManifestOut, append(manifestData, '\n'), 0o600); err != nil {
|
||||
return GenerateReport{}, err
|
||||
}
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func generatorSources(input string, info os.FileInfo) ([]string, error) {
|
||||
if !info.IsDir() {
|
||||
if genExtSupported(input) {
|
||||
return []string{input}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported data table source %s", input)
|
||||
}
|
||||
var sources []string
|
||||
if err := filepath.WalkDir(input, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(path), ".") {
|
||||
return nil
|
||||
}
|
||||
if genExtSupported(path) {
|
||||
sources = append(sources, path)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(sources)
|
||||
return sources, nil
|
||||
}
|
||||
|
||||
func genExtSupported(path string) bool {
|
||||
switch strings.ToLower(filepath.Ext(path)) {
|
||||
case ".json", ".csv", ".xlsx":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func loadGeneratedRows(path, format string) ([]map[string]any, error) {
|
||||
if format == "" || format == "auto" {
|
||||
format = strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
|
||||
}
|
||||
switch format {
|
||||
case "json":
|
||||
return loadGeneratedJSON(path)
|
||||
case "csv":
|
||||
return loadGeneratedCSV(path)
|
||||
case "xlsx":
|
||||
return loadGeneratedXLSX(path)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported data table format %q", format)
|
||||
}
|
||||
}
|
||||
|
||||
func loadGeneratedJSON(path string) ([]map[string]any, error) {
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []map[string]any
|
||||
if err := json.Unmarshal(data, &rows); err == nil {
|
||||
return rows, nil
|
||||
}
|
||||
var wrapped struct {
|
||||
Items []map[string]any `json:"items"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &wrapped); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wrapped.Items == nil {
|
||||
return nil, fmt.Errorf("json table must be an array or object with items")
|
||||
}
|
||||
return wrapped.Items, nil
|
||||
}
|
||||
|
||||
func loadGeneratedCSV(path string) ([]map[string]any, error) {
|
||||
file, err := os.Open(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
reader := csv.NewReader(file)
|
||||
reader.TrimLeadingSpace = true
|
||||
reader.FieldsPerRecord = -1
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return recordsToRows(records)
|
||||
}
|
||||
|
||||
func loadGeneratedXLSX(path string) ([]map[string]any, error) {
|
||||
reader, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
shared, err := readXLSXStrings(&reader.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sheet, err := readZipFile(&reader.Reader, "xl/worksheets/sheet1.xml")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records, err := parseXLSXSheet(sheet, shared)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return recordsToRows(records)
|
||||
}
|
||||
|
||||
func recordsToRows(records [][]string) ([]map[string]any, error) {
|
||||
if len(records) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
headers := normalizeHeaders(records[0])
|
||||
if len(headers) == 0 {
|
||||
return nil, fmt.Errorf("data table header row is empty")
|
||||
}
|
||||
rows := make([]map[string]any, 0, len(records)-1)
|
||||
for _, record := range records[1:] {
|
||||
if emptyRecord(record) {
|
||||
continue
|
||||
}
|
||||
row := make(map[string]any, len(headers))
|
||||
for idx, header := range headers {
|
||||
value := ""
|
||||
if idx < len(record) {
|
||||
value = strings.TrimSpace(record[idx])
|
||||
}
|
||||
row[header] = inferGeneratedValue(value)
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func normalizeHeaders(headers []string) []string {
|
||||
out := make([]string, 0, len(headers))
|
||||
seen := make(map[string]struct{}, len(headers))
|
||||
for idx, header := range headers {
|
||||
header = strings.TrimSpace(header)
|
||||
if header == "" {
|
||||
header = fmt.Sprintf("col_%d", idx+1)
|
||||
}
|
||||
base := header
|
||||
for suffix := 2; ; suffix++ {
|
||||
if _, ok := seen[header]; !ok {
|
||||
break
|
||||
}
|
||||
header = fmt.Sprintf("%s_%d", base, suffix)
|
||||
}
|
||||
seen[header] = struct{}{}
|
||||
out = append(out, header)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func emptyRecord(record []string) bool {
|
||||
for _, value := range record {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func inferGeneratedValue(value string) any {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
switch strings.ToLower(value) {
|
||||
case "true", "false":
|
||||
parsed, _ := strconv.ParseBool(strings.ToLower(value))
|
||||
return parsed
|
||||
}
|
||||
if parsed, err := strconv.ParseInt(value, 10, 64); err == nil {
|
||||
return parsed
|
||||
}
|
||||
if parsed, err := strconv.ParseFloat(value, 64); err == nil {
|
||||
return parsed
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func marshalGeneratedRows(rows []map[string]any, pretty bool) ([]byte, error) {
|
||||
var data []byte
|
||||
var err error
|
||||
if pretty {
|
||||
data, err = json.MarshalIndent(rows, "", " ")
|
||||
} else {
|
||||
data, err = json.Marshal(rows)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(data, '\n'), nil
|
||||
}
|
||||
|
||||
func tableNameFromFile(path string) string {
|
||||
name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "table"
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func readZipFile(reader *zip.Reader, name string) ([]byte, error) {
|
||||
for _, file := range reader.File {
|
||||
if filepath.ToSlash(file.Name) != name {
|
||||
continue
|
||||
}
|
||||
rc, err := file.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rc.Close() }()
|
||||
return io.ReadAll(rc)
|
||||
}
|
||||
return nil, fmt.Errorf("%w: %s", errXLSXPartNotFound, name)
|
||||
}
|
||||
|
||||
func readXLSXStrings(reader *zip.Reader) ([]string, error) {
|
||||
data, err := readZipFile(reader, "xl/sharedStrings.xml")
|
||||
if err != nil {
|
||||
if errors.Is(err, errXLSXPartNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
decoder := xml.NewDecoder(bytes.NewReader(data))
|
||||
var values []string
|
||||
var current strings.Builder
|
||||
inText := false
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch t := token.(type) {
|
||||
case xml.StartElement:
|
||||
if t.Name.Local == "si" {
|
||||
current.Reset()
|
||||
}
|
||||
if t.Name.Local == "t" {
|
||||
inText = true
|
||||
}
|
||||
case xml.EndElement:
|
||||
if t.Name.Local == "t" {
|
||||
inText = false
|
||||
}
|
||||
if t.Name.Local == "si" {
|
||||
values = append(values, current.String())
|
||||
}
|
||||
case xml.CharData:
|
||||
if inText {
|
||||
current.Write([]byte(t))
|
||||
}
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func parseXLSXSheet(data []byte, shared []string) ([][]string, error) {
|
||||
decoder := xml.NewDecoder(bytes.NewReader(data))
|
||||
var records [][]string
|
||||
var row []string
|
||||
var cellRef string
|
||||
var cellType string
|
||||
var value strings.Builder
|
||||
inValue := false
|
||||
inInlineText := false
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch t := token.(type) {
|
||||
case xml.StartElement:
|
||||
switch t.Name.Local {
|
||||
case "row":
|
||||
row = nil
|
||||
case "c":
|
||||
cellRef = ""
|
||||
cellType = ""
|
||||
value.Reset()
|
||||
for _, attr := range t.Attr {
|
||||
switch attr.Name.Local {
|
||||
case "r":
|
||||
cellRef = attr.Value
|
||||
case "t":
|
||||
cellType = attr.Value
|
||||
}
|
||||
}
|
||||
case "v":
|
||||
inValue = true
|
||||
case "t":
|
||||
if cellType == "inlineStr" {
|
||||
inInlineText = true
|
||||
}
|
||||
}
|
||||
case xml.EndElement:
|
||||
switch t.Name.Local {
|
||||
case "v":
|
||||
inValue = false
|
||||
case "t":
|
||||
inInlineText = false
|
||||
case "c":
|
||||
text := decodeXLSXCell(value.String(), cellType, shared)
|
||||
idx := xlsxCellIndex(cellRef)
|
||||
if idx < 0 {
|
||||
idx = len(row)
|
||||
}
|
||||
for len(row) <= idx {
|
||||
row = append(row, "")
|
||||
}
|
||||
row[idx] = text
|
||||
case "row":
|
||||
records = append(records, append([]string(nil), row...))
|
||||
}
|
||||
case xml.CharData:
|
||||
if inValue || inInlineText {
|
||||
value.Write([]byte(t))
|
||||
}
|
||||
}
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func decodeXLSXCell(raw, cellType string, shared []string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if cellType != "s" {
|
||||
return raw
|
||||
}
|
||||
idx, err := strconv.Atoi(raw)
|
||||
if err != nil || idx < 0 || idx >= len(shared) {
|
||||
return raw
|
||||
}
|
||||
return shared[idx]
|
||||
}
|
||||
|
||||
func xlsxCellIndex(ref string) int {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return -1
|
||||
}
|
||||
idx := 0
|
||||
found := false
|
||||
for _, r := range ref {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
idx = idx*26 + int(r-'A'+1)
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
if r >= 'a' && r <= 'z' {
|
||||
idx = idx*26 + int(r-'a'+1)
|
||||
found = true
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
return -1
|
||||
}
|
||||
return idx - 1
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const DefaultPageLimit = 100
|
||||
|
||||
type PageOptions struct {
|
||||
Offset int `json:"offset,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
Cursor string `json:"cursor,omitempty"`
|
||||
}
|
||||
|
||||
type Page[T any] struct {
|
||||
Table Table `json:"table"`
|
||||
Manifest Manifest `json:"manifest"`
|
||||
Rows []T `json:"rows,omitempty"`
|
||||
Offset int `json:"offset"`
|
||||
Limit int `json:"limit"`
|
||||
Total int `json:"total"`
|
||||
NextCursor string `json:"next_cursor,omitempty"`
|
||||
HasMore bool `json:"has_more"`
|
||||
}
|
||||
|
||||
func LoadRowsPage[T any](ctx context.Context, source RowSource, name string, options PageOptions) (Page[T], error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return Page[T]{}, err
|
||||
}
|
||||
rows, err := LoadRows[T](source, name)
|
||||
if err != nil {
|
||||
return Page[T]{}, err
|
||||
}
|
||||
return PageRows(rows, options)
|
||||
}
|
||||
|
||||
func PageRows[T any](set RowSet[T], options PageOptions) (Page[T], error) {
|
||||
offset, limit, err := normalizePageOptions(options, len(set.Rows))
|
||||
if err != nil {
|
||||
return Page[T]{}, err
|
||||
}
|
||||
end := offset + limit
|
||||
if end > len(set.Rows) {
|
||||
end = len(set.Rows)
|
||||
}
|
||||
pageRows := append([]T(nil), set.Rows[offset:end]...)
|
||||
next := ""
|
||||
hasMore := end < len(set.Rows)
|
||||
if hasMore {
|
||||
next = EncodePageCursor(end)
|
||||
}
|
||||
return Page[T]{
|
||||
Table: set.Table,
|
||||
Manifest: set.Manifest,
|
||||
Rows: pageRows,
|
||||
Offset: offset,
|
||||
Limit: limit,
|
||||
Total: len(set.Rows),
|
||||
NextCursor: next,
|
||||
HasMore: hasMore,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s RowSet[T]) Page(options PageOptions) (Page[T], error) {
|
||||
return PageRows(s, options)
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Page(options PageOptions) (Page[T], error) {
|
||||
if v == nil {
|
||||
return Page[T]{}, ErrTableRequired
|
||||
}
|
||||
return v.Snapshot().Page(options)
|
||||
}
|
||||
|
||||
func EncodePageCursor(offset int) string {
|
||||
if offset <= 0 {
|
||||
return ""
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]int{"offset": offset})
|
||||
return base64.RawURLEncoding.EncodeToString(payload)
|
||||
}
|
||||
|
||||
func DecodePageCursor(cursor string) (int, error) {
|
||||
cursor = strings.TrimSpace(cursor)
|
||||
if cursor == "" {
|
||||
return 0, nil
|
||||
}
|
||||
data, err := base64.RawURLEncoding.DecodeString(cursor)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("decode page cursor: %w", err)
|
||||
}
|
||||
var payload struct {
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
return 0, fmt.Errorf("parse page cursor: %w", err)
|
||||
}
|
||||
if payload.Offset < 0 {
|
||||
return 0, fmt.Errorf("page cursor offset must be non-negative")
|
||||
}
|
||||
return payload.Offset, nil
|
||||
}
|
||||
|
||||
func normalizePageOptions(options PageOptions, total int) (int, int, error) {
|
||||
offset := options.Offset
|
||||
if strings.TrimSpace(options.Cursor) != "" {
|
||||
decoded, err := DecodePageCursor(options.Cursor)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
offset = decoded
|
||||
}
|
||||
if offset < 0 {
|
||||
return 0, 0, fmt.Errorf("page offset must be non-negative")
|
||||
}
|
||||
if offset > total {
|
||||
offset = total
|
||||
}
|
||||
limit := options.Limit
|
||||
if limit < 0 {
|
||||
return 0, 0, fmt.Errorf("page limit must be positive")
|
||||
}
|
||||
if limit == 0 {
|
||||
limit = DefaultPageLimit
|
||||
}
|
||||
return offset, limit, nil
|
||||
}
|
||||
|
||||
func ParsePageOptions(offsetText, limitText, cursor string) (PageOptions, error) {
|
||||
options := PageOptions{Cursor: strings.TrimSpace(cursor)}
|
||||
if strings.TrimSpace(offsetText) != "" {
|
||||
offset, err := strconv.Atoi(strings.TrimSpace(offsetText))
|
||||
if err != nil {
|
||||
return PageOptions{}, fmt.Errorf("parse page offset: %w", err)
|
||||
}
|
||||
options.Offset = offset
|
||||
}
|
||||
if strings.TrimSpace(limitText) != "" {
|
||||
limit, err := strconv.Atoi(strings.TrimSpace(limitText))
|
||||
if err != nil {
|
||||
return PageOptions{}, fmt.Errorf("parse page limit: %w", err)
|
||||
}
|
||||
options.Limit = limit
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrPipelineInvalid = errors.New("data table pipeline is invalid")
|
||||
|
||||
type PipelinePlan struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Baseline RegionInput `json:"baseline"`
|
||||
Candidate RegionInput `json:"candidate"`
|
||||
Regions []RegionInput `json:"regions,omitempty"`
|
||||
RequireVersionChange bool `json:"require_version_change,omitempty"`
|
||||
MaxAdded *int `json:"max_added,omitempty"`
|
||||
MaxChanged *int `json:"max_changed,omitempty"`
|
||||
MaxRemoved *int `json:"max_removed,omitempty"`
|
||||
RequireRegionClean bool `json:"require_region_clean,omitempty"`
|
||||
}
|
||||
|
||||
type PipelineReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Baseline RegionSnapshot `json:"baseline"`
|
||||
Candidate RegionSnapshot `json:"candidate"`
|
||||
Diff Diff `json:"diff"`
|
||||
RegionReport *RegionReport `json:"region_report,omitempty"`
|
||||
Accepted bool `json:"accepted"`
|
||||
Violations []string `json:"violations,omitempty"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
}
|
||||
|
||||
func ValidatePipeline(ctx context.Context, plan PipelinePlan) (PipelineReport, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return PipelineReport{}, err
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
plan = normPipelinePlan(plan)
|
||||
if plan.Baseline.Root == "" || plan.Candidate.Root == "" {
|
||||
return PipelineReport{}, fmt.Errorf("%w: baseline and candidate roots are required", ErrPipelineInvalid)
|
||||
}
|
||||
|
||||
baseline, err := loadDirectory(ctx, plan.Baseline.Root, plan.Baseline.Version)
|
||||
if err != nil {
|
||||
return PipelineReport{}, fmt.Errorf("load baseline data tables: %w", err)
|
||||
}
|
||||
candidate, err := loadDirectory(ctx, plan.Candidate.Root, plan.Candidate.Version)
|
||||
if err != nil {
|
||||
return PipelineReport{}, fmt.Errorf("load candidate data tables: %w", err)
|
||||
}
|
||||
|
||||
diff := diffTables(baseline.tables, candidate.tables)
|
||||
report := PipelineReport{
|
||||
Name: plan.Name,
|
||||
Baseline: RegionSnapshot{
|
||||
Name: plan.Baseline.Name,
|
||||
Root: plan.Baseline.Root,
|
||||
Version: plan.Baseline.Version,
|
||||
Manifest: baseline.Manifest(),
|
||||
},
|
||||
Candidate: RegionSnapshot{
|
||||
Name: plan.Candidate.Name,
|
||||
Root: plan.Candidate.Root,
|
||||
Version: plan.Candidate.Version,
|
||||
Manifest: candidate.Manifest(),
|
||||
},
|
||||
Diff: cloneDiff(diff),
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
}
|
||||
report.Violations = append(report.Violations, validatePipelineDiff(plan, baseline, candidate, diff)...)
|
||||
|
||||
if len(plan.Regions) > 0 {
|
||||
regionReport, err := CompareRegions(ctx, plan.Regions, plan.Baseline.Name)
|
||||
if err != nil {
|
||||
return PipelineReport{}, err
|
||||
}
|
||||
report.RegionReport = ®ionReport
|
||||
if plan.RequireRegionClean && !regionReport.Summary.Consistent {
|
||||
report.Violations = append(report.Violations, "region diff must be clean")
|
||||
}
|
||||
}
|
||||
report.Accepted = len(report.Violations) == 0
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func normPipelinePlan(plan PipelinePlan) PipelinePlan {
|
||||
plan.Name = strings.TrimSpace(plan.Name)
|
||||
plan.Baseline.Name = firstPipelineString(strings.TrimSpace(plan.Baseline.Name), "baseline")
|
||||
plan.Baseline.Root = strings.TrimSpace(plan.Baseline.Root)
|
||||
plan.Baseline.Version = strings.TrimSpace(plan.Baseline.Version)
|
||||
plan.Candidate.Name = firstPipelineString(strings.TrimSpace(plan.Candidate.Name), "candidate")
|
||||
plan.Candidate.Root = strings.TrimSpace(plan.Candidate.Root)
|
||||
plan.Candidate.Version = strings.TrimSpace(plan.Candidate.Version)
|
||||
for idx := range plan.Regions {
|
||||
plan.Regions[idx].Name = strings.TrimSpace(plan.Regions[idx].Name)
|
||||
plan.Regions[idx].Root = strings.TrimSpace(plan.Regions[idx].Root)
|
||||
plan.Regions[idx].Version = strings.TrimSpace(plan.Regions[idx].Version)
|
||||
}
|
||||
return plan
|
||||
}
|
||||
|
||||
func validatePipelineDiff(plan PipelinePlan, baseline, candidate Dataset, diff Diff) []string {
|
||||
var violations []string
|
||||
if plan.RequireVersionChange && baseline.Manifest().Version == candidate.Manifest().Version {
|
||||
violations = append(violations, "candidate version must differ from baseline")
|
||||
}
|
||||
if plan.MaxAdded != nil && len(diff.Added) > *plan.MaxAdded {
|
||||
violations = append(violations, fmt.Sprintf("added tables %d exceeds max %d", len(diff.Added), *plan.MaxAdded))
|
||||
}
|
||||
if plan.MaxChanged != nil && len(diff.Changed) > *plan.MaxChanged {
|
||||
violations = append(violations, fmt.Sprintf("changed tables %d exceeds max %d", len(diff.Changed), *plan.MaxChanged))
|
||||
}
|
||||
if plan.MaxRemoved != nil && len(diff.Removed) > *plan.MaxRemoved {
|
||||
violations = append(violations, fmt.Sprintf("removed tables %d exceeds max %d", len(diff.Removed), *plan.MaxRemoved))
|
||||
}
|
||||
return violations
|
||||
}
|
||||
|
||||
func firstPipelineString(values ...string) string {
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRegionRequired = errors.New("data table region is required")
|
||||
ErrBaselineNotFound = errors.New("data table baseline region not found")
|
||||
ErrRegionDiffRequires2 = errors.New("data table region diff requires at least two regions")
|
||||
)
|
||||
|
||||
type RegionInput struct {
|
||||
Name string `json:"name"`
|
||||
Root string `json:"root"`
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
type RegionSnapshot struct {
|
||||
Name string `json:"name"`
|
||||
Root string `json:"root"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Manifest Manifest `json:"manifest"`
|
||||
}
|
||||
|
||||
type RegionDiff struct {
|
||||
Region string `json:"region"`
|
||||
Baseline string `json:"baseline"`
|
||||
Manifest Manifest `json:"manifest"`
|
||||
BaselineManifest Manifest `json:"baseline_manifest"`
|
||||
Diff Diff `json:"diff"`
|
||||
}
|
||||
|
||||
type RegionDiffSummary struct {
|
||||
Regions int `json:"regions"`
|
||||
Compared int `json:"compared"`
|
||||
Baseline string `json:"baseline"`
|
||||
BaselineChecksum string `json:"baseline_checksum,omitempty"`
|
||||
Consistent bool `json:"consistent"`
|
||||
DriftedRegions []string `json:"drifted_regions,omitempty"`
|
||||
Added int `json:"added"`
|
||||
Changed int `json:"changed"`
|
||||
Removed int `json:"removed"`
|
||||
Unchanged int `json:"unchanged"`
|
||||
}
|
||||
|
||||
type RegionReport struct {
|
||||
Baseline string `json:"baseline"`
|
||||
Regions []RegionSnapshot `json:"regions"`
|
||||
Diffs []RegionDiff `json:"diffs"`
|
||||
Summary RegionDiffSummary `json:"summary"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
}
|
||||
|
||||
func CompareRegions(ctx context.Context, regions []RegionInput, baseline string) (RegionReport, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return RegionReport{}, err
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if len(regions) == 0 {
|
||||
return RegionReport{}, ErrRegionRequired
|
||||
}
|
||||
if len(regions) < 2 {
|
||||
return RegionReport{}, ErrRegionDiffRequires2
|
||||
}
|
||||
|
||||
datasets := make(map[string]Dataset, len(regions))
|
||||
snapshots := make([]RegionSnapshot, 0, len(regions))
|
||||
for _, input := range regions {
|
||||
normalized, err := normalizeRegionInput(input)
|
||||
if err != nil {
|
||||
return RegionReport{}, err
|
||||
}
|
||||
if _, exists := datasets[normalized.Name]; exists {
|
||||
return RegionReport{}, fmt.Errorf("%w: duplicate region %s", ErrRegionRequired, normalized.Name)
|
||||
}
|
||||
dataset, err := loadDirectory(ctx, normalized.Root, normalized.Version)
|
||||
if err != nil {
|
||||
return RegionReport{}, fmt.Errorf("load data table region %s: %w", normalized.Name, err)
|
||||
}
|
||||
datasets[normalized.Name] = dataset
|
||||
snapshots = append(snapshots, RegionSnapshot{
|
||||
Name: normalized.Name,
|
||||
Root: normalized.Root,
|
||||
Version: normalized.Version,
|
||||
Manifest: dataset.Manifest(),
|
||||
})
|
||||
if strings.TrimSpace(baseline) == "" {
|
||||
baseline = normalized.Name
|
||||
}
|
||||
}
|
||||
|
||||
baseline = strings.TrimSpace(baseline)
|
||||
base, ok := datasets[baseline]
|
||||
if !ok {
|
||||
return RegionReport{}, fmt.Errorf("%w: %s", ErrBaselineNotFound, baseline)
|
||||
}
|
||||
|
||||
sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Name < snapshots[j].Name })
|
||||
report := RegionReport{
|
||||
Baseline: baseline,
|
||||
Regions: snapshots,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
}
|
||||
report.Summary = RegionDiffSummary{
|
||||
Regions: len(snapshots),
|
||||
Baseline: baseline,
|
||||
BaselineChecksum: base.Manifest().Checksum,
|
||||
Consistent: true,
|
||||
}
|
||||
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.Name == baseline {
|
||||
continue
|
||||
}
|
||||
dataset := datasets[snapshot.Name]
|
||||
diff := diffTables(base.tables, dataset.tables)
|
||||
regionDiff := RegionDiff{
|
||||
Region: snapshot.Name,
|
||||
Baseline: baseline,
|
||||
Manifest: dataset.Manifest(),
|
||||
BaselineManifest: base.Manifest(),
|
||||
Diff: cloneDiff(diff),
|
||||
}
|
||||
report.Diffs = append(report.Diffs, regionDiff)
|
||||
report.Summary.Compared++
|
||||
report.Summary.Added += len(diff.Added)
|
||||
report.Summary.Changed += len(diff.Changed)
|
||||
report.Summary.Removed += len(diff.Removed)
|
||||
report.Summary.Unchanged += len(diff.Unchanged)
|
||||
if diff.HasChanges() {
|
||||
report.Summary.Consistent = false
|
||||
report.Summary.DriftedRegions = append(report.Summary.DriftedRegions, snapshot.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(report.Summary.DriftedRegions)
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func normalizeRegionInput(input RegionInput) (RegionInput, error) {
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Root = strings.TrimSpace(input.Root)
|
||||
input.Version = strings.TrimSpace(input.Version)
|
||||
if input.Root == "" {
|
||||
return RegionInput{}, ErrDirectoryRequired
|
||||
}
|
||||
if input.Name == "" {
|
||||
input.Name = filepath.Base(filepath.Clean(input.Root))
|
||||
}
|
||||
if input.Name == "" || input.Name == "." || input.Name == string(filepath.Separator) {
|
||||
return RegionInput{}, ErrRegionRequired
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
@@ -0,0 +1,384 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrDirectoryRequired = errors.New("data table directory is required")
|
||||
ErrRegistryRequired = errors.New("data table registry is required")
|
||||
ErrTableRequired = errors.New("data table name is required")
|
||||
ErrTableNotFound = errors.New("data table not found")
|
||||
)
|
||||
|
||||
type Table struct {
|
||||
Name string `json:"name"`
|
||||
Path string `json:"path"`
|
||||
Version string `json:"version"`
|
||||
Checksum string `json:"checksum"`
|
||||
Rows int `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
LoadedAt time.Time `json:"loaded_at"`
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
root string
|
||||
version string
|
||||
|
||||
mu sync.RWMutex
|
||||
tables map[string]Table
|
||||
raw map[string][]byte
|
||||
manifest Manifest
|
||||
watchers map[chan LoadResult]struct{}
|
||||
}
|
||||
|
||||
func NewRegistry(root, version string) *Registry {
|
||||
root = strings.TrimSpace(root)
|
||||
version = strings.TrimSpace(version)
|
||||
return &Registry{
|
||||
root: root,
|
||||
version: version,
|
||||
tables: make(map[string]Table),
|
||||
raw: make(map[string][]byte),
|
||||
manifest: Manifest{Root: root, Version: version},
|
||||
watchers: make(map[chan LoadResult]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) Load(ctx context.Context) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
r.mu.RLock()
|
||||
root := strings.TrimSpace(r.root)
|
||||
version := r.version
|
||||
r.mu.RUnlock()
|
||||
_, err := r.ReloadDir(ctx, root, version, ReloadOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Registry) LoadDir(ctx context.Context, root, version string) error {
|
||||
_, err := r.ReloadDir(ctx, root, version, ReloadOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *Registry) Reload(ctx context.Context, options ReloadOptions) (LoadResult, error) {
|
||||
if r == nil {
|
||||
return LoadResult{}, nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
r.mu.RLock()
|
||||
root := strings.TrimSpace(r.root)
|
||||
version := r.version
|
||||
r.mu.RUnlock()
|
||||
return r.ReloadDir(ctx, root, version, options)
|
||||
}
|
||||
|
||||
func (r *Registry) ReloadDir(ctx context.Context, root, version string, options ReloadOptions) (LoadResult, error) {
|
||||
if r == nil {
|
||||
return LoadResult{}, nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
dataset, err := loadDirectory(ctx, root, version)
|
||||
if err != nil {
|
||||
return LoadResult{}, err
|
||||
}
|
||||
if err := options.validate(ctx, dataset); err != nil {
|
||||
return LoadResult{}, err
|
||||
}
|
||||
|
||||
r.mu.Lock()
|
||||
diff := diffTables(r.tables, dataset.tables)
|
||||
result := LoadResult{
|
||||
Manifest: dataset.manifest,
|
||||
Diff: diff,
|
||||
LoadedAt: dataset.manifest.LoadedAt,
|
||||
}
|
||||
r.root = dataset.root
|
||||
r.version = dataset.version
|
||||
r.tables = cloneTableMap(dataset.tables)
|
||||
r.raw = cloneRawMap(dataset.raw)
|
||||
r.manifest = dataset.manifest
|
||||
for ch := range r.watchers {
|
||||
select {
|
||||
case ch <- cloneLoadResult(result):
|
||||
default:
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
return cloneLoadResult(result), nil
|
||||
}
|
||||
|
||||
func (r *Registry) Configure(root, version string) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.root = strings.TrimSpace(root)
|
||||
r.version = strings.TrimSpace(version)
|
||||
if len(r.tables) == 0 {
|
||||
r.manifest = Manifest{Root: r.root, Version: r.version}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Registry) Clear() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
r.tables = make(map[string]Table)
|
||||
r.raw = make(map[string][]byte)
|
||||
r.manifest = Manifest{Root: r.root, Version: r.version}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Registry) Snapshot() []Table {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
out := make([]Table, 0, len(r.tables))
|
||||
for _, table := range r.tables {
|
||||
out = append(out, table)
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *Registry) Manifest() Manifest {
|
||||
if r == nil {
|
||||
return Manifest{}
|
||||
}
|
||||
r.mu.RLock()
|
||||
manifest := r.manifest
|
||||
r.mu.RUnlock()
|
||||
return manifest
|
||||
}
|
||||
|
||||
func (r *Registry) Decode(name string, out any) error {
|
||||
if r == nil {
|
||||
return ErrTableNotFound
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return ErrTableRequired
|
||||
}
|
||||
if out == nil {
|
||||
return fmt.Errorf("decode target for data table %q is nil", name)
|
||||
}
|
||||
r.mu.RLock()
|
||||
data, ok := r.raw[name]
|
||||
data = append([]byte(nil), data...)
|
||||
r.mu.RUnlock()
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrTableNotFound, name)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return fmt.Errorf("decode data table %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) Get(name string) (Table, bool) {
|
||||
if r == nil {
|
||||
return Table{}, false
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
r.mu.RLock()
|
||||
table, ok := r.tables[name]
|
||||
r.mu.RUnlock()
|
||||
return table, ok
|
||||
}
|
||||
|
||||
func (r *Registry) Root() string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
r.mu.RLock()
|
||||
root := r.root
|
||||
r.mu.RUnlock()
|
||||
return root
|
||||
}
|
||||
|
||||
func (r *Registry) Version() string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
r.mu.RLock()
|
||||
version := r.version
|
||||
r.mu.RUnlock()
|
||||
return version
|
||||
}
|
||||
|
||||
func (r *Registry) Watch(ctx context.Context, buffer int) (<-chan LoadResult, error) {
|
||||
if ctxErr := contextErr(ctx); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if r == nil {
|
||||
return nil, ErrRegistryRequired
|
||||
}
|
||||
if buffer < 0 {
|
||||
buffer = 0
|
||||
}
|
||||
ch := make(chan LoadResult, buffer)
|
||||
r.mu.Lock()
|
||||
if r.watchers == nil {
|
||||
r.watchers = make(map[chan LoadResult]struct{})
|
||||
}
|
||||
r.watchers[ch] = struct{}{}
|
||||
r.mu.Unlock()
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
r.mu.Lock()
|
||||
if _, ok := r.watchers[ch]; ok {
|
||||
delete(r.watchers, ch)
|
||||
close(ch)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func loadDirectory(ctx context.Context, root, version string) (Dataset, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
root = strings.TrimSpace(root)
|
||||
version = strings.TrimSpace(version)
|
||||
if root == "" {
|
||||
return Dataset{}, ErrDirectoryRequired
|
||||
}
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return Dataset{}, fmt.Errorf("stat data table directory %q: %w", root, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return Dataset{}, fmt.Errorf("data table directory %q is not a directory", root)
|
||||
}
|
||||
|
||||
loadedAt := time.Now().UTC()
|
||||
tables := make(map[string]Table)
|
||||
raw := make(map[string][]byte)
|
||||
if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return ctxErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !isJSONTable(entry.Name()) {
|
||||
return nil
|
||||
}
|
||||
table, data, err := loadOne(root, version, path, loadedAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing, ok := tables[table.Name]; ok {
|
||||
return fmt.Errorf("duplicate data table name %q from %q and %q", table.Name, existing.Path, table.Path)
|
||||
}
|
||||
tables[table.Name] = table
|
||||
raw[table.Name] = data
|
||||
return nil
|
||||
}); err != nil {
|
||||
return Dataset{}, err
|
||||
}
|
||||
manifest := buildManifest(root, version, tables, loadedAt)
|
||||
return Dataset{
|
||||
root: root,
|
||||
version: version,
|
||||
tables: tables,
|
||||
raw: raw,
|
||||
manifest: manifest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func loadOne(root, version, path string, loadedAt time.Time) (Table, []byte, error) {
|
||||
data, err := os.ReadFile(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
|
||||
if err != nil {
|
||||
return Table{}, nil, fmt.Errorf("read data table %q: %w", path, err)
|
||||
}
|
||||
rows, err := countRows(data)
|
||||
if err != nil {
|
||||
return Table{}, nil, fmt.Errorf("parse data table %q: %w", path, err)
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return Table{}, nil, fmt.Errorf("resolve data table path %q: %w", path, err)
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
name := strings.TrimSuffix(rel, filepath.Ext(rel))
|
||||
sum := sha256.Sum256(data)
|
||||
return Table{
|
||||
Name: name,
|
||||
Path: rel,
|
||||
Version: version,
|
||||
Checksum: hex.EncodeToString(sum[:]),
|
||||
Rows: rows,
|
||||
Bytes: int64(len(data)),
|
||||
LoadedAt: loadedAt,
|
||||
}, append([]byte(nil), data...), nil
|
||||
}
|
||||
|
||||
func countRows(data []byte) (int, error) {
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.UseNumber()
|
||||
var value any
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); err != io.EOF {
|
||||
if err == nil {
|
||||
return 0, fmt.Errorf("data table contains more than one JSON value")
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case []any:
|
||||
return len(typed), nil
|
||||
case map[string]any:
|
||||
if items, ok := typed["items"].([]any); ok {
|
||||
return len(items), nil
|
||||
}
|
||||
return 1, nil
|
||||
default:
|
||||
return 1, nil
|
||||
}
|
||||
}
|
||||
|
||||
func isJSONTable(name string) bool {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || name == ".DS_Store" || strings.HasPrefix(name, "._") {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(filepath.Ext(name), ".json")
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
Root string `json:"root,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Tables int `json:"tables"`
|
||||
Rows int `json:"rows"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Checksum string `json:"checksum,omitempty"`
|
||||
LoadedAt time.Time `json:"loaded_at,omitempty"`
|
||||
}
|
||||
|
||||
type ChangeType string
|
||||
|
||||
const (
|
||||
ChangeAdded ChangeType = "added"
|
||||
ChangeChanged ChangeType = "changed"
|
||||
ChangeRemoved ChangeType = "removed"
|
||||
ChangeUnchanged ChangeType = "unchanged"
|
||||
)
|
||||
|
||||
type TableChange struct {
|
||||
Type ChangeType `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Before *Table `json:"before,omitempty"`
|
||||
After *Table `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
type Diff struct {
|
||||
Added []TableChange `json:"added,omitempty"`
|
||||
Changed []TableChange `json:"changed,omitempty"`
|
||||
Removed []TableChange `json:"removed,omitempty"`
|
||||
Unchanged []TableChange `json:"unchanged,omitempty"`
|
||||
}
|
||||
|
||||
func (d Diff) HasChanges() bool {
|
||||
return len(d.Added) > 0 || len(d.Changed) > 0 || len(d.Removed) > 0
|
||||
}
|
||||
|
||||
type LoadResult struct {
|
||||
Manifest Manifest `json:"manifest"`
|
||||
Diff Diff `json:"diff"`
|
||||
LoadedAt time.Time `json:"loaded_at,omitempty"`
|
||||
}
|
||||
|
||||
type ReloadOptions struct {
|
||||
Validators []Validator
|
||||
}
|
||||
|
||||
func (o ReloadOptions) validate(ctx context.Context, dataset Dataset) error {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
for _, validator := range o.Validators {
|
||||
if validator == nil {
|
||||
continue
|
||||
}
|
||||
if err := validator.ValidateDataTables(ctx, dataset); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type Validator interface {
|
||||
ValidateDataTables(context.Context, Dataset) error
|
||||
}
|
||||
|
||||
type ValidatorFunc func(context.Context, Dataset) error
|
||||
|
||||
func (f ValidatorFunc) ValidateDataTables(ctx context.Context, dataset Dataset) error {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return f(ctx, dataset)
|
||||
}
|
||||
|
||||
type Dataset struct {
|
||||
root string
|
||||
version string
|
||||
tables map[string]Table
|
||||
raw map[string][]byte
|
||||
manifest Manifest
|
||||
}
|
||||
|
||||
func (d Dataset) Root() string {
|
||||
return d.root
|
||||
}
|
||||
|
||||
func (d Dataset) Version() string {
|
||||
return d.version
|
||||
}
|
||||
|
||||
func (d Dataset) Manifest() Manifest {
|
||||
return d.manifest
|
||||
}
|
||||
|
||||
func (d Dataset) Snapshot() []Table {
|
||||
return sortedTables(d.tables)
|
||||
}
|
||||
|
||||
func (d Dataset) Get(name string) (Table, bool) {
|
||||
name = strings.TrimSpace(name)
|
||||
table, ok := d.tables[name]
|
||||
return table, ok
|
||||
}
|
||||
|
||||
func (d Dataset) Decode(name string, out any) error {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return ErrTableRequired
|
||||
}
|
||||
if out == nil {
|
||||
return fmt.Errorf("decode target for data table %q is nil", name)
|
||||
}
|
||||
data, ok := d.raw[name]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrTableNotFound, name)
|
||||
}
|
||||
if err := json.Unmarshal(append([]byte(nil), data...), out); err != nil {
|
||||
return fmt.Errorf("decode data table %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d Dataset) Raw(name string) ([]byte, bool) {
|
||||
name = strings.TrimSpace(name)
|
||||
data, ok := d.raw[name]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return append([]byte(nil), data...), true
|
||||
}
|
||||
|
||||
func buildManifest(root, version string, tables map[string]Table, loadedAt time.Time) Manifest {
|
||||
snapshot := sortedTables(tables)
|
||||
manifest := Manifest{
|
||||
Root: root,
|
||||
Version: version,
|
||||
Tables: len(snapshot),
|
||||
LoadedAt: loadedAt.UTC(),
|
||||
}
|
||||
hash := sha256.New()
|
||||
for _, table := range snapshot {
|
||||
manifest.Rows += table.Rows
|
||||
manifest.Bytes += table.Bytes
|
||||
_, _ = io.WriteString(hash, table.Name)
|
||||
hash.Write([]byte{0})
|
||||
_, _ = io.WriteString(hash, table.Path)
|
||||
hash.Write([]byte{0})
|
||||
_, _ = io.WriteString(hash, table.Checksum)
|
||||
hash.Write([]byte{0})
|
||||
_, _ = io.WriteString(hash, fmt.Sprintf("%d:%d", table.Rows, table.Bytes))
|
||||
hash.Write([]byte{0})
|
||||
}
|
||||
manifest.Checksum = hex.EncodeToString(hash.Sum(nil))
|
||||
return manifest
|
||||
}
|
||||
|
||||
func diffTables(before, after map[string]Table) Diff {
|
||||
names := make(map[string]struct{}, len(before)+len(after))
|
||||
for name := range before {
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
for name := range after {
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
ordered := make([]string, 0, len(names))
|
||||
for name := range names {
|
||||
ordered = append(ordered, name)
|
||||
}
|
||||
sort.Strings(ordered)
|
||||
|
||||
var diff Diff
|
||||
for _, name := range ordered {
|
||||
oldTable, hadOld := before[name]
|
||||
newTable, hasNew := after[name]
|
||||
switch {
|
||||
case !hadOld && hasNew:
|
||||
diff.Added = append(diff.Added, TableChange{Type: ChangeAdded, Name: name, After: tableRef(newTable)})
|
||||
case hadOld && !hasNew:
|
||||
diff.Removed = append(diff.Removed, TableChange{Type: ChangeRemoved, Name: name, Before: tableRef(oldTable)})
|
||||
case sameTableContent(oldTable, newTable):
|
||||
diff.Unchanged = append(diff.Unchanged, TableChange{Type: ChangeUnchanged, Name: name, Before: tableRef(oldTable), After: tableRef(newTable)})
|
||||
default:
|
||||
diff.Changed = append(diff.Changed, TableChange{Type: ChangeChanged, Name: name, Before: tableRef(oldTable), After: tableRef(newTable)})
|
||||
}
|
||||
}
|
||||
return diff
|
||||
}
|
||||
|
||||
func sameTableContent(left, right Table) bool {
|
||||
return left.Name == right.Name &&
|
||||
left.Path == right.Path &&
|
||||
left.Checksum == right.Checksum &&
|
||||
left.Rows == right.Rows &&
|
||||
left.Bytes == right.Bytes
|
||||
}
|
||||
|
||||
func sortedTables(tables map[string]Table) []Table {
|
||||
out := make([]Table, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
out = append(out, table)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneTableMap(in map[string]Table) map[string]Table {
|
||||
out := make(map[string]Table, len(in))
|
||||
for name, table := range in {
|
||||
out[name] = table
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneRawMap(in map[string][]byte) map[string][]byte {
|
||||
out := make(map[string][]byte, len(in))
|
||||
for name, data := range in {
|
||||
out[name] = append([]byte(nil), data...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneLoadResult(in LoadResult) LoadResult {
|
||||
return LoadResult{
|
||||
Manifest: in.Manifest,
|
||||
Diff: cloneDiff(in.Diff),
|
||||
LoadedAt: in.LoadedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func cloneDiff(in Diff) Diff {
|
||||
return Diff{
|
||||
Added: cloneTableChanges(in.Added),
|
||||
Changed: cloneTableChanges(in.Changed),
|
||||
Removed: cloneTableChanges(in.Removed),
|
||||
Unchanged: cloneTableChanges(in.Unchanged),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneTableChanges(in []TableChange) []TableChange {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]TableChange, len(in))
|
||||
for i, change := range in {
|
||||
out[i] = TableChange{
|
||||
Type: change.Type,
|
||||
Name: change.Name,
|
||||
Before: cloneTableRef(change.Before),
|
||||
After: cloneTableRef(change.After),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func tableRef(table Table) *Table {
|
||||
return &Table{
|
||||
Name: table.Name,
|
||||
Path: table.Path,
|
||||
Version: table.Version,
|
||||
Checksum: table.Checksum,
|
||||
Rows: table.Rows,
|
||||
Bytes: table.Bytes,
|
||||
LoadedAt: table.LoadedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func cloneTableRef(table *Table) *Table {
|
||||
if table == nil {
|
||||
return nil
|
||||
}
|
||||
return tableRef(*table)
|
||||
}
|
||||
|
||||
func contextErr(ctx context.Context) error {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrDataSourceRequired = errors.New("data table source is required")
|
||||
ErrIndexKeyRequired = errors.New("data table index key function is required")
|
||||
ErrDuplicateIndexKey = errors.New("data table index key is duplicated")
|
||||
ErrMismatchedTableView = errors.New("data table view name does not match row set")
|
||||
ErrPreparedViewInvalid = errors.New("prepared data table view is invalid")
|
||||
)
|
||||
|
||||
type RowSource interface {
|
||||
Decode(name string, out any) error
|
||||
Get(name string) (Table, bool)
|
||||
Manifest() Manifest
|
||||
}
|
||||
|
||||
type KeyFunc[T any, K comparable] func(T) (K, bool)
|
||||
|
||||
type ViewBinding interface {
|
||||
TableName() string
|
||||
Prepare(context.Context, RowSource) (PreparedView, error)
|
||||
}
|
||||
|
||||
type PreparedView struct {
|
||||
Table string `json:"table"`
|
||||
Rows int `json:"rows"`
|
||||
apply func()
|
||||
commit func() error
|
||||
rollback func()
|
||||
}
|
||||
|
||||
func NewPreparedView(table string, rows int, apply func()) (PreparedView, error) {
|
||||
return NewViewWithRollback(table, rows, apply, nil)
|
||||
}
|
||||
|
||||
func NewViewWithRollback(table string, rows int, apply func(), rollback func()) (PreparedView, error) {
|
||||
var commit func() error
|
||||
if apply != nil {
|
||||
commit = func() error { apply(); return nil }
|
||||
}
|
||||
view := PreparedView{
|
||||
Table: strings.TrimSpace(table),
|
||||
Rows: rows,
|
||||
apply: apply,
|
||||
commit: commit,
|
||||
rollback: rollback,
|
||||
}
|
||||
if err := view.Validate(); err != nil {
|
||||
return PreparedView{}, err
|
||||
}
|
||||
return view, nil
|
||||
}
|
||||
|
||||
func (p PreparedView) Validate() error {
|
||||
if strings.TrimSpace(p.Table) == "" || p.commit == nil {
|
||||
return ErrPreparedViewInvalid
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p PreparedView) Commit() error {
|
||||
if err := p.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.commit()
|
||||
}
|
||||
|
||||
func (p PreparedView) Rollback() {
|
||||
if p.rollback != nil {
|
||||
p.rollback()
|
||||
}
|
||||
}
|
||||
|
||||
type RowSet[T any] struct {
|
||||
Table Table `json:"table"`
|
||||
Manifest Manifest `json:"manifest"`
|
||||
Rows []T `json:"rows,omitempty"`
|
||||
}
|
||||
|
||||
func (s RowSet[T]) Clone() RowSet[T] {
|
||||
s.Rows = cloneRows(s.Rows)
|
||||
return s
|
||||
}
|
||||
|
||||
type IndexedRowSet[T any, K comparable] struct {
|
||||
RowSet[T]
|
||||
Index map[K]T `json:"-"`
|
||||
}
|
||||
|
||||
func (s IndexedRowSet[T, K]) Clone() IndexedRowSet[T, K] {
|
||||
out := IndexedRowSet[T, K]{
|
||||
RowSet: s.RowSet.Clone(),
|
||||
Index: make(map[K]T, len(s.Index)),
|
||||
}
|
||||
for key, row := range s.Index {
|
||||
out.Index[key] = cloneValue(row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func LoadRows[T any](source RowSource, name string) (RowSet[T], error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if source == nil {
|
||||
return RowSet[T]{}, ErrDataSourceRequired
|
||||
}
|
||||
if name == "" {
|
||||
return RowSet[T]{}, ErrTableRequired
|
||||
}
|
||||
table, ok := source.Get(name)
|
||||
if !ok {
|
||||
return RowSet[T]{}, fmt.Errorf("%w: %s", ErrTableNotFound, name)
|
||||
}
|
||||
var rows []T
|
||||
if err := source.Decode(name, &rows); err != nil {
|
||||
var wrapped struct {
|
||||
Items []T `json:"items"`
|
||||
}
|
||||
if wrappedErr := source.Decode(name, &wrapped); wrappedErr != nil || wrapped.Items == nil {
|
||||
return RowSet[T]{}, err
|
||||
}
|
||||
rows = wrapped.Items
|
||||
}
|
||||
return RowSet[T]{
|
||||
Table: table,
|
||||
Manifest: source.Manifest(),
|
||||
Rows: append([]T(nil), rows...),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UniqueIndex[T any, K comparable](rows []T, keyFn KeyFunc[T, K]) (map[K]T, error) {
|
||||
if keyFn == nil {
|
||||
return nil, ErrIndexKeyRequired
|
||||
}
|
||||
index := make(map[K]T, len(rows))
|
||||
for _, row := range rows {
|
||||
key, ok := keyFn(row)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := index[key]; exists {
|
||||
return nil, fmt.Errorf("%w: %v", ErrDuplicateIndexKey, key)
|
||||
}
|
||||
index[key] = row
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func MultiIndex[T any, K comparable](rows []T, keyFn KeyFunc[T, K]) (map[K][]T, error) {
|
||||
if keyFn == nil {
|
||||
return nil, ErrIndexKeyRequired
|
||||
}
|
||||
index := make(map[K][]T)
|
||||
for _, row := range rows {
|
||||
key, ok := keyFn(row)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
index[key] = append(index[key], row)
|
||||
}
|
||||
return cloneMultiIndex(index), nil
|
||||
}
|
||||
|
||||
type TableView[T any] struct {
|
||||
name string
|
||||
mu sync.RWMutex
|
||||
set RowSet[T]
|
||||
}
|
||||
|
||||
func NewTableView[T any](name string) *TableView[T] {
|
||||
return &TableView[T]{name: strings.TrimSpace(name)}
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Name() string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return v.name
|
||||
}
|
||||
|
||||
func (v *TableView[T]) TableName() string {
|
||||
return v.Name()
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Prepare(ctx context.Context, source RowSource) (PreparedView, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return PreparedView{}, err
|
||||
}
|
||||
if v == nil {
|
||||
return PreparedView{}, ErrTableRequired
|
||||
}
|
||||
next, err := LoadRows[T](source, v.name)
|
||||
if err != nil {
|
||||
return PreparedView{}, err
|
||||
}
|
||||
v.mu.RLock()
|
||||
previous := v.set.Clone()
|
||||
v.mu.RUnlock()
|
||||
return NewViewWithRollback(v.name, len(next.Rows), func() {
|
||||
v.mu.Lock()
|
||||
v.set = next.Clone()
|
||||
v.mu.Unlock()
|
||||
}, func() {
|
||||
v.mu.Lock()
|
||||
v.set = previous.Clone()
|
||||
v.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Reload(ctx context.Context, source RowSource) (RowSet[T], error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return RowSet[T]{}, err
|
||||
}
|
||||
if v == nil {
|
||||
return RowSet[T]{}, ErrTableRequired
|
||||
}
|
||||
next, err := LoadRows[T](source, v.name)
|
||||
if err != nil {
|
||||
return RowSet[T]{}, err
|
||||
}
|
||||
v.mu.Lock()
|
||||
v.set = next.Clone()
|
||||
v.mu.Unlock()
|
||||
return next.Clone(), nil
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Replace(set RowSet[T]) error {
|
||||
if v == nil {
|
||||
return ErrTableRequired
|
||||
}
|
||||
if strings.TrimSpace(v.name) != strings.TrimSpace(set.Table.Name) {
|
||||
return fmt.Errorf("%w: view=%s rows=%s", ErrMismatchedTableView, v.name, set.Table.Name)
|
||||
}
|
||||
v.mu.Lock()
|
||||
v.set = set.Clone()
|
||||
v.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Snapshot() RowSet[T] {
|
||||
if v == nil {
|
||||
return RowSet[T]{}
|
||||
}
|
||||
v.mu.RLock()
|
||||
set := v.set.Clone()
|
||||
v.mu.RUnlock()
|
||||
return set
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Rows() []T {
|
||||
return v.Snapshot().Rows
|
||||
}
|
||||
|
||||
func (v *TableView[T]) Clear() {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
v.mu.Lock()
|
||||
v.set = RowSet[T]{Table: Table{Name: v.name}}
|
||||
v.mu.Unlock()
|
||||
}
|
||||
|
||||
type UniqueIndexView[T any, K comparable] struct {
|
||||
name string
|
||||
keyFn KeyFunc[T, K]
|
||||
|
||||
mu sync.RWMutex
|
||||
set IndexedRowSet[T, K]
|
||||
}
|
||||
|
||||
func NewUniqueIndexView[T any, K comparable](name string, keyFn KeyFunc[T, K]) *UniqueIndexView[T, K] {
|
||||
return &UniqueIndexView[T, K]{
|
||||
name: strings.TrimSpace(name),
|
||||
keyFn: keyFn,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Name() string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return v.name
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) TableName() string {
|
||||
return v.Name()
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Prepare(ctx context.Context, source RowSource) (PreparedView, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return PreparedView{}, err
|
||||
}
|
||||
if v == nil {
|
||||
return PreparedView{}, ErrTableRequired
|
||||
}
|
||||
next, err := v.build(source)
|
||||
if err != nil {
|
||||
return PreparedView{}, err
|
||||
}
|
||||
v.mu.RLock()
|
||||
previous := v.set.Clone()
|
||||
v.mu.RUnlock()
|
||||
return NewViewWithRollback(v.name, len(next.Rows), func() {
|
||||
v.mu.Lock()
|
||||
v.set = next.Clone()
|
||||
v.mu.Unlock()
|
||||
}, func() {
|
||||
v.mu.Lock()
|
||||
v.set = previous.Clone()
|
||||
v.mu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Reload(ctx context.Context, source RowSource) (IndexedRowSet[T, K], error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return IndexedRowSet[T, K]{}, err
|
||||
}
|
||||
if v == nil {
|
||||
return IndexedRowSet[T, K]{}, ErrTableRequired
|
||||
}
|
||||
next, err := v.build(source)
|
||||
if err != nil {
|
||||
return IndexedRowSet[T, K]{}, err
|
||||
}
|
||||
v.mu.Lock()
|
||||
v.set = next.Clone()
|
||||
v.mu.Unlock()
|
||||
return next.Clone(), nil
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Snapshot() IndexedRowSet[T, K] {
|
||||
if v == nil {
|
||||
return IndexedRowSet[T, K]{}
|
||||
}
|
||||
v.mu.RLock()
|
||||
set := v.set.Clone()
|
||||
v.mu.RUnlock()
|
||||
return set
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Rows() []T {
|
||||
return v.Snapshot().Rows
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Clear() {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
v.mu.Lock()
|
||||
v.set = IndexedRowSet[T, K]{
|
||||
RowSet: RowSet[T]{Table: Table{Name: v.name}},
|
||||
Index: make(map[K]T),
|
||||
}
|
||||
v.mu.Unlock()
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) Get(key K) (T, bool) {
|
||||
var zero T
|
||||
if v == nil {
|
||||
return zero, false
|
||||
}
|
||||
v.mu.RLock()
|
||||
row, ok := v.set.Index[key]
|
||||
v.mu.RUnlock()
|
||||
if !ok {
|
||||
return zero, false
|
||||
}
|
||||
return cloneValue(row), true
|
||||
}
|
||||
|
||||
func (v *UniqueIndexView[T, K]) build(source RowSource) (IndexedRowSet[T, K], error) {
|
||||
rows, err := LoadRows[T](source, v.name)
|
||||
if err != nil {
|
||||
return IndexedRowSet[T, K]{}, err
|
||||
}
|
||||
index, err := UniqueIndex(rows.Rows, v.keyFn)
|
||||
if err != nil {
|
||||
return IndexedRowSet[T, K]{}, err
|
||||
}
|
||||
return IndexedRowSet[T, K]{
|
||||
RowSet: rows.Clone(),
|
||||
Index: index,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cloneMultiIndex[T any, K comparable](in map[K][]T) map[K][]T {
|
||||
out := make(map[K][]T, len(in))
|
||||
for key, rows := range in {
|
||||
out[key] = cloneRows(rows)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneRows[T any](rows []T) []T {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]T, len(rows))
|
||||
for i, row := range rows {
|
||||
out[i] = cloneValue(row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneValue[T any](value T) T {
|
||||
cloned := cloneReflect(reflect.ValueOf(value))
|
||||
if !cloned.IsValid() {
|
||||
return value
|
||||
}
|
||||
out, ok := cloned.Interface().(T)
|
||||
if !ok {
|
||||
return value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneReflect(v reflect.Value) reflect.Value {
|
||||
if !v.IsValid() {
|
||||
return v
|
||||
}
|
||||
switch v.Kind() {
|
||||
case reflect.Pointer:
|
||||
if v.IsNil() {
|
||||
return v
|
||||
}
|
||||
out := reflect.New(v.Type().Elem())
|
||||
elem := cloneReflect(v.Elem())
|
||||
if elem.IsValid() && elem.Type().AssignableTo(v.Type().Elem()) {
|
||||
out.Elem().Set(elem)
|
||||
} else {
|
||||
out.Elem().Set(v.Elem())
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if v.IsNil() {
|
||||
return v
|
||||
}
|
||||
out := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
|
||||
for i := 0; i < v.Len(); i++ {
|
||||
setClonedValue(out.Index(i), v.Index(i))
|
||||
}
|
||||
return out
|
||||
case reflect.Map:
|
||||
if v.IsNil() {
|
||||
return v
|
||||
}
|
||||
out := reflect.MakeMapWithSize(v.Type(), v.Len())
|
||||
iter := v.MapRange()
|
||||
for iter.Next() {
|
||||
value := cloneReflect(iter.Value())
|
||||
if value.IsValid() && value.Type().AssignableTo(v.Type().Elem()) {
|
||||
out.SetMapIndex(iter.Key(), value)
|
||||
} else {
|
||||
out.SetMapIndex(iter.Key(), iter.Value())
|
||||
}
|
||||
}
|
||||
return out
|
||||
case reflect.Struct:
|
||||
out := reflect.New(v.Type()).Elem()
|
||||
for i := 0; i < v.NumField(); i++ {
|
||||
if !out.Field(i).CanSet() {
|
||||
return v
|
||||
}
|
||||
setClonedValue(out.Field(i), v.Field(i))
|
||||
}
|
||||
return out
|
||||
case reflect.Interface:
|
||||
if v.IsNil() {
|
||||
return v
|
||||
}
|
||||
elem := cloneReflect(v.Elem())
|
||||
if elem.IsValid() && elem.Type().AssignableTo(v.Type()) {
|
||||
return elem
|
||||
}
|
||||
out := reflect.New(v.Type()).Elem()
|
||||
if elem.IsValid() && elem.Type().AssignableTo(out.Type()) {
|
||||
out.Set(elem)
|
||||
return out
|
||||
}
|
||||
return v
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
func setClonedValue(dst, src reflect.Value) {
|
||||
value := cloneReflect(src)
|
||||
if value.IsValid() && value.Type().AssignableTo(dst.Type()) {
|
||||
dst.Set(value)
|
||||
return
|
||||
}
|
||||
dst.Set(src)
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package datatable
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrViewRequired = errors.New("data table view binding is required")
|
||||
)
|
||||
|
||||
type ViewRefreshResult struct {
|
||||
Manifest Manifest `json:"manifest"`
|
||||
Tables []string `json:"tables,omitempty"`
|
||||
Views int `json:"views"`
|
||||
Skipped []string `json:"skipped,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type ViewManagerSnapshot struct {
|
||||
Tables []string `json:"tables,omitempty"`
|
||||
Views int `json:"views"`
|
||||
LastResult ViewRefreshResult `json:"last_result"`
|
||||
}
|
||||
|
||||
type ViewManager struct {
|
||||
mu sync.RWMutex
|
||||
views map[string][]ViewBinding
|
||||
last ViewRefreshResult
|
||||
}
|
||||
|
||||
func NewViewManager(bindings ...ViewBinding) (*ViewManager, error) {
|
||||
manager := &ViewManager{
|
||||
views: make(map[string][]ViewBinding),
|
||||
}
|
||||
if err := manager.Register(bindings...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return manager, nil
|
||||
}
|
||||
|
||||
func (m *ViewManager) Register(bindings ...ViewBinding) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.views == nil {
|
||||
m.views = make(map[string][]ViewBinding)
|
||||
}
|
||||
for _, binding := range bindings {
|
||||
if binding == nil {
|
||||
return ErrViewRequired
|
||||
}
|
||||
name := strings.TrimSpace(binding.TableName())
|
||||
if name == "" {
|
||||
return ErrTableRequired
|
||||
}
|
||||
m.views[name] = append(m.views[name], binding)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ViewManager) Refresh(ctx context.Context, source RowSource, load LoadResult) (ViewRefreshResult, error) {
|
||||
return m.refresh(ctx, source, load.Manifest, affectedTableNames(load.Diff), false)
|
||||
}
|
||||
|
||||
func (m *ViewManager) RefreshAll(ctx context.Context, source RowSource) (ViewRefreshResult, error) {
|
||||
manifest := Manifest{}
|
||||
if source != nil {
|
||||
manifest = source.Manifest()
|
||||
}
|
||||
return m.refresh(ctx, source, manifest, nil, true)
|
||||
}
|
||||
|
||||
func (m *ViewManager) ValidateDataTables(ctx context.Context, dataset Dataset) error {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
bindings, _, _ := m.selectBindings(nil, true)
|
||||
for _, binding := range bindings {
|
||||
view, err := binding.Prepare(ctx, dataset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare data table view %q: %w", binding.TableName(), err)
|
||||
}
|
||||
if err := view.Validate(); err != nil {
|
||||
return fmt.Errorf("validate data table view %q: %w", view.Table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ViewManager) Clear() ViewRefreshResult {
|
||||
if m == nil {
|
||||
return ViewRefreshResult{}
|
||||
}
|
||||
bindings, skipped, tables := m.selectBindings(nil, true)
|
||||
result := ViewRefreshResult{
|
||||
Tables: tables,
|
||||
Skipped: skipped,
|
||||
At: time.Now().UTC(),
|
||||
}
|
||||
for _, binding := range bindings {
|
||||
clearer, ok := binding.(interface{ Clear() })
|
||||
if !ok {
|
||||
result.Skipped = append(result.Skipped, binding.TableName())
|
||||
continue
|
||||
}
|
||||
clearer.Clear()
|
||||
result.Views++
|
||||
}
|
||||
result.Skipped = normalizeTableNames(result.Skipped)
|
||||
m.record(result)
|
||||
return cloneViewRefresh(result)
|
||||
}
|
||||
|
||||
func (m *ViewManager) Run(ctx context.Context, source RowSource, changes <-chan LoadResult, onError func(ViewRefreshResult, error)) <-chan struct{} {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
if changes == nil {
|
||||
close(done)
|
||||
return done
|
||||
}
|
||||
go func() {
|
||||
defer close(done)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case change, ok := <-changes:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
result, err := m.Refresh(ctx, source, change)
|
||||
if err != nil && onError != nil {
|
||||
onError(result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
func (m *ViewManager) Snapshot() ViewManagerSnapshot {
|
||||
if m == nil {
|
||||
return ViewManagerSnapshot{}
|
||||
}
|
||||
m.mu.RLock()
|
||||
tables := make([]string, 0, len(m.views))
|
||||
views := 0
|
||||
for table, bindings := range m.views {
|
||||
tables = append(tables, table)
|
||||
views += len(bindings)
|
||||
}
|
||||
last := cloneViewRefresh(m.last)
|
||||
m.mu.RUnlock()
|
||||
sort.Strings(tables)
|
||||
return ViewManagerSnapshot{
|
||||
Tables: tables,
|
||||
Views: views,
|
||||
LastResult: last,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *ViewManager) refresh(ctx context.Context, source RowSource, manifest Manifest, names []string, all bool) (ViewRefreshResult, error) {
|
||||
if err := contextErr(ctx); err != nil {
|
||||
return ViewRefreshResult{}, err
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if m == nil {
|
||||
return ViewRefreshResult{}, nil
|
||||
}
|
||||
if source == nil {
|
||||
return ViewRefreshResult{}, ErrDataSourceRequired
|
||||
}
|
||||
names = normalizeTableNames(names)
|
||||
bindings, missing, selected := m.selectBindings(names, all)
|
||||
result := ViewRefreshResult{
|
||||
Manifest: manifest,
|
||||
Tables: selected,
|
||||
Skipped: missing,
|
||||
At: time.Now().UTC(),
|
||||
}
|
||||
if len(bindings) == 0 {
|
||||
m.record(result)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
prepared := make([]PreparedView, 0, len(bindings))
|
||||
for _, binding := range bindings {
|
||||
view, err := binding.Prepare(ctx, source)
|
||||
if err != nil {
|
||||
return m.fail(result, fmt.Errorf("prepare data table view %q: %w", binding.TableName(), err))
|
||||
}
|
||||
prepared = append(prepared, view)
|
||||
}
|
||||
for _, view := range prepared {
|
||||
if err := view.Validate(); err != nil {
|
||||
return m.fail(result, fmt.Errorf("validate data table view %q: %w", view.Table, err))
|
||||
}
|
||||
}
|
||||
committed := make([]PreparedView, 0, len(prepared))
|
||||
for _, view := range prepared {
|
||||
if err := view.Commit(); err != nil {
|
||||
for i := len(committed) - 1; i >= 0; i-- {
|
||||
committed[i].Rollback()
|
||||
}
|
||||
return m.fail(result, fmt.Errorf("commit data table view %q: %w", view.Table, err))
|
||||
}
|
||||
committed = append(committed, view)
|
||||
}
|
||||
result.Views = len(prepared)
|
||||
m.record(result)
|
||||
return cloneViewRefresh(result), nil
|
||||
}
|
||||
|
||||
func (m *ViewManager) selectBindings(names []string, all bool) ([]ViewBinding, []string, []string) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if all {
|
||||
tables := make([]string, 0, len(m.views))
|
||||
for table := range m.views {
|
||||
tables = append(tables, table)
|
||||
}
|
||||
sort.Strings(tables)
|
||||
out := make([]ViewBinding, 0)
|
||||
for _, table := range tables {
|
||||
out = append(out, m.views[table]...)
|
||||
}
|
||||
return out, nil, tables
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
out := make([]ViewBinding, 0)
|
||||
missing := make([]string, 0)
|
||||
for _, name := range names {
|
||||
bindings := m.views[name]
|
||||
if len(bindings) == 0 {
|
||||
missing = append(missing, name)
|
||||
continue
|
||||
}
|
||||
out = append(out, bindings...)
|
||||
}
|
||||
return out, missing, append([]string(nil), names...)
|
||||
}
|
||||
|
||||
func (m *ViewManager) record(result ViewRefreshResult) {
|
||||
m.mu.Lock()
|
||||
m.last = cloneViewRefresh(result)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *ViewManager) fail(result ViewRefreshResult, err error) (ViewRefreshResult, error) {
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
}
|
||||
m.record(result)
|
||||
return cloneViewRefresh(result), err
|
||||
}
|
||||
|
||||
func affectedTableNames(diff Diff) []string {
|
||||
names := make([]string, 0, len(diff.Added)+len(diff.Changed)+len(diff.Removed))
|
||||
for _, change := range diff.Added {
|
||||
names = append(names, change.Name)
|
||||
}
|
||||
for _, change := range diff.Changed {
|
||||
names = append(names, change.Name)
|
||||
}
|
||||
for _, change := range diff.Removed {
|
||||
names = append(names, change.Name)
|
||||
}
|
||||
return normalizeTableNames(names)
|
||||
}
|
||||
|
||||
func normalizeTableNames(names []string) []string {
|
||||
if len(names) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(names))
|
||||
out := make([]string, 0, len(names))
|
||||
for _, name := range names {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneViewRefresh(result ViewRefreshResult) ViewRefreshResult {
|
||||
result.Tables = append([]string(nil), result.Tables...)
|
||||
result.Skipped = append([]string(nil), result.Skipped...)
|
||||
return result
|
||||
}
|
||||
Reference in new issue
Block a user