90CNServer/pvf/internal/pvf/archive.go

package pvf

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"pvfine/internal/buildmode"
	"pvfine/internal/rendering"
)

// Header is the decrypted 48-byte archive header (packed, little-endian).
type Header struct {
	Signature     uint32
	Guid          [20]byte
	FileCount     int32
	Padding       int32
	BodySize      int32
	GroupCount    int32
	HashTableSize int32
	NameTableSize int32
}

type fileItem struct {
	identity         uint64 // process-local identity, preserved by clones and removal
	nameOff, pathOff int32  // magic offsets into the string pools
	chunk            int32  // body chunk index
	off, size        int32  // location of the file payload inside the decompressed chunk
	typ              int32  // TypeScript / TypeUnicode
}

var nextFileIdentity atomic.Uint64

// FileIdentity identifies a live entry independently of its compact file index.
// It survives CloneForBatch and RemoveFiles, but is never serialized to PVF.
func (a *Archive) FileIdentity(index int32) uint64 {
	if index < 0 || index >= int32(len(a.items)) {
		return 0
	}
	return a.items[index].identity
}

type removedFileSpan struct {
	off, size int32
}

// MutationKind identifies one effective archive entry mutation. The mutation
// journal is intentionally small: services use it to decide which derived
// indexes are affected after a compound operation has committed.
type MutationKind string

const (
	MutationModified MutationKind = "modified"
	MutationAdded    MutationKind = "added"
	MutationRemoved  MutationKind = "removed"
)

// FileMutation is one effective file-level change. Path is captured before a
// structural edit can renumber the remaining entries.
type FileMutation struct {
	Index    int32
	Identity uint64 // captured before structural edits can compact the file table
	Path     string
	Kind     MutationKind
}

// MutationSummary describes changes recorded after a mutation checkpoint.
type MutationSummary struct {
	Files      []FileMutation
	Structural bool
}

type groupItem struct{ compSize, origSize int32 }

// Archive is a parsed PVF container. It is not safe for concurrent
// modification, but Chunk decompression is cached and read-only use of
// accessors from multiple goroutines is fine.
type Archive struct {
	cacheMu sync.Mutex

	data          []byte // original file bytes (nil for archives built from scratch)
	contentDigest string // cached hash of immutable data; cacheMu protects it
	hdr           Header
	guard         bool
	keys          keySet // per-section LCG seeds (standard or recovered)

	// format separates container transport from client content conventions.
	// pageKeys retains the per-page state needed to write the container back.
	format       formatProfile
	pageKeys     []byte
	act60        *act60Archive     // immutable 60ACT transport metadata
	act60Sources map[uint64]string // edited ACT60 sources, keyed by stable file identity
	tw           *twArchive        // immutable TW transport and index metadata

	sourcePath   string // file the archive was opened from
	textEncoding string // optional native text / narrow pool codec override

	tableOff, hashOff, nameOff, grpiOff, bodyOff int
	hashSize, nameSize, grpiSize                 int

	items  []fileItem
	groups []groupItem // decrypted GRPI

	strA, strW       []byte // decompressed string pools (UTF-8 / UTF-16LE)
	strAIdx, strWIdx map[string]int32
	poolsDirty       bool // pools gained appended strings since parse

	resolveCache      map[int32]string
	resolveCacheOrder []int32
	resolveCacheBytes int64
	chunkCache        map[int32][]byte
	chunkCacheMeta    map[int32]chunkCacheEntry
	chunkCacheBytes   int64
	chunkCacheClock   uint64
	overlay           map[int32][]byte // index -> replacement payload
	pathIndex         map[string]int32
	compactPaths      []uint64 // immutable sorted fingerprints for large archives
	structuralDirty   bool     // file entries were added or removed since the last save
	removedSpans      map[int32][]removedFileSpan
	mutations         []FileMutation // effective changes since the last checkpoint

	// scriptRenderer controls the user-facing decompiled layout. The
	// canonical renderer is kept stable so version content hashes do not
	// depend on presentation-only configuration.
	scriptRenderer          *rendering.Engine
	canonicalScriptRenderer *rendering.Engine

	// tables caches the lazily loaded string tables used to resolve
	// `<index::key>` placeholders in item names.
	tables struct {
		mu       sync.Mutex
		state    *stringTableState
		stats    StringTableIndexStats
		observer func(StringTableIndexEvent)
	}
}

// MutationCheckpoint returns the current position in the archive mutation
// journal. The journal is local to one Archive and is not part of the packed
// file format.
func (a *Archive) MutationCheckpoint() int {
	if a == nil {
		return 0
	}
	return len(a.mutations)
}

// MutationsSince returns effective file changes recorded after checkpoint.
// Callers may safely retain the returned slices.
func (a *Archive) MutationsSince(checkpoint int) MutationSummary {
	if a == nil {
		return MutationSummary{}
	}
	if checkpoint < 0 {
		checkpoint = 0
	}
	if checkpoint > len(a.mutations) {
		checkpoint = len(a.mutations)
	}
	result := MutationSummary{
		Files: make([]FileMutation, len(a.mutations)-checkpoint),
	}
	copy(result.Files, a.mutations[checkpoint:])
	for _, mutation := range result.Files {
		if mutation.Kind == MutationAdded || mutation.Kind == MutationRemoved {
			result.Structural = true
			break
		}
	}
	return result
}

// ClearMutations forgets journal entries that have already been consumed by
// the owning service. It does not alter archive content.
func (a *Archive) ClearMutations() {
	if a != nil {
		a.mutations = nil
	}
}

// PendingEdits lists the entries that currently differ from the packed
// baseline: replacement payloads held in the overlay and entries added since
// the archive was parsed. Removed entries cannot be reported because their
// paths are gone from the file table. Callers must not mutate the archive
// while iterating; the returned slice is detached.
func (a *Archive) PendingEdits() []FileMutation {
	if a == nil {
		return nil
	}
	result := make([]FileMutation, 0, len(a.overlay))
	for i := range a.items {
		index := int32(i)
		if a.items[i].chunk < 0 {
			result = append(result, FileMutation{Index: index, Identity: a.FileIdentity(index), Path: a.Path(index), Kind: MutationAdded})
			continue
		}
		if _, ok := a.overlay[index]; ok {
			result = append(result, FileMutation{Index: index, Identity: a.FileIdentity(index), Path: a.Path(index), Kind: MutationModified})
		}
	}
	return result
}

func (a *Archive) recordMutation(index int32, path string, kind MutationKind) {
	if a == nil || path == "" {
		return
	}
	// A ready projection must not disagree with direct placeholder lookups.
	// Ordinary text/raw edits and structural changes share this journal entry
	// point, including late localization files and mapping-list replacements.
	if strings.HasSuffix(strings.ToLower(path), ".str") || IsStringTableMappingPath(path) {
		a.InvalidateStringTables()
	}
	a.mutations = append(a.mutations, FileMutation{Index: index, Identity: a.FileIdentity(index), Path: path, Kind: kind})
}

type chunkCacheEntry struct {
	bytes int64
	used  uint64
}

const defaultChunkCacheLimit = int64(64 << 20)
const defaultResolveCacheLimit = int64(16 << 20)

// Open reads and parses the archive at path. Paged containers may
// keep their per-page keys in sibling "sk.dat" and "DFO.exe" / "DNF.exe" files.
// Complete bundled 110US / 100CN parameters are tried first. Other revisions
// can supply matching sk.dat and client executables next to the archive.
func Open(path string) (*Archive, error) {
	return OpenWithSidecars(path, "")
}

// OpenWithSidecars reads and parses the archive at path while resolving
// container key sidecars from sidecarDir instead of the file's own directory.
// Backups of paged archives live outside the client folder, so their page
// keys have to be borrowed from the directory of the file they came from.
// An empty sidecarDir behaves like Open.
func OpenWithSidecars(path, sidecarDir string) (*Archive, error) {
	return OpenWithTextEncoding(path, sidecarDir, "auto")
}

// OpenWithTextEncoding applies the selected codec during the initial parse.
// sidecarDir has the same meaning as in OpenWithSidecars.
func OpenWithTextEncoding(path, sidecarDir, encoding string) (*Archive, error) {
	if _, err := textCodec(encoding); err != nil {
		return nil, err
	}
	if encoding == "auto" {
		encoding = ""
	}
	data, err := os.ReadFile(path)
	if err != nil {
		return nil, err
	}
	if sidecarDir == "" {
		sidecarDir = filepath.Dir(path)
	}
	a, err := parseOwned(data, sidecarDir, true, encoding)
	if err != nil {
		return nil, err
	}
	a.sourcePath = path
	return a, nil
}

// Parse parses an archive from memory. The byte slice is retained; treat it
// as read-only afterwards. Paged parsing uses the bundled 110US key table; an
// archive-specific sidecar can be supplied by using Open.
func Parse(data []byte) (*Archive, error) {
	return parse(data, "")
}

func parse(data []byte, sidecarDir string) (*Archive, error) {
	return parseOwned(data, sidecarDir, false)
}

func parseOwned(data []byte, sidecarDir string, owned bool, codecs ...string) (*Archive, error) {
	if tree, body, count, legacy, ok := probeACT60(data); ok {
		return parseACT60(data, tree, body, count, legacy, codecs...)
	}
	if bytes.HasPrefix(data, []byte(act60Signature)) {
		return nil, fmt.Errorf("%w: ACT60 directory header or checksum", ErrInvalidSection)
	}
	if isTW(data) {
		return parseTW(data, codecs...)
	}
	detected, err := detectArchive(data, sidecarDir, owned)
	if err != nil {
		return nil, err
	}
	return parseDetected(detected, codecs...)
}

// parseDetected is the shared logical-container parser, independent of file
// discovery and physical page unlocking.
func parseDetected(detected detectedArchive, codecs ...string) (*Archive, error) {
	data := detected.data
	renderer := defaultScriptRenderer()
	a := &Archive{
		data:                    data,
		resolveCache:            map[int32]string{},
		chunkCache:              map[int32][]byte{},
		chunkCacheMeta:          map[int32]chunkCacheEntry{},
		overlay:                 map[int32][]byte{},
		pathIndex:               map[string]int32{},
		removedSpans:            make(map[int32][]removedFileSpan),
		scriptRenderer:          renderer,
		canonicalScriptRenderer: renderer,
	}

	if len(codecs) > 0 {
		a.textEncoding = codecs[0]
	}
	a.hdr, a.guard, a.keys = detected.hdr, detected.guard, detected.keys
	a.format, a.pageKeys = detected.format, detected.pageKeys

	// Section layout.
	pos := headerSize
	a.tableOff = pos
	pos += int(a.hdr.FileCount) * 0x18
	a.hashOff = pos
	a.hashSize = int(a.hdr.HashTableSize)
	pos += a.hashSize
	a.nameOff = pos
	a.nameSize = int(a.hdr.NameTableSize)
	pos += a.nameSize
	a.grpiOff = pos
	a.grpiSize = int(a.hdr.GroupCount) * 8
	pos += a.grpiSize
	a.bodyOff = pos

	declaredEnd := pos + int(a.hdr.BodySize)
	if declaredEnd > len(data) {
		return nil, ErrOverflow
	}

	// File table (plaintext).
	a.items = make([]fileItem, a.hdr.FileCount)
	for i := range a.items {
		it := &a.items[i]
		it.identity = nextFileIdentity.Add(1)
		base := a.tableOff + i*0x18
		it.nameOff = int32(binary.LittleEndian.Uint32(data[base:]))
		it.pathOff = int32(binary.LittleEndian.Uint32(data[base+4:]))
		it.chunk = int32(binary.LittleEndian.Uint32(data[base+8:]))
		it.off = int32(binary.LittleEndian.Uint32(data[base+12:]))
		it.size = int32(binary.LittleEndian.Uint32(data[base+16:]))
		it.typ = int32(binary.LittleEndian.Uint32(data[base+20:]))
	}

	// GRPI (cumulative compressed chunk sizes).
	if a.grpiSize > 0 {
		grpiRaw := data[a.grpiOff : a.grpiOff+a.grpiSize]
		grpi := make([]byte, a.grpiSize)
		copy(grpi, grpiRaw)
		cryptSeed(a.keys.grpi.seed, a.keys.grpi.magic, grpi)
		if !a.grpiLooksSane(grpi) {
			// Non-standard seed: recover it from the BodySize anchor and the
			// monotonicity invariants of the cumulative size table.
			if key, ok := recoverGRPISeed(grpiRaw, int(a.hdr.GroupCount), a.hdr.BodySize); ok {
				a.keys.grpi = key
				copy(grpi, grpiRaw)
				cryptSeed(key.seed, key.magic, grpi)
			} else {
				return nil, fmt.Errorf("%w: GRPI cannot be decoded", ErrInvalidSection)
			}
		}
		a.groups = make([]groupItem, a.hdr.GroupCount)
		for i := range a.groups {
			a.groups[i].compSize = int32(binary.LittleEndian.Uint32(grpi[i*8:]))
			a.groups[i].origSize = int32(binary.LittleEndian.Uint32(grpi[i*8+4:]))
		}
	}

	// String pools.
	if err := a.parseNameTable(data[a.nameOff : a.nameOff+a.nameSize]); err != nil {
		return nil, err
	}

	// Hash section seed: known for the standard key set and the alternate
	// variant family. When it is not (an unidentified variant, the Paged110
	// containers), solve it from the section itself now that the string pools
	// are available to validate candidate offsets.
	if !a.keys.hashKnown && a.hashSize > 0 {
		if key, ok := a.recoverHashSeed(data[a.hashOff:a.hashOff+a.hashSize], int(a.hdr.FileCount)); ok {
			a.keys.hash = key
			a.keys.hashKnown = true
		}
	}

	// Body seed: for variants the standard key fails, and chunk 0's zlib
	// header plus GRPI's original size let it be recovered from the data.
	if !a.bodyKeyWorks() {
		if key, ok := recoverZlibSeed(a.firstChunkSpan(), a.firstChunkOrigSize()); ok {
			a.keys.body = key
			a.keys.bodyRecovered = true
		} else {
			return nil, fmt.Errorf("%w: body cannot be decoded", ErrInvalidSection)
		}
	}

	// Path index (case-insensitive, mirrors the reference GM tool).
	a.rebuildPathIndex()
	return a, nil
}

// New returns an empty archive ready for AddFile + SaveTo.
func New() *Archive {
	renderer := defaultScriptRenderer()
	return &Archive{
		hdr:                     Header{Signature: MagicSignature},
		keys:                    standardKeys(),
		format:                  standardProfile,
		strA:                    []byte{0},
		strW:                    []byte{0, 0},
		poolsDirty:              true,
		resolveCache:            map[int32]string{},
		chunkCache:              map[int32][]byte{},
		chunkCacheMeta:          map[int32]chunkCacheEntry{},
		overlay:                 map[int32][]byte{},
		pathIndex:               map[string]int32{},
		removedSpans:            make(map[int32][]removedFileSpan),
		scriptRenderer:          renderer,
		canonicalScriptRenderer: renderer,
	}
}

func defaultScriptRenderer() *rendering.Engine {
	engine, _ := rendering.LoadDefault()
	return engine
}

func decodeHeader(b [headerSize]byte) Header {
	return Header{
		Signature:     binary.LittleEndian.Uint32(b[0:]),
		FileCount:     int32(binary.LittleEndian.Uint32(b[24:])),
		Padding:       int32(binary.LittleEndian.Uint32(b[28:])),
		BodySize:      int32(binary.LittleEndian.Uint32(b[32:])),
		GroupCount:    int32(binary.LittleEndian.Uint32(b[36:])),
		HashTableSize: int32(binary.LittleEndian.Uint32(b[40:])),
		NameTableSize: int32(binary.LittleEndian.Uint32(b[44:])),
	}
}

// Header returns the (decrypted) archive header.
func (a *Archive) Header() Header { return a.hdr }

// UsesGuard reports whether the archive header uses the 0x55 guard variant.
func (a *Archive) UsesGuard() bool { return a.guard }

// IsPaged reports whether the archive uses page guards, independently of its
// client version.
func (a *Archive) IsPaged() bool { return a.format.container == pagedContainer }

// IsPaged110 is the legacy name for the shared page-guard capability.
// Deprecated: use IsPaged for capabilities or ClientVersion for client identity.
func (a *Archive) IsPaged110() bool { return a.IsPaged() }

// FileCount returns the number of file entries.
func (a *Archive) FileCount() int32 { return int32(len(a.items)) }

// File describes one entry.
type File struct {
	Index      int32
	Name, Path string // resolved from the string pools; Path is the directory part
	ChunkIndex int32
	DataOffset int32
	DataSize   int32
	DataType   int32
}

// File returns entry i.
func (a *Archive) File(i int32) File {
	it := &a.items[i]
	size := it.size
	if payload, ok := a.overlay[i]; ok {
		size = int32(len(payload))
	}
	a.cacheMu.Lock()
	name := a.resolveStringLocked(it.nameOff)
	path := a.resolveStringLocked(it.pathOff)
	a.cacheMu.Unlock()
	return File{
		Index:      i,
		Name:       name,
		Path:       path,
		ChunkIndex: it.chunk,
		DataOffset: it.off,
		DataSize:   size,
		DataType:   it.typ,
	}
}

// FullPath returns the canonical "dir/name" path of entry i.
func (a *Archive) FullPath(i int32) string {
	it := &a.items[i]
	a.cacheMu.Lock()
	dir := a.resolveStringLocked(it.pathOff)
	name := a.resolveStringLocked(it.nameOff)
	a.cacheMu.Unlock()
	return joinPath(dir, name)
}

// Path is an alias of FullPath.
func (a *Archive) Path(i int32) string { return a.FullPath(i) }

// Find looks up an entry by "dir/name" path (case-insensitive).
func (a *Archive) Find(path string) (int32, bool) {
	return a.findNormalizedPath(normalizePath(path))
}

// FindList looks up a `.lst` index across client layouts. The 90US clients keep
// a list next to the files it indexes (`equipment/equipment.lst`), while the
// 110US clients collect every list under `list/` (`list/equipment.lst`) and
// store archive-root-relative entry paths inside it. Both layouts are tried in
// that order, so a configured 90US path also resolves on a newer archive.
func (a *Archive) FindList(path string) (int32, bool) {
	for _, candidate := range listLookupCandidates(path) {
		if i, ok := a.Find(candidate); ok {
			return i, true
		}
	}
	return 0, false
}

func listLookupCandidates(path string) []string {
	trimmed := strings.Trim(strings.ReplaceAll(strings.TrimSpace(path), "\\", "/"), "/")
	if trimmed == "" {
		return nil
	}
	candidates := []string{trimmed}
	base := trimmed
	if slash := strings.LastIndexByte(base, '/'); slash >= 0 {
		base = base[slash+1:]
	}
	if base == "" || strings.EqualFold(base, trimmed) {
		return candidates
	}
	candidates = append(candidates, "list/"+base)
	// A configured `list/x.lst` also names the 90US location `<stem>/x.lst`,
	// e.g. `list/equipment.lst` -> `equipment/equipment.lst`.
	if dir := trimmed[:strings.LastIndexByte(trimmed, '/')]; strings.EqualFold(dir, "list") {
		stem := strings.TrimSuffix(base, pathExt(base))
		if stem != "" {
			candidates = append(candidates, stem+"/"+base)
		}
	}
	return candidates
}

func pathExt(name string) string {
	if dot := strings.LastIndexByte(name, '.'); dot > 0 {
		return name[dot:]
	}
	return ""
}

// Chunk returns decompressed chunk ci, caching the result.
func (a *Archive) Chunk(ci int32) ([]byte, error) {
	var started time.Time
	if buildmode.Development {
		started = time.Now()
	}
	a.cacheMu.Lock()
	defer a.cacheMu.Unlock()
	cached := false
	bytes := 0
	if buildmode.Development {
		wait := time.Since(started)
		defer func() {
			if elapsed := time.Since(started); elapsed >= 50*time.Millisecond {
				log.Printf("[DEBUG] pvf.Chunk chunk=%d cache-hit=%t bytes=%d cache-lock-wait=%s work=%s total=%s",
					ci, cached, bytes, wait, elapsed-wait, elapsed)
			}
		}()
	}
	if ch, ok := a.chunkCache[ci]; ok {
		cached, bytes = true, len(ch)
		a.chunkCacheClock++
		entry := a.chunkCacheMeta[ci]
		entry.used = a.chunkCacheClock
		a.chunkCacheMeta[ci] = entry
		return ch, nil
	}
	raw, err := a.decompressChunk(ci)
	bytes = len(raw)
	if err != nil {
		return nil, err
	}
	if raw == nil {
		return nil, nil
	}
	if a.chunkCacheMeta == nil {
		a.chunkCacheMeta = make(map[int32]chunkCacheEntry)
	}
	a.chunkCacheClock++
	if int64(len(raw)) <= defaultChunkCacheLimit {
		for a.chunkCacheBytes+int64(len(raw)) > defaultChunkCacheLimit && len(a.chunkCache) > 0 {
			var oldest int32
			var oldestUsed uint64
			first := true
			for index, entry := range a.chunkCacheMeta {
				if first || entry.used < oldestUsed {
					oldest, oldestUsed, first = index, entry.used, false
				}
			}
			if first {
				break
			}
			delete(a.chunkCache, oldest)
			if entry, ok := a.chunkCacheMeta[oldest]; ok {
				a.chunkCacheBytes -= entry.bytes
			}
			delete(a.chunkCacheMeta, oldest)
		}
		a.chunkCache[ci] = raw
		a.chunkCacheMeta[ci] = chunkCacheEntry{bytes: int64(len(raw)), used: a.chunkCacheClock}
		a.chunkCacheBytes += int64(len(raw))
	}
	return raw, nil
}

// chunkSpan returns the raw (still encrypted) body byte range of chunk ci.
func (a *Archive) chunkSpan(ci int32) ([]byte, bool) {
	if ci < 0 || ci >= int32(len(a.groups)) {
		return nil, false
	}
	prev := int64(0)
	if ci > 0 {
		prev = int64(a.groups[ci-1].compSize)
	}
	start := a.bodyOff + int(prev)
	end := a.bodyOff + int(a.groups[ci].compSize)
	if start < a.bodyOff || start > end || end > len(a.data) {
		return nil, false
	}
	return a.data[start:end], true
}

// normalizePath mirrors the reference implementation's path normalization.
func normalizePath(p string) string {
	p = strings.ReplaceAll(p, "\\", "/")
	p = strings.TrimSpace(p)
	for strings.HasPrefix(p, "./") || strings.HasPrefix(p, "/") {
		if strings.HasPrefix(p, "./") {
			p = p[2:]
		} else {
			p = p[1:]
		}
	}
	return strings.ToLower(strings.TrimRight(p, "/"))
}

// joinPath joins raw pool strings for display. No normalization: entries
// may legitimately contain odd whitespace (the reference implementation
// preserves it and only normalizes lookup keys).
func joinPath(dir, name string) string {
	switch {
	case dir == "":
		return name
	case name == "":
		return dir
	default:
		return dir + "/" + name
	}
}

// ArchiveInfoView is a UI-facing snapshot of the archive state. Paged110 is a
// legacy wire field for the shared paged capability, not a client version.
type ArchiveInfoView struct {
	Path              string            `json:"path"`
	FileCount         int32             `json:"fileCount"`
	GroupCount        int32             `json:"groupCount"`
	BodySize          int32             `json:"bodySize"`
	ModifiedCount     int               `json:"modifiedCount"`
	UsesGuard         bool              `json:"usesGuard"`
	Paged110          bool              `json:"paged110"`
	Format            string            `json:"format"`
	ContentRules      ContentRules      `json:"contentRules"`
	WriteCapabilities WriteCapabilities `json:"writeCapabilities"`
	TextEncoding      string            `json:"textEncoding"`
	TextEncodingLabel string            `json:"textEncodingLabel"`
}

// Info returns the current state snapshot.
func (a *Archive) Info() ArchiveInfoView {
	return ArchiveInfoView{
		TextEncoding:      a.TextEncoding(),
		TextEncodingLabel: a.TextEncodingLabel(),
		Path:              a.sourcePath,
		FileCount:         int32(len(a.items)),
		GroupCount:        a.hdr.GroupCount,
		BodySize:          a.hdr.BodySize,
		ModifiedCount:     a.ModifiedCount(),
		UsesGuard:         a.guard,
		Paged110:          a.IsPaged(),
		Format:            a.Format(),
		ContentRules:      a.ContentRules(),
		WriteCapabilities: a.WriteCapabilities(),
	}
}