90CNServer/pvf/services/advanced_search_delta.go

package services

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"pvfine/internal/pvf"
)

var errAdvancedPoolChanged = errors.New("字符串池偏移代际不兼容")

func (d *advancedSQLite) requestContext() context.Context {
	if d.requestCtx != nil {
		return d.requestCtx
	}
	return d.ctx
}

func (d *advancedSQLite) captureBaseline(c *core, a *pvf.Archive) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.advancedDisk != d || c.archive != a || d.ctx.Err() != nil {
		return context.Canceled
	}
	d.entries = archiveEntryIndexes(a)
	d.poolState = a.StringPoolState()
	d.indexedEpoch = d.epoch
	d.published.Store(true)
	return nil
}

// Only the metadata/pool database becomes writable. Immutable, potentially
// gigabyte-sized reference shards and cache hard links are never modified.
func (d *advancedSQLite) prepareOverlay() error {
	if d.mutable {
		return nil
	}
	ctx := d.requestContext()
	tmp, err := os.CreateTemp(d.dir, ".delta-*.db")
	if err != nil {
		return err
	}
	path := tmp.Name()
	_ = tmp.Close()
	defer os.Remove(path)
	if err = backupSQLiteDatabase(ctx, d.db, path); err != nil {
		return err
	}
	db, err := openArchiveIndexDatabase(path, false)
	if err != nil {
		return err
	}
	if err = configureArchiveIndex(db, false); err != nil {
		_ = db.Close()
		return err
	}
	if !d.overlay {
		_, err = db.ExecContext(ctx, `CREATE TABLE file_map(orig_index INTEGER PRIMARY KEY,file_index INTEGER UNIQUE,overridden INTEGER NOT NULL DEFAULT 0);
INSERT INTO file_map(orig_index,file_index) SELECT file_index,file_index FROM files;
CREATE TABLE delta_refs(file_index INTEGER,offset INTEGER,occurrences INTEGER,types INTEGER,fields INTEGER,PRIMARY KEY(file_index,offset)) WITHOUT ROWID;
CREATE INDEX delta_refs_offset ON delta_refs(offset,file_index)`)
	}
	_ = db.Close()
	if err != nil {
		return err
	}
	if err = d.db.Close(); err != nil {
		return err
	}
	mainPath := filepath.Join(d.dir, "index.db")
	if err = os.Rename(path, mainPath); err != nil {
		d.db, _ = openAdvancedReadDB(mainPath)
		return err
	}
	d.overlay = true
	d.db, err = openArchiveIndexDatabase(mainPath, false)
	if err == nil {
		err = configureArchiveIndex(d.db, false)
	}
	if err == nil {
		err = enableArchiveIndexWAL(d.db)
	}
	if err != nil {
		return err
	}
	d.overlay, d.mutable, d.cachePath = true, true, ""
	return nil
}

type advancedDeltaFile struct {
	index int32
	path  string
	file  pvf.File
	refs  []pvf.StringReference
}

func (d *advancedSQLite) syncDelta(c *core, a *pvf.Archive, epoch uint64) error {
	if d.db == nil {
		return errors.New("高级搜索投影连接不可用")
	}
	ctx := d.requestContext()
	c.mu.RLock()
	if c.advancedDisk != d || c.archive != a || d.epoch != epoch || ctx.Err() != nil {
		c.mu.RUnlock()
		return ErrAdvancedCursorStale
	}
	revision := c.batchRevision
	ids := make(map[uint64]struct{}, len(d.pendingIDs))
	for id := range d.pendingIDs {
		ids[id] = struct{}{}
	}
	snapshot := a.CloneForBatch()
	c.mu.RUnlock()
	if !snapshot.ExtendsStringPools(d.poolState) {
		return errAdvancedPoolChanged
	}
	next := archiveEntryIndexes(snapshot)
	oldByID := make(map[uint64]int32, len(d.entries))
	for index, id := range d.entries {
		oldByID[id] = int32(index)
	}
	retained := make([]bool, len(d.entries))
	moved := make(map[int32]int32)
	var changedIndexes []int32
	for index, id := range next {
		oldIndex, exists := oldByID[id]
		if exists {
			retained[oldIndex] = true
			if oldIndex != int32(index) {
				moved[oldIndex] = int32(index)
			}
		}
		_, changed := ids[id]
		if !exists || changed {
			changedIndexes = append(changedIndexes, int32(index))
		}
	}
	var changes []advancedDeltaFile
	scanner := snapshot.NewSelectedStringReferenceScanner(changedIndexes)
	for {
		index, refs, ok, err := scanner.Next(ctx)
		if err != nil {
			return err
		}
		if !ok {
			break
		}
		changes = append(changes, advancedDeltaFile{index, snapshot.Path(index), snapshot.File(index), refs})
	}
	var appended []pvf.StringPoolEntry
	pools := snapshot.NewAppendedStringPoolScanner(d.poolState)
	for {
		entry, ok, err := pools.Next(ctx)
		if err != nil {
			return err
		}
		if !ok {
			break
		}
		appended = append(appended, entry)
	}
	// Resolve only existing placeholder strings, not every pool value.
	resolved := make(map[int32]string)
	rows, err := d.db.QueryContext(ctx, `SELECT offset,value FROM pool WHERE instr(value,'<')>0 AND instr(value,'::')>0`)
	if err != nil {
		return err
	}
	for rows.Next() {
		var offset int32
		var value string
		if err = rows.Scan(&offset, &value); err != nil {
			break
		}
		text := snapshot.ResolvePlaceholders(value)
		if text == value {
			text = ""
		}
		resolved[offset] = text
	}
	if err == nil {
		err = rows.Err()
	}
	_ = rows.Close()
	if err != nil {
		return err
	}
	state := snapshot.StringPoolState()
	if err = d.prepareOverlay(); err != nil {
		return err
	}
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.advancedDisk != d || c.archive != a || d.epoch != epoch || c.batchRevision != revision || ctx.Err() != nil {
		return ErrAdvancedCursorStale
	}
	tx, err := d.db.BeginTx(ctx, nil)
	if err != nil {
		return err
	}
	defer tx.Rollback()
	if _, err = tx.ExecContext(ctx, `CREATE TEMP TABLE advance_map(old_index INTEGER PRIMARY KEY,new_index INTEGER)`); err != nil {
		return err
	}
	mapStmt, err := tx.PrepareContext(ctx, `INSERT INTO advance_map VALUES(?,?)`)
	if err != nil {
		return err
	}
	defer mapStmt.Close()
	for oldIndex, newIndex := range moved {
		if _, err = mapStmt.ExecContext(ctx, oldIndex, newIndex); err != nil {
			return err
		}
	}
	for index, keep := range retained {
		if keep {
			continue
		}
		for _, table := range []string{"files", "file_map", "delta_refs"} {
			if _, err = tx.ExecContext(ctx, `DELETE FROM `+table+` WHERE file_index=?`, index); err != nil {
				return err
			}
		}
	}
	for _, table := range []string{"files", "file_map", "delta_refs"} {
		if _, err = tx.ExecContext(ctx, `UPDATE `+table+` SET file_index=-file_index-1 WHERE file_index IN (SELECT old_index FROM advance_map)`); err != nil {
			return err
		}
		if _, err = tx.ExecContext(ctx, `UPDATE `+table+` SET file_index=(SELECT new_index FROM advance_map WHERE old_index=-file_index-1) WHERE file_index<0`); err != nil {
			return err
		}
	}
	refStmt, err := tx.PrepareContext(ctx, `INSERT INTO delta_refs VALUES(?,?,?,?,?)`)
	if err != nil {
		return err
	}
	defer refStmt.Close()
	for _, change := range changes {
		if _, err = tx.ExecContext(ctx, `INSERT INTO files VALUES(?,?,?,?,?) ON CONFLICT(file_index) DO UPDATE SET path=excluded.path,lower_path=excluded.lower_path,size=excluded.size,data_type=excluded.data_type`, change.index, change.path, strings.ToLower(change.path), change.file.DataSize, change.file.DataType); err != nil {
			return err
		}
		if _, err = tx.ExecContext(ctx, `UPDATE file_map SET overridden=1 WHERE file_index=?`, change.index); err != nil {
			return err
		}
		if _, err = tx.ExecContext(ctx, `DELETE FROM delta_refs WHERE file_index=?`, change.index); err != nil {
			return err
		}
		for _, ref := range change.refs {
			types, fields := 0, 0
			for n, token := range ref.TokenTypes {
				types |= int(token) << (n * 4)
			}
			for _, field := range ref.FileFields {
				if field == "name" {
					fields |= 1
				}
				if field == "path" {
					fields |= 2
				}
			}
			if _, err = refStmt.ExecContext(ctx, change.index, ref.Offset, ref.Occurrences, types, fields); err != nil {
				return err
			}
		}
	}
	for _, entry := range appended {
		text := snapshot.ResolvePlaceholders(entry.Value)
		if text == entry.Value {
			text = ""
		}
		if _, err = tx.ExecContext(ctx, `INSERT INTO pool VALUES(?,?,?,?)`, entry.Offset, entry.Pool, entry.Value, text); err != nil {
			return err
		}
	}
	for offset, text := range resolved {
		if _, err = tx.ExecContext(ctx, `UPDATE pool SET resolved=? WHERE offset=?`, text, offset); err != nil {
			return err
		}
	}
	if _, err = tx.ExecContext(ctx, `DROP TABLE advance_map`); err != nil {
		return err
	}
	if err = tx.Commit(); err != nil {
		return err
	}
	for _, q := range d.queries {
		_ = os.Remove(q.path)
	}
	d.queries = make(map[uint64]*advancedDiskQuery)
	d.entries, d.poolState, d.indexedEpoch, d.pendingIDs = next, state, epoch, nil
	c.advancedStatus = AdvancedSearchIndexStatus{State: AdvancedIndexStateReady, Stage: "ready-delta", Done: len(changes), Total: int(snapshot.FileCount())}
	developmentLog("[pvfine:advanced] sqlite delta files=%d remapped=%d appended_strings=%d", len(changes), len(moved), len(appended))
	return nil
}