90CNServer/pvf/services/sqlite_index_test.go

package services

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"path/filepath"
	"strings"
	"testing"
	"time"

	_ "modernc.org/sqlite"
	sqlite3 "modernc.org/sqlite/lib"
	"pvfine/internal/pvf"
)

type sqliteBackupTestError int

func (e sqliteBackupTestError) Error() string { return fmt.Sprintf("SQLite code %d", e) }
func (e sqliteBackupTestError) Code() int     { return int(e) }

func TestStepSQLiteBackupRetriesLocks(t *testing.T) {
	for _, code := range []int{sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED, sqlite3.SQLITE_BUSY_RECOVERY, sqlite3.SQLITE_LOCKED_SHAREDCACHE} {
		for _, wantMore := range []bool{false, true} {
			t.Run(fmt.Sprintf("code=%d/more=%t", code, wantMore), func(t *testing.T) {
				calls := 0
				more, err := stepSQLiteBackup(context.Background(), func(pages int32) (bool, error) {
					if pages != 512 {
						t.Fatalf("pages = %d, want 512", pages)
					}
					calls++
					if calls <= 2 {
						return false, fmt.Errorf("backup: %w", sqliteBackupTestError(code))
					}
					return wantMore, nil
				})
				if err != nil || more != wantMore || calls != 3 {
					t.Fatalf("more = %t, err = %v, calls = %d", more, err, calls)
				}
			})
		}
	}
}

func TestStepSQLiteBackupDoesNotRetryOtherErrors(t *testing.T) {
	for _, want := range []error{sqliteBackupTestError(sqlite3.SQLITE_IOERR), errors.New("database is locked")} {
		calls := 0
		_, err := stepSQLiteBackup(context.Background(), func(int32) (bool, error) {
			calls++
			return false, want
		})
		if !errors.Is(err, want) || calls != 1 {
			t.Fatalf("err = %v, calls = %d, want %v on first call", err, calls, want)
		}
	}
}

func TestStepSQLiteBackupCancellation(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	calls := 0
	_, err := stepSQLiteBackup(ctx, func(int32) (bool, error) {
		calls++
		cancel()
		return false, sqliteBackupTestError(sqlite3.SQLITE_BUSY_RECOVERY)
	})
	if !errors.Is(err, context.Canceled) || calls != 1 {
		t.Fatalf("err = %v, calls = %d", err, calls)
	}
	_, err = stepSQLiteBackup(ctx, func(int32) (bool, error) {
		t.Fatal("cancelled backup called Step")
		return false, nil
	})
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("err = %v, want cancellation", err)
	}
}

func TestStepSQLiteBackupLockTimeout(t *testing.T) {
	want := sqliteBackupTestError(sqlite3.SQLITE_BUSY_RECOVERY)
	calls := 0
	started := time.Now()
	_, err := stepSQLiteBackup(context.Background(), func(int32) (bool, error) {
		calls++
		return false, want
	})
	if !errors.Is(err, want) || calls < 2 || time.Since(started) < 5*time.Second {
		t.Fatalf("err = %v, calls = %d, elapsed = %s", err, calls, time.Since(started))
	}
}

func TestBackupSQLiteDatabaseWaitsForSourceLock(t *testing.T) {
	dir := t.TempDir()
	source, err := sql.Open("sqlite", filepath.Join(dir, "source.db"))
	if err != nil {
		t.Fatal(err)
	}
	defer source.Close()
	if _, err := source.Exec("CREATE TABLE data(value TEXT); INSERT INTO data VALUES('original')"); err != nil {
		t.Fatal(err)
	}
	// Pin the locked writer so the backup must acquire a separate connection.
	writer, err := source.Conn(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	defer writer.Close()
	if _, err := writer.ExecContext(context.Background(), "BEGIN EXCLUSIVE; UPDATE data SET value='committed'"); err != nil {
		t.Fatal(err)
	}
	defer writer.ExecContext(context.Background(), "ROLLBACK")
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	target := filepath.Join(dir, "backup.db")
	done := make(chan error, 1)
	go func() { done <- backupSQLiteDatabase(ctx, source, target) }()
	select {
	case err := <-done:
		t.Fatalf("backup returned before source lock release: %v", err)
	case <-time.After(150 * time.Millisecond):
	}
	_, commitErr := writer.ExecContext(context.Background(), "COMMIT")
	backupErr := <-done
	if commitErr != nil {
		t.Fatal(commitErr)
	}
	if backupErr != nil {
		t.Fatal(backupErr)
	}
	snapshot, err := sql.Open("sqlite", target)
	if err != nil {
		t.Fatal(err)
	}
	defer snapshot.Close()
	var value string
	if err := snapshot.QueryRow("SELECT value FROM data").Scan(&value); err != nil || value != "committed" {
		t.Fatalf("backup value = %q, err = %v", value, err)
	}
}

func TestSQLiteWildcardSearchBeyondCandidateBatch(t *testing.T) {
	db, err := sql.Open("sqlite", ":memory:")
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err := buildSQLiteFileIndex(db, pvf.New(), "wildcard-pagination"); err != nil {
		t.Fatal(err)
	}
	// More non-matching records than the old candidate batch for a 200-hit page.
	if _, err := db.Exec(`WITH RECURSIVE seq(n) AS (
		SELECT 1 UNION ALL SELECT n+1 FROM seq WHERE n<6500
	) INSERT INTO records(file_index,name,lower_name,record_id,lower_id,path,lower_path,category,list_path,size,data_type)
	SELECT n,'','','','','misc/'||n||'.txt','misc/'||n||'.txt','file','',0,0 FROM seq`); err != nil {
		t.Fatal(err)
	}
	want := []string{"equipment/character/项.equ", "equipment/character/链.equ"}
	for n, path := range want {
		if _, err := db.Exec(`INSERT INTO records(file_index,name,lower_name,record_id,lower_id,path,lower_path,category,list_path,size,data_type)
		VALUES(?,'','','','',?,?,'file','',0,0)`, 6501+n, path, path); err != nil {
			t.Fatal(err)
		}
	}
	index := &sqliteArchiveIndex{db: db}
	for _, exact := range []bool{false, true} {
		for _, pattern := range []string{"*.equ", "EQUIPMENT/*/?.EQU"} {
			for _, limit := range []int{1, 200} {
				t.Run(fmt.Sprintf("%s/exact=%t/limit=%d", pattern, exact, limit), func(t *testing.T) {
					cursor := 0
					var paths []string
					for page := 0; ; page++ {
						if page > len(want) {
							t.Fatal("pagination did not terminate")
						}
						result, err := index.search(pattern, cursor, limit, exact)
						if err != nil {
							t.Fatal(err)
						}
						if len(result.Hits) == 0 && result.NextCursor >= 0 {
							t.Fatal("empty intermediate wildcard page")
						}
						for _, hit := range result.Hits {
							paths = append(paths, hit.Path)
						}
						if result.NextCursor < 0 {
							break
						}
						if result.NextCursor <= cursor {
							t.Fatal("cursor did not advance")
						}
						cursor = result.NextCursor
					}
					if len(paths) != len(want) {
						t.Fatalf("paths = %v, want %v", paths, want)
					}
					for n := range want {
						if paths[n] != want[n] {
							t.Fatalf("paths = %v, want %v", paths, want)
						}
					}
				})
			}
		}
	}
	result, err := index.search("*.missing", 0, 200, false)
	if err != nil {
		t.Fatal(err)
	}
	if len(result.Hits) != 0 || result.NextCursor != -1 {
		t.Fatalf("unmatched wildcard result = %#v", result)
	}
}

func TestSQLiteArchiveIndexQueries(t *testing.T) {
	a := pvf.New()
	a.AddFile("equip/a.equ", []byte("[name]\n`A`"), pvf.TypeScript)
	a.AddFile("equip/b.equ", []byte("[name]\n`B`"), pvf.TypeScript)
	a.AddFile("text/readme.str", []byte{0}, pvf.TypeUnicode)

	db, err := sql.Open("sqlite", ":memory:")
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err := configureArchiveIndex(db, true); err != nil {
		t.Fatal(err)
	}
	if err := buildSQLiteFileIndex(db, a, "test"); err != nil {
		t.Fatal(err)
	}
	index := &sqliteArchiveIndex{db: db}
	roots, err := index.children("")
	if err != nil {
		t.Fatal(err)
	}
	for _, root := range roots {
		if root.Path == "" {
			t.Fatal("root sentinel leaked into children")
		}
	}
	if _, err := index.children("equip"); err != nil {
		t.Fatal(err)
	}
	node, err := index.resolve("equip/a.equ")
	if err != nil || node == nil || node.FileIndex != 0 {
		t.Fatalf("resolve = %#v, %v", node, err)
	}
	if _, _, err := index.buildSemantic(context.Background(), NewCore(), a, nil, 1); err != nil {
		t.Fatal(err)
	}
	result, err := index.search("equip", 0, 10, false)
	if err != nil {
		t.Fatal(err)
	}
	if len(result.Hits) != 2 {
		t.Fatalf("search hits = %d, want 2", len(result.Hits))
	}
}

func assertSQLiteFilePathQueryUsesIndex(t *testing.T, db *sql.DB) {
	t.Helper()
	rows, err := db.Query(`EXPLAIN QUERY PLAN SELECT name,path,file_index,size,data_type FROM files WHERE path=? ORDER BY file_index LIMIT 1`, "equip/missing.equ")
	if err != nil {
		t.Fatal(err)
	}
	defer rows.Close()
	var plan []string
	for rows.Next() {
		var id, parent, unused int
		var detail string
		if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
			t.Fatal(err)
		}
		plan = append(plan, detail)
	}
	if err := rows.Err(); err != nil {
		t.Fatal(err)
	}
	detail := strings.Join(plan, "\n")
	if !strings.Contains(detail, "SEARCH files USING INDEX files_path") ||
		strings.Contains(detail, "SCAN files") || strings.Contains(detail, "TEMP B-TREE") {
		t.Fatalf("path query must use indexed lookup without scanning or sorting: %s", detail)
	}
}

func TestSQLiteFilePathIndexQueriesAndBatchResolution(t *testing.T) {
	a := pvf.New()
	for i := 0; i < 100; i++ {
		if _, err := a.AddFileText(fmt.Sprintf("equip/%03d.equ", i), "[name]\n`A`", pvf.TypeScript); err != nil {
			t.Fatal(err)
		}
	}
	index, _, err := openSQLiteArchiveIndex(a)
	if err != nil {
		t.Fatal(err)
	}
	defer index.close()
	assertSQLiteFilePathQueryUsesIndex(t, index.db)
	node, err := index.resolve("equip/missing.equ")
	if err != nil || node != nil {
		t.Fatalf("missing path = %#v, %v", node, err)
	}
	// Duplicate archive paths remain legal; resolve must choose the lowest index.
	if _, err := index.db.Exec(`INSERT INTO files
SELECT 100,path,lower_path,name,lower_name,parent,size,data_type,change_kind FROM files WHERE file_index=99`); err != nil {
		t.Fatal(err)
	}
	node, err = index.resolve("equip/099.equ")
	if err != nil || node == nil || node.FileIndex != 99 {
		t.Fatalf("duplicate path = %#v, %v", node, err)
	}
	c := NewCore()
	c.archive, c.diskIndex = a, index
	paths := []string{"equip/missing.equ", "equip/099.equ"}
	for i := 0; i < 40; i++ {
		paths = append(paths, fmt.Sprintf("equip/%03d.equ", i))
	}
	paths = append(paths, "equip/099.equ")
	nodes, err := NewArchiveService(c).ResolveFiles(paths)
	if err != nil {
		t.Fatal(err)
	}
	if len(nodes) != 41 || nodes[0].FileIndex != 99 || nodes[40].FileIndex != 39 {
		t.Fatalf("batch resolution returned unexpected results: %#v", nodes)
	}
}

func TestSQLiteFilePathIndexUpgradesCachedArchive(t *testing.T) {
	cacheDir := t.TempDir()
	t.Setenv("LocalAppData", cacheDir)
	t.Setenv("XDG_CACHE_HOME", cacheDir)
	t.Setenv("HOME", cacheDir)
	a := pvf.New()
	if _, err := a.AddFileText("equip/a.equ", "[name]\n`A`", pvf.TypeScript); err != nil {
		t.Fatal(err)
	}
	path := filepath.Join(t.TempDir(), "cache.pvf")
	if err := a.SaveAs(path); err != nil {
		t.Fatal(err)
	}
	a, err := pvf.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer a.Release()
	index, hit, err := openSQLiteArchiveIndex(a)
	if err != nil {
		t.Fatal(err)
	}
	if hit {
		index.close()
		t.Fatal("fresh archive unexpectedly hit the cache")
	}
	index.close()
	// Working indexes are private. Simulate the old format in the persisted
	// cache itself, rather than modifying a session snapshot.
	cachePath, _, err := archiveIndexCachePath(a)
	if err != nil {
		t.Fatal(err)
	}
	cache, err := openArchiveIndexDatabase(cachePath, false)
	if err != nil {
		t.Fatal(err)
	}
	// Simulate a complete cache written before the path index was introduced.
	if _, err := cache.Exec(`DROP INDEX files_path;
INSERT INTO records(file_index,name,lower_name,record_id,lower_id,path,lower_path,category,list_path,size,data_type)
VALUES(0,'preserved','preserved','42','42','equip/a.equ','equip/a.equ','equipment','equip/equipment.lst',0,1);
UPDATE meta SET value='1' WHERE key='complete';`); err != nil {
		cache.Close()
		t.Fatal(err)
	}
	if err := cache.Close(); err != nil {
		t.Fatal(err)
	}
	for pass := 0; pass < 2; pass++ {
		index, hit, err = openSQLiteArchiveIndex(a)
		if err != nil {
			t.Fatal(err)
		}
		func() {
			defer index.close()
			if !hit || !index.ready {
				t.Fatal("upgrade must preserve the complete semantic cache")
			}
			if index.path == cachePath || index.tempDir == "" {
				t.Fatal("cache must be private before the first edit")
			}
			assertSQLiteFilePathQueryUsesIndex(t, index.db)
			var name string
			if err := index.db.QueryRow("SELECT name FROM records WHERE record_id='42'").Scan(&name); err != nil || name != "preserved" {
				t.Fatalf("semantic cache changed: name=%q error=%v", name, err)
			}
			node, err := index.resolve("equip/a.equ")
			if err != nil || node == nil || node.FileIndex != 0 {
				t.Fatalf("resolve after upgrade = %#v, %v", node, err)
			}
		}()
	}
}

func TestSQLiteFilePathIndexUpgradeCancellation(t *testing.T) {
	db, err := sql.Open("sqlite", ":memory:")
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err := buildSQLiteFileIndex(db, pvf.New(), "cancel-path-index"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("DROP INDEX files_path"); err != nil {
		t.Fatal(err)
	}
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if err := ensureSQLiteFilePathIndex(ctx, db); !errors.Is(err, context.Canceled) {
		t.Fatalf("upgrade error = %v, want cancellation", err)
	}
	if err := ensureSQLiteFilePathIndex(context.Background(), db); err != nil {
		t.Fatal(err)
	}
	assertSQLiteFilePathQueryUsesIndex(t, db)
}

func TestSQLiteArchiveIndexPublishesSemanticCandidate(t *testing.T) {
	a := pvf.New()
	a.AddFile("equip/a.equ", []byte("[name]\n`A`"), pvf.TypeScript)

	index, _, err := openSQLiteArchiveIndex(a)
	if err != nil {
		t.Fatal(err)
	}
	defer index.close()
	c := NewCore()
	c.mu.Lock()
	c.archive = a
	c.diskIndex = index
	c.indexGen = 1
	c.mu.Unlock()
	oldDB := index.db
	if _, _, err := index.buildSemantic(context.Background(), c, a, nil, 1); err != nil {
		t.Fatal(err)
	}
	if index.db == oldDB {
		t.Fatal("semantic rebuild reused the old SQLite connection")
	}
	if result, err := index.search("equip", 0, 10, false); err != nil || len(result.Hits) != 1 {
		t.Fatalf("search after reopen = %#v, %v", result, err)
	}
}

func TestSQLiteArchiveIndexConfiguresEveryConnection(t *testing.T) {
	a := pvf.New()
	a.AddFile("equip/a.equ", []byte("[name]\n`A`"), pvf.TypeScript)
	index, _, err := openSQLiteArchiveIndex(a)
	if err != nil {
		t.Fatal(err)
	}
	defer index.close()
	checkConnections := func(t *testing.T) {
		for n := 0; n < 3; n++ {
			conn, err := index.db.Conn(context.Background())
			if err != nil {
				t.Fatal(err)
			}
			// Keep each connection pinned until all pool slots are checked.
			defer conn.Close()
			for pragma, want := range map[string]int{
				"busy_timeout": 5000,
				"foreign_keys": 1,
				"temp_store":   1,
				"cache_size":   -8192,
				"mmap_size":    0,
				"synchronous":  1,
			} {
				var got int
				if err := conn.QueryRowContext(context.Background(), "PRAGMA "+pragma).Scan(&got); err != nil {
					t.Fatal(err)
				}
				if got != want {
					t.Errorf("connection %d: %s = %d, want %d", n, pragma, got, want)
				}
			}
		}
	}
	t.Run("opened", checkConnections)
	c := NewCore()
	c.archive = a
	c.diskIndex = index
	c.indexGen = 1
	if _, _, err := index.buildSemantic(context.Background(), c, a, nil, 1); err != nil {
		t.Fatal(err)
	}
	t.Run("published", checkConnections)
}

func TestSQLiteArchiveIndexNewReaderWaitsForLock(t *testing.T) {
	db, err := openArchiveIndexDatabase(filepath.Join(t.TempDir(), "index.db"), false)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	if err := configureArchiveIndex(db, false); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE data(value TEXT); INSERT INTO data VALUES('original')"); err != nil {
		t.Fatal(err)
	}
	writer, err := db.Conn(context.Background())
	if err != nil {
		t.Fatal(err)
	}
	defer writer.Close()
	if _, err := writer.ExecContext(context.Background(), "BEGIN EXCLUSIVE; UPDATE data SET value='committed'"); err != nil {
		t.Fatal(err)
	}
	defer writer.ExecContext(context.Background(), "ROLLBACK")
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	done := make(chan error, 1)
	go func() {
		var value string
		err := db.QueryRowContext(ctx, "SELECT value FROM data").Scan(&value)
		if err == nil && value != "committed" {
			err = fmt.Errorf("value = %q, want committed", value)
		}
		done <- err
	}()
	select {
	case err := <-done:
		t.Fatalf("reader returned before source lock release: %v", err)
	case <-time.After(150 * time.Millisecond):
	}
	_, commitErr := writer.ExecContext(context.Background(), "COMMIT")
	queryErr := <-done
	if commitErr != nil {
		t.Fatal(commitErr)
	}
	if queryErr != nil {
		t.Fatal(queryErr)
	}
}