90CNServer/server/internal/services/dnfbridge/repro_20raid_test.go

package dnfbridge

// repro_20raid_test.go — 20-client raid harness.
//
// WHY THIS FILE EXISTS
//
// The existing stress fixtures (stress_raid_projection_test.go) build sessions
// WITHOUT an event loop. postGameSessionCharacterEvent then takes its
// `session.events == nil` shortcut and runs the closure inline, so those tests
// measure the projection alone and can never observe the delivery coupling.
//
// This harness builds a raid whose members each own a REAL eventloop.Loop and a
// REAL writer goroutine (startGameSessionEvents), backed by a socket whose Write
// can be throttled or stalled. That is what makes it possible to ask the two
// questions the live incident left open:
//
//	A) how much does one scene-boundary fan-out cost at 20 members, and
//	B) does one stalled client make OTHER members stop working?
//
// Nothing in this file is production code.

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"sync"
	"testing"
	"time"

	"longheng.io/server/internal/modules/dnf/alignedcmd"
	"longheng.io/server/internal/modules/dnf/channelcatalog"
	dnfrepo "longheng.io/server/internal/modules/dnf/repository"
	dnfrepomemory "longheng.io/server/internal/modules/dnf/repository/memory"
)

// ---------------------------------------------------------------------------
// A socket whose Write can be throttled or stalled.
// ---------------------------------------------------------------------------

type reproConn struct {
	*bufferConn
	name string

	// remote, when set, is the authenticated IPv4 TCP tuple this simulated
	// client dialled from. Party formation keys every relay and publish decision
	// on it (currentPartySessionRemoteAddress), so a simulation of the party
	// paths needs a real *net.TCPAddr instead of the fixture's synthetic name.
	remote *net.TCPAddr

	mu      sync.Mutex
	writes  int64
	bytes   int64
	stalled bool
	// stallUntil: while now < stallUntil every Write parks. This models the
	// client that stopped reading, which is what makes the writer goroutine
	// (and therefore that session's loop) slow.
	stallUntil time.Time
}

func (c *reproConn) RemoteAddr() net.Addr {
	if c.remote != nil {
		return c.remote
	}
	return c.bufferConn.RemoteAddr()
}

func (c *reproConn) LocalAddr() net.Addr {
	if c.remote != nil {
		return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 10238}
	}
	return c.bufferConn.LocalAddr()
}

func (c *reproConn) Write(data []byte) (int, error) {
	for {
		c.mu.Lock()
		until := c.stallUntil
		c.mu.Unlock()
		if until.IsZero() || !time.Now().Before(until) {
			break
		}
		time.Sleep(2 * time.Millisecond)
	}
	c.mu.Lock()
	c.writes++
	c.bytes += int64(len(data))
	c.mu.Unlock()
	return c.bufferConn.Write(data)
}

func (c *reproConn) stallFor(d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.stalled = true
	c.stallUntil = time.Now().Add(d)
}

func (c *reproConn) release() {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.stalled = false
	c.stallUntil = time.Time{}
}

func (c *reproConn) counters() (writes int64, bytes int64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.writes, c.bytes
}

// ---------------------------------------------------------------------------
// Fixture
// ---------------------------------------------------------------------------

const reproChannelID = 200

type reproMember struct {
	index   int
	charID  uint16
	session *gameSession
	conn    *reproConn
}

type reproRaid struct {
	t       *testing.T
	service *Service
	repos   dnfrepo.Group
	ctx     context.Context
	cancel  context.CancelFunc
	raidKey uint32
	members []reproMember
}

// newReproRaid builds memberCount members, each with a real session loop, real
// writer goroutine and a stallable socket, and registers them as one started
// raid roster so the raid-wide fan-out paths run.
func newReproRaid(t *testing.T, memberCount int) *reproRaid {
	t.Helper()
	if memberCount < 2 {
		t.Fatalf("a raid of %d members cannot exercise the fan-out", memberCount)
	}
	ctx, cancel := context.WithCancel(context.Background())
	repos := dnfrepomemory.NewMemoryGroup()
	service := &Service{
		options:            options{gameUpperBodyCodec: gameUpperBodyCodecPlain},
		onlinePlayers:      newOnlinePlayerManager(),
		raids:              make(map[uint32]*runtimeRaidState),
		repositoryProvider: func() (dnfrepo.Group, bool) { return repos, true },
	}
	channel := channelcatalog.Channel{ID: reproChannelID, Type: 23, Name: "anton", Port: 10238}
	raid := &reproRaid{
		t:       t,
		service: service,
		repos:   repos,
		ctx:     ctx,
		cancel:  cancel,
		raidKey: uint32(0x00c8b000) + uint32(memberCount),
		members: make([]reproMember, 0, memberCount),
	}

	roster := make([]runtimeRaidMemberState, 0, memberCount)
	for index := 0; index < memberCount; index++ {
		charID := uint16(3001 + index)
		id := strconv.Itoa(int(charID))
		// Reuse the existing seeder: a complete structured snapshot so the
		// measured path never enters the character-initialization backfill.
		stressRaidSeedCharacter(t, repos, charID, index)
		conn := &reproConn{
			bufferConn: &bufferConn{},
			name:       "repro-" + id,
			// A distinct IPv4 TCP tuple per simulated client: party formation keys
			// the relay legs and the direct endpoint table on it.
			remote: &net.TCPAddr{
				IP:   net.IPv4(127, 0, 0, byte(1+index%200)),
				Port: 41000 + index,
			},
		}
		session := &gameSession{
			conn:                            conn,
			connID:                          conn.name,
			accountID:                       "repro-account-" + id,
			channel:                         channel,
			townActorOwnerChannel:           byte(channel.ID),
			connectionTownActorOwnerChannel: byte(channel.ID),
			selectedCharacterID:             charID,
			residentChannel:                 channel,
			ctx:                             ctx,
		}
		service.bindGameSessionCharacter(session, charID)
		if err := service.startGameSessionEvents(ctx, session); err != nil {
			t.Fatalf("start session events for char %d: %v", charID, err)
		}
		roster = append(roster, runtimeRaidMemberState{
			CharID:     charID,
			Name:       "repro" + id,
			Level:      90,
			GroupIndex: byte(index/runtimeRaidGroupSize) + 1,
			SlotOrder:  byte(index % runtimeRaidGroupSize),
			UserState:  1,
			HPPercent:  100,
			MPPercent:  100,
		})
		raid.members = append(raid.members, reproMember{
			index:   index,
			charID:  charID,
			session: session,
			conn:    conn,
		})
	}

	service.raids[raid.raidKey] = &runtimeRaidState{
		RaidKey:        raid.raidKey,
		LeaderID:       roster[0].CharID,
		NameBytes:      []byte("anton"),
		TailFlag:       5,
		Members:        append([]runtimeRaidMemberState(nil), roster...),
		FrozenMembers:  append([]runtimeRaidMemberState(nil), roster...),
		Started:        true,
		EntryCostPaid:  true,
		StartAttempt:   1,
		ChargedMembers: runtimeRaidMemberIDs(runtimeRaidState{Members: roster}),
		Lifecycle:      newRuntimeRaidLifecycleState(time.Now().UTC()),
	}
	return raid
}

func (r *reproRaid) close() {
	if r == nil {
		return
	}
	for _, member := range r.members {
		if member.conn != nil {
			member.conn.release()
		}
		if member.session != nil && member.session.events != nil {
			shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
			_ = r.service.stopGameSessionEvents(shutdownCtx, member.session)
			r.service.stopGameSessionWriter(shutdownCtx, member.session)
			cancel()
		}
	}
	r.cancel()
}

func (r *reproRaid) state() runtimeRaidState {
	return cloneRuntimeRaidState(*r.service.raids[r.raidKey])
}

func (r *reproRaid) totalWrites() int64 {
	var total int64
	for _, member := range r.members {
		writes, _ := member.conn.counters()
		total += writes
	}
	return total
}

// queueDepths reports every member's event-loop queue depth, so a fan-out that
// cannot be drained shows up as a growing queue rather than as a slower clock.
func (r *reproRaid) queueDepths() []int {
	out := make([]int, 0, len(r.members))
	for _, member := range r.members {
		if member.session.events == nil || member.session.events.loop == nil {
			out = append(out, -1)
			continue
		}
		lengths := member.session.events.loop.QueueLengths()
		out = append(out, lengths.Medium+lengths.High+lengths.Low)
	}
	return out
}

// runBoundaryFrom drives one scene-boundary fan-out FROM a member's own event
// loop, which is what production does (the lifecycle boundary runs on the
// session that reported it). Returns how long the call itself took.
func (r *reproRaid) runBoundaryFrom(caller *reproMember, source string) time.Duration {
	state := r.state()
	start := time.Now()
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
	defer cancel()
	err := r.service.callGameSession(ctx, caller.session, source, func() error {
		r.service.syncRuntimeRaidMateActorsAllAtSceneBoundary(state, source, false)
		return nil
	})
	if err != nil {
		r.t.Logf("boundary via %s returned %v after %s", caller.conn.name, err, time.Since(start))
	}
	return time.Since(start)
}

// canStillWork reports whether another member's loop is still able to run a
// trivial task within the budget. This is the "is everyone frozen" probe.
func (r *reproRaid) canStillWork(member *reproMember, budget time.Duration) bool {
	ctx, cancel := context.WithTimeout(context.Background(), budget)
	defer cancel()
	done := make(chan error, 1)
	go func() {
		done <- r.service.callGameSession(ctx, member.session, "repro-liveness", func() error { return nil })
	}()
	select {
	case err := <-done:
		return err == nil
	case <-time.After(budget + 200*time.Millisecond):
		return false
	}
}

// ---------------------------------------------------------------------------
// (A) What does one 20-member boundary cost when every client is healthy?
// ---------------------------------------------------------------------------

func TestRepro20RaidBoundaryCost(t *testing.T) {
	for _, memberCount := range []int{8, 20} {
		raid := newReproRaid(t, memberCount)
		// Warm the caches the way a second boundary in the same raid would.
		raid.runBoundaryFrom(&raid.members[0], "repro_warm_raid_mate_actors")
		time.Sleep(300 * time.Millisecond)

		samples := make([]time.Duration, 0, 3)
		for i := 0; i < 3; i++ {
			samples = append(samples, raid.runBoundaryFrom(&raid.members[0],
				fmt.Sprintf("repro_dungeon_loaded_raid_mate_actors_%d", i)))
		}
		var worst time.Duration
		for _, sample := range samples {
			if sample > worst {
				worst = sample
			}
		}
		time.Sleep(500 * time.Millisecond)
		t.Logf("N=%d healthy: boundary wall (caller's loop) worst=%s samples=%v queue_depths=%v writes=%d",
			memberCount, worst, samples, raid.queueDepths(), raid.totalWrites())
		raid.close()
	}
}

// ---------------------------------------------------------------------------
// (B) Does ONE stalled client make the raid stop?
// ---------------------------------------------------------------------------

// TestRepro20RaidOneStalledClient is the falsification test for the delivery
// coupling. It stalls exactly one member's socket, then drives a boundary from
// a different member's loop and asks two things:
//
//  1. does the caller's own loop come back (or does it park on the stalled peer)?
//  2. can an unrelated third member still do work while that is happening?
func TestRepro20RaidOneStalledClient(t *testing.T) {
	raid := newReproRaid(t, 20)
	raid.runBoundaryFrom(&raid.members[0], "repro_warm_raid_mate_actors")
	time.Sleep(500 * time.Millisecond)

	victim := &raid.members[7]
	caller := &raid.members[0]
	third := &raid.members[13]

	victim.conn.stallFor(5 * time.Second)
	t.Logf("stalled %s for 5s; driving a boundary from %s", victim.conn.name, caller.conn.name)

	start := time.Now()
	wall := raid.runBoundaryFrom(caller, "repro_stalled_boundary_raid_mate_actors")
	alive := raid.canStillWork(third, 2*time.Second)
	t.Logf("boundary returned after %s; third member (%s) still workable=%t; queue_depths=%v",
		wall, third.conn.name, alive, raid.queueDepths())

	// While the victim is still stalled, does a SECOND boundary from a healthy
	// caller also pay the penalty? That is the "one client freezes the raid"
	// prediction, and it must not reproduce once the coupling is removed.
	start2 := time.Now()
	wall2 := raid.runBoundaryFrom(third, "repro_stalled_boundary2_raid_mate_actors")
	t.Logf("second boundary (caller %s) returned after %s (elapsed since stall start %s)",
		third.conn.name, wall2, time.Since(start))

	victim.conn.release()
	time.Sleep(1 * time.Second)
	t.Logf("after release: canStillWork(victim)=%t queue_depths=%v",
		raid.canStillWork(victim, 2*time.Second), raid.queueDepths())
	_ = start2
	raid.close()
}

// ---------------------------------------------------------------------------
// (D) The SYNCHRONOUS fan-out: does the initiator's own loop stop?
// ---------------------------------------------------------------------------

// TestRepro20RaidSyncFanoutParksInitiator measures the property the 2026-09-24
// incident was blamed on: fanOutToGameSessions runs on the INITIATOR's event
// loop and wg.Wait()s for every member, so one slow member parks that loop --
// and with it every other request of that one character.
//
// memberCount targets, one of them slow. The caller is member[0]'s loop.
func TestRepro20RaidSyncFanoutParksInitiator(t *testing.T) {
	raid := newReproRaid(t, 20)
	time.Sleep(200 * time.Millisecond)

	const slowMember = 7
	const slowWork = 3 * time.Second

	targets := make([]sessionFanoutTarget, 0, len(raid.members))
	for i := range raid.members {
		targets = append(targets, sessionFanoutTarget{
			label:       raid.members[i].conn.name,
			characterID: raid.members[i].charID,
			session:     raid.members[i].session,
		})
	}

	work := func(target sessionFanoutTarget) error {
		if target.characterID == raid.members[slowMember].charID {
			time.Sleep(slowWork)
		} else {
			time.Sleep(10 * time.Millisecond)
		}
		return nil
	}

	caller := &raid.members[0]
	other := &raid.members[13]

	var fanoutWall time.Duration
	callerLoopBusy := make(chan bool, 1)
	done := make(chan struct{})
	go func() {
		defer close(done)
		start := time.Now()
		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()
		_ = raid.service.callGameSession(ctx, caller.session, "repro-sync-fanout", func() error {
			// Probe this loop's availability from another goroutine while the
			// fan-out is running: if the fan-out parks the loop, the probe
			// cannot be served until it finishes.
			go func() {
				time.Sleep(300 * time.Millisecond)
				callerLoopBusy <- !raid.canStillWork(caller, 500*time.Millisecond)
			}()
			return raid.service.fanOutToGameSessions(context.Background(), targets,
				"repro_party_dungeon_loading_release", work)
		})
		fanoutWall = time.Since(start)
	}()

	select {
	case <-done:
	case <-time.After(25 * time.Second):
		t.Fatalf("synchronous fan-out did not finish within 25s")
	}
	t.Logf("fanOutToGameSessions(20 targets, 1 slow=%s): caller loop was parked for %s",
		slowWork, fanoutWall)

	select {
	case parked := <-callerLoopBusy:
		t.Logf("initiator loop unresponsive during the fan-out = %t (expected true on the current design)", parked)
	case <-time.After(2 * time.Second):
		t.Logf("initiator loop probe never answered")
	}

	// The other 18 members must be unaffected: they only paid 10ms each.
	t.Logf("unrelated member %s still workable = %t", other.conn.name, raid.canStillWork(other, 2*time.Second))
	raid.close()
}

// ---------------------------------------------------------------------------
// (E) The ASYNC fan-out must not park anyone (regression guard for the fix).
// ---------------------------------------------------------------------------

// TestRepro20RaidAsyncFanoutDoesNotParkInitiator is the guard for the loading
// release fix. Same shape as test D (20 targets, one of them slow) but through
// fanOutToGameSessionsAsync, which is what the loading release now uses.
//
// The synchronous form parked the caller's loop for the whole 10s budget; the
// asynchronous form must return as soon as the work is handed over, and it must
// still run `then` only after every target finished.
func TestRepro20RaidAsyncFanoutDoesNotParkInitiator(t *testing.T) {
	raid := newReproRaid(t, 20)
	time.Sleep(200 * time.Millisecond)

	const slowMember = 7
	const slowWork = 1500 * time.Millisecond

	targets := make([]sessionFanoutTarget, 0, len(raid.members))
	for i := range raid.members {
		targets = append(targets, sessionFanoutTarget{
			label:       raid.members[i].conn.name,
			characterID: raid.members[i].charID,
			session:     raid.members[i].session,
		})
	}
	slowCharID := raid.members[slowMember].charID

	thenCalled := make(chan time.Duration, 1)
	start := time.Now()
	raid.service.fanOutToGameSessionsAsync(context.Background(), targets,
		"repro_party_dungeon_loading_release",
		func(target sessionFanoutTarget) error {
			if target.characterID == slowCharID {
				time.Sleep(slowWork)
			}
			return nil
		},
		func() { thenCalled <- time.Since(start) })

	// The submit itself must come straight back: that is the whole fix.
	handover := time.Since(start)
	if handover > 500*time.Millisecond {
		t.Fatalf("async fan-out parked the initiator for %s (must hand over immediately)", handover)
	}

	// `then` (the leader-last step) must still wait for the slow member.
	select {
	case took := <-thenCalled:
		if took < slowWork {
			t.Fatalf("then ran after %s, before the slow target finished (%s): ordering contract broken", took, slowWork)
		}
		t.Logf("handover=%s then_fired=%s (slow target needed %s) — no parking, ordering kept", handover, took, slowWork)
	case <-time.After(10 * time.Second):
		t.Fatalf("then never ran: the completion barrier is broken")
	}
	raid.close()
}

// ---------------------------------------------------------------------------
// (C) Does a full queue make the SUBMITTER spin without a deadline?
// ---------------------------------------------------------------------------

// TestReproPostGameSessionEventQueueFullBudget pins the property that a
// submission must always come back: a session whose loop cannot drain (stalled
// socket + a backlog) must never park the submitter forever. It floods one
// session with more events than its queue can hold and measures the submitter's
// worst block against the wall clock.
func TestReproPostGameSessionEventQueueFullBudget(t *testing.T) {
	raid := newReproRaid(t, 4)
	victim := &raid.members[1]
	victim.conn.stallFor(8 * time.Second)

	const bursts = gameSessionEventQueueSize * 2
	var worst time.Duration
	submitter := &raid.members[0]

	t.Logf("flooding %s with %d events while its socket is stalled", victim.conn.name, bursts)
	deadline := time.Now().Add(10 * time.Second)
	for i := 0; i < bursts; i++ {
		if time.Now().After(deadline) {
			t.Logf("flood stopped at %d submits: 10s wall-clock budget spent", i)
			break
		}
		start := time.Now()
		// This is the exact call the raid fan-out uses to hand work to another
		// member's loop; it must be bounded.
		err := raid.service.postGameSessionCharacterEvent(
			victim.session,
			"repro_flood",
			victim.charID,
			victim.session.characterGeneration,
			func() error { return nil },
		)
		blocked := time.Since(start)
		if blocked > worst {
			worst = blocked
		}
		if err != nil {
			t.Logf("submit %d returned %v after %s", i, err, blocked)
			break
		}
		if time.Since(start) > 3*time.Second {
			break
		}
	}
	t.Logf("worst single submitGameSession block = %s ; caller(%s) still workable=%t ; victim queue=%d",
		worst, submitter.conn.name, raid.canStillWork(submitter, 2*time.Second),
		raid.queueDepths()[1])
	victim.conn.release()
	time.Sleep(1500 * time.Millisecond)
	t.Logf("after release: victim queue=%d", raid.queueDepths()[1])
	raid.close()
}

// ---------------------------------------------------------------------------
// (F) 20 simulated clients: squad formation with one member per squad whose
//     native UDP probe answer never arrived, and with the relay enabled.
//
// This is the shape the operator reported as "组队还组不上 显示连接中": one member
// never registers a probe endpoint, so the direct op11 table cannot be built and
// the whole party used to be left unpublished. The relay is the production
// transport (instance.json enables it), so the formation must still complete.
// ---------------------------------------------------------------------------

func TestRepro20RaidSquadFormationWithUnprobedMember(t *testing.T) {
	raid := newReproRaid(t, 20)
	defer raid.close()

	portStart := testCurrentPartyUDPRelayPortStart(t)
	relay, err := newCurrentPartyUDPRelay(true, "127.0.0.1", portStart, 512, nil)
	if err != nil {
		t.Fatalf("new UDP relay: %v", err)
	}
	defer relay.Close()
	raid.service.partyUDPRelay = relay
	raid.service.gameplayTimers = newPartyPublishTestQueue()

	// Every squad's last member never probes; the other three do.
	probed := make(map[*reproMember]bool)
	for index := range raid.members {
		if index%runtimeRaidGroupSize == runtimeRaidGroupSize-1 {
			continue
		}
		registerPartyPeerEndpointForTest(raid.members[index].session, 3, uint16(40000+index))
		probed[&raid.members[index]] = true
	}

	published := 0
	started := time.Now()
	for squad := 0; squad < 5; squad++ {
		leader := &raid.members[squad*runtimeRaidGroupSize]
		for slot := 1; slot < runtimeRaidGroupSize; slot++ {
			member := &raid.members[squad*runtimeRaidGroupSize+slot]
			state, result, ok := raid.service.createOrJoinManagedRuntimeParty(member.session, leader.session)
			if !ok || state.PartyID <= 0 {
				t.Fatalf("squad %d slot %d join failed: ok=%t result=%+v", squad, slot, ok, result)
			}
			done, err := raid.service.publishManagedRuntimePartySnapshots(state)
			if err != nil {
				t.Fatalf("squad %d slot %d publish: %v", squad, slot, err)
			}
			if !done {
				t.Fatalf("squad %d slot %d formation deferred (member %d has no probe and the relay should carry it)",
					squad, slot, member.charID)
			}
			published++
		}
	}
	elapsed := time.Since(started)
	t.Logf("20 clients: %d squad joins published in %s", published, elapsed)
	if elapsed > 5*time.Second {
		t.Fatalf("squad formation took %s for 20 clients", elapsed)
	}

	// Every member of every squad owns a complete directed relay leg set, and the
	// member without a probe gets its legs too (no seed endpoint, learned live).
	for squad := 0; squad < 5; squad++ {
		leader := &raid.members[squad*runtimeRaidGroupSize]
		leaderState := runtimePartyStateSnapshot(leader.session)
		if leaderState.PartyID <= 0 || len(runtimePartyMembers(leaderState)) != runtimeRaidGroupSize {
			t.Fatalf("squad %d leader party state=%+v", squad, leaderState)
		}
		for slot := 0; slot < runtimeRaidGroupSize; slot++ {
			member := &raid.members[squad*runtimeRaidGroupSize+slot]
			for other := 0; other < runtimeRaidGroupSize; other++ {
				if other == slot {
					continue
				}
				peer := &raid.members[squad*runtimeRaidGroupSize+other]
				if _, _, ok := relay.Endpoint(uint16(leaderState.PartyID), member.charID, peer.charID); !ok {
					t.Fatalf("squad %d relay leg %d->%d missing", squad, member.charID, peer.charID)
				}
			}
		}
	}

	// No simulated client's event loop may be parked by the formation burst.
	for _, member := range raid.members {
		if !raid.canStillWork(&member, 2*time.Second) {
			t.Fatalf("client %d stopped working during squad formation", member.charID)
		}
	}

	// The unprobed members were admitted through the relay table, and their op11
	// row carries a relay port inside the configured window.
	unprobed := &raid.members[runtimeRaidGroupSize-1]
	frame := unprobed.conn.bufferConn.write.Bytes()
	if len(frame) == 0 {
		t.Fatalf("unprobed member %d received no formation frame", unprobed.charID)
	}
	packets := splitAllCurrentGameServerUpperPackets(t, frame)
	var peerRow []byte
	for _, packet := range packets {
		if packet.Header.Classification == 0 && packet.Header.MsgID == 0x000b {
			peerRow = packet.Body
			break
		}
	}
	if len(peerRow) == 0 || peerRow[0] == 0 {
		t.Fatalf("unprobed member %d op11 body=% X", unprobed.charID, peerRow)
	}
	relayPort := uint16(peerRow[11])<<8 | uint16(peerRow[12])
	if relayPort < uint16(portStart) || relayPort >= uint16(portStart+512) {
		t.Fatalf("unprobed member op11 port %d outside the relay window %d..%d",
			relayPort, portStart, portStart+511)
	}
}

// With the relay switched off the same 20-client shape must fall back to the
// bounded publish retry: the formation defers, an attempt is armed per party, and
// the attempt publishes as soon as the missing probe answer lands.
func TestRepro20RaidSquadFormationRetriesWithoutRelay(t *testing.T) {
	raid := newReproRaid(t, 20)
	defer raid.close()
	queue := newPartyPublishTestQueue()
	raid.service.gameplayTimers = queue

	for index := range raid.members {
		if index%runtimeRaidGroupSize != runtimeRaidGroupSize-1 {
			registerPartyPeerEndpointForTest(raid.members[index].session, 3, uint16(40000+index))
		}
	}
	var lastState alignedcmd.PartyState
	var lastMember *reproMember
	for squad := 0; squad < 5; squad++ {
		leader := &raid.members[squad*runtimeRaidGroupSize]
		for slot := 1; slot < runtimeRaidGroupSize; slot++ {
			member := &raid.members[squad*runtimeRaidGroupSize+slot]
			state, _, ok := raid.service.createOrJoinManagedRuntimeParty(member.session, leader.session)
			if !ok {
				t.Fatalf("squad %d slot %d join failed", squad, slot)
			}
			done, err := raid.service.publishManagedRuntimePartySnapshots(state)
			if err != nil {
				t.Fatalf("squad %d slot %d publish: %v", squad, slot, err)
			}
			if slot < runtimeRaidGroupSize-1 {
				// Every member so far has a probe answer, so the direct table is
				// complete and the formation must publish immediately.
				if !done {
					t.Fatalf("squad %d slot %d deferred although every member had probed", squad, slot)
				}
				continue
			}
			// The last member of the squad never probed and there is no relay: the
			// formation is deferred and the bounded retry is what completes it.
			if done {
				t.Fatalf("squad %d published with an unprobed member and no relay", squad)
			}
			lastState = state
			lastMember = member
		}
	}
	names := queue.namesMatching("party-snapshot-publish-retry:")
	if len(names) != 5*len(currentPartySnapshotPublishRetryDelays) {
		t.Fatalf("armed %d publish-retry attempts for 5 squads, want %d",
			len(names), 5*len(currentPartySnapshotPublishRetryDelays))
	}
	// The last unprobed member finally answers its reachability probe.
	registerPartyPeerEndpointForTest(lastMember.session, 3, 60999)
	target := ""
	for _, name := range names {
		if name == fmt.Sprintf("party-snapshot-publish-retry:%d:0", lastState.PartyID) {
			target = name
			break
		}
	}
	if target == "" {
		t.Fatalf("no retry armed for party %d: %v", lastState.PartyID, names)
	}
	queue.fire(t, target)
	if len(lastMember.conn.bufferConn.write.Bytes()) == 0 {
		t.Fatalf("retry did not publish the formation frame for member %d", lastMember.charID)
	}
	for _, member := range raid.members {
		if !raid.canStillWork(&member, 2*time.Second) {
			t.Fatalf("client %d stopped working during the deferred formation", member.charID)
		}
	}
}