Commonwealth-Online/Commonwealth-Online-Server
3
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

Acceptance: load/fault tier — 16/32/64 clients, burst, churn, handoff-under-load (#15) (#24)

Browse Source 
* Acceptance: add the load/fault tier (16/32/64 clients, burst, churn, handoff)

Extend the end-to-end harness (issue #15), harness-only — no server change.
Uses the existing ServerTuning seam to widen the local connect budget and
capacity so many clients can run over a single loopback IP.

- 16/32/64 clients across distinct cells: unique server-owned ids and no
  cross-cell transform spam (generalized from the prior 16-client case)
- burst transform traffic engages the rate limiter without tearing sessions
  down, and the server still relays a fresh transform afterward
- reconnect churn (12 cycles): ids stay monotonic and never reused, and each
  disconnect leaves no stale active session
- authority handoff stays deterministic under churn: retiring the current
  authority repeatedly hands off to the next lowest id with strictly
  increasing epochs

Packet loss/reorder on snapshot traffic is inherently an unreliable-transport
(GNS) property and stays deferred with the GNS sections (blocked on #2).

26/26 pass on real sockets, deterministic across repeated runs.

* Strip verbose comments from the acceptance harness and ServerTuning

Remove the prose/narration comments across the end-to-end harness (keeping only
section dividers) and the ServerTuning header block, matching the repo's terse
comment style.
This commit is contained in:
Nomads_Reach
2026-08-16 20:24:16 -04:00
committed by GitHub
parent 098cf34355
commit ece634450b
2 changed files with 110 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/acceptance/Program.cs
+110 -63
View File
@@ -6,16 +6,6 @@ using CommonwealthOnline.Server;
namespace CommonwealthOnline.Server.Acceptance;
// End-to-end acceptance harness for issue #15. Unlike the in-memory unit tests,
// this stands up the real AuthoritativeServer behind the real TCP transport on a
// loopback port and drives real SyntheticProtocolClient sockets through it, so it
// exercises framing, admission, relay, interest filtering and session teardown
// over an actual connection.
//
// Covered here: the Baseline protocol, TCP compatibility, interest-management,
// movement/combat-validation, and NPC-authority sections of the matrix. GNS
// sections wait on #2 (real-bridge loopback); the 32/64-client load/fault runs
// are deferred follow-ups (see the summary printed at the end).
internal static class Program
{
private static int _passed;
@@ -45,17 +35,22 @@ internal static class Program
await Run("combat: combat hit at a disconnected target is rejected", DisconnectedTargetCombatRejected);
await Run("combat: out-of-interest combat hit is rejected", OutOfInterestCombatRejected);
await Run("interest: same-cell relays, distant is filtered", InterestSameCellVsDistant);
await Run("interest: 16 clients across cells, no cross-cell transform spam", SixteenClientsNoCrossCellSpam);
await Run("authority: independent populated scopes get independent authorities", IndependentNpcAuthorities);
await Run("authority: a stale npc authority epoch is rejected over transport", StaleNpcAuthorityEpochRejected);
await Run("authority: disconnect deterministically hands off to a new owner", AuthorityHandoffOnDisconnect);
await Run("authority: cell transition hands off and blocks the previous owner", AuthorityHandoffOnCellTransition);
await Run("baseline: handshake timeout closes an unactivated session", HandshakeTimeoutCloses);
await Run("baseline: idle timeout closes a stale active session", IdleTimeoutCloses);
await Run("load: 16 clients across cells, no cross-cell transform spam", () => NoCrossCellSpam(16));
await Run("load: 32 clients across cells, no cross-cell transform spam", () => NoCrossCellSpam(32));
await Run("load: 64 clients across cells, no cross-cell transform spam", () => NoCrossCellSpam(64));
await Run("load: burst transform traffic keeps the server responsive", BurstTransformTraffic);
await Run("load: reconnect churn leaves no stale sessions", ReconnectChurn);
await Run("load: authority handoff stays deterministic under churn", AuthorityHandoffUnderLoad);
Console.WriteLine();
Console.WriteLine($"acceptance: {_passed} passed, {_failed} failed");
Console.WriteLine("deferred (follow-ups): 32/64-client load + packet-loss/reorder + reconnect-churn, and all GNS sections (blocked on #2).");
Console.WriteLine("deferred (follow-ups): packet loss/reorder on snapshot traffic and all GNS sections (both need the unreliable GNS path, blocked on #2).");
return _failed == 0 ? 0 : 1;
}
@@ -78,8 +73,6 @@ internal static class Program
await using var b = await Connect(s.Port);
await Drain(a, 300); await Drain(b, 300);
// a claims a bogus playerId inside its own transform; the server must relay
// b a transform stamped with a's real server-owned id, never the claim.
var spoof = TransformPacket("0000AAAA", "0000BBBB", 0, 0, "spawn");
spoof["playerId"] = 999999;
await a.SendAsync(spoof);
@@ -115,8 +108,6 @@ internal static class Program
await raw.ExpectType("sessionReady", 2000);
var transformsBefore = Stat(s.Server, "transformPacketsReceived");
// A single line larger than the protocol maximum must terminate the session
// at the transport before it is ever decoded or applied.
await raw.WriteRaw(new byte[ProtocolConstants.MaxMessageBytes + 1024]);
True(await raw.WaitClosed(4000), "server closed the connection on oversize input");
@@ -127,7 +118,6 @@ internal static class Program
{
await using var s = new TestServer();
await using var a = await Connect(s.Port);
// Burst well past the 120 packets/second window in a single window.
for (var i = 0; i < 400; i++)
{
try { await a.SendAsync(new JsonObject { ["type"] = "keepAlive" }); }
@@ -142,19 +132,16 @@ internal static class Program
private static async Task TwoClientSmoke()
{
await using var s = new TestServer();
await using var a = await Connect(s.Port); // a connects first -> world-state host
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000CAFE", "0000F00D", 0, 0, "spawn"));
await b.SendAsync(TransformPacket("0000CAFE", "0000F00D", 0, 0, "spawn"));
await Drain(a, 400); await Drain(b, 400);
// transform relay
await a.SendAsync(TransformPacket("0000CAFE", "0000F00D", 10, 0));
NotNull(await ReceiveUntil(b, p => Type(p) == "transform" && UInt(p["playerId"]) == a.PlayerId, 3000),
"b received a's transform");
// durable playerState relay: an in-interest peer receives the durable
// fields (and, being in scope, the live action events too).
await a.SendAsync(new JsonObject
{
["type"] = "playerState",
@@ -165,7 +152,6 @@ internal static class Program
NotNull(ps, "b received a's playerState");
Equal("Nomad", JsonHelpers.String(ps!["characterName"]), "playerState carried the character name");
// world-state from the host relays
await a.SendAsync(new JsonObject { ["type"] = "worldState", ["timeHours"] = 12.0 });
NotNull(await ReceiveUntil(b, p => Type(p) == "worldState", 3000), "b received world state from the host");
}
@@ -183,8 +169,6 @@ internal static class Program
});
await Drain(a, 300);
// a late joiner is replayed a's durable state, which must carry the durable
// fields but never the discrete action events.
await using var late = await Connect(s.Port);
var replay = await ReceiveUntil(late, p => Type(p) == "playerState" && UInt(p["playerId"]) == a.PlayerId, 3000);
NotNull(replay, "late joiner received a's durable player state");
@@ -217,7 +201,6 @@ internal static class Program
await using var s = new TestServer();
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
// both anchor in the same cell
await a.SendAsync(TransformPacket("0000BEEF", "0000BEEF", 0, 0, "spawn"));
await b.SendAsync(TransformPacket("0000BEEF", "0000BEEF", 0, 0, "spawn"));
await Drain(a, 400); await Drain(b, 400);
@@ -227,8 +210,6 @@ internal static class Program
"same-cell transform relayed to b");
var filteredBefore = Stat(s.Server, "transformPacketsInterestFiltered");
// a makes a valid transition to a distant, unrelated cell/worldspace; the
// move itself is legal (transition type) but b is no longer in interest.
await a.SendAsync(TransformPacket("00000111", "00000222", 5_000_000, 5_000_000, "cell_change"));
await Drain(a, 300);
True(await ExpectNone(b, p => Type(p) == "transform" && UInt(p["playerId"]) == a.PlayerId, 1200),
@@ -237,35 +218,28 @@ internal static class Program
True(Stat(s.Server, "transformPacketsInterestFiltered") > filteredBefore, "distant transform counted as interest-filtered");
}
// 16 clients over a single loopback IP exceed the production per-IP connect
// throttle (8/10s), so this case runs the test server with a widened local
// connect budget. The 32/64-client load runs remain follow-ups.
private const int SixteenClientCount = 16;
private static async Task SixteenClientsNoCrossCellSpam()
private static async Task NoCrossCellSpam(int count)
{
await using var s = new TestServer(tuning: new ServerTuning { MaxConnectAttempts = 64 });
await using var s = new TestServer(maxPlayers: Math.Max(64, count),
tuning: new ServerTuning { MaxConnectAttempts = count * 4 });
var clients = new List<SyntheticProtocolClient>();
try
{
for (var i = 0; i < SixteenClientCount; i++)
for (var i = 0; i < count; i++)
{
var c = await Connect(s.Port);
clients.Add(c);
// every client sits in its own cell AND worldspace (coordinates kept
// well within MaxAbsCoordinate; distinct scopes do the isolating)
await c.SendAsync(TransformPacket((0x2000 + i).ToString("X8"), (0x9000 + i).ToString("X8"), i * 1000.0, 0, "spawn"));
}
var ids = clients.Select(c => c.PlayerId).ToArray();
Equal(SixteenClientCount, ids.Distinct().Count(), "unique server-owned ids for every client");
foreach (var c in clients) await Drain(c, 150);
Equal(count, ids.Distinct().Count(), "unique server-owned ids for every client");
foreach (var c in clients) await Drain(c, 100);
// each client moves; nobody should see anybody else's transform
for (var i = 0; i < clients.Count; i++)
await clients[i].SendAsync(TransformPacket((0x2000 + i).ToString("X8"), (0x9000 + i).ToString("X8"), i * 1000.0 + 10, 0));
foreach (var c in clients)
True(await ExpectNone(c, p => Type(p) == "transform", 500), "client received no cross-cell transform spam");
True(await ExpectNone(c, p => Type(p) == "transform", 400), "client received no cross-cell transform spam");
}
finally
{
@@ -280,13 +254,11 @@ internal static class Program
await using var s = new TestServer();
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
// both anchor in the same cell so b would normally receive a's transforms
await a.SendAsync(TransformPacket("0000D00D", "0000D00D", 0, 0, "spawn"));
await b.SendAsync(TransformPacket("0000D00D", "0000D00D", 0, 0, "spawn"));
await Drain(a, 300); await Drain(b, 300);
var rejectedBefore = Stat(s.Server, "movementPacketsRejected");
// a "normal" jump of a million units in the same cell is physically impossible
await a.SendAsync(TransformPacket("0000D00D", "0000D00D", 1_000_000, 0, "normal"));
NotNull(await ReceiveUntil(a, p => Type(p) == "positionCorrection", 3000),
@@ -322,13 +294,11 @@ internal static class Program
await b.SendAsync(TransformPacket("0000CB01", "0000CB01", 0, 0, "spawn"));
await Drain(a, 300); await Drain(b, 300);
// first hit at sequence 5 routes to the in-interest target
await a.SendAsync(CombatHit(b.PlayerId, 5));
NotNull(await ReceiveUntil(b, p => Type(p) == "combatHit", 3000), "in-interest combat hit routed to the target");
var routedBefore = Stat(s.Server, "combatHitsRouted");
var rejectedBefore = Stat(s.Server, "packetsRejected");
// an older sequence must be rejected and not routed
await a.SendAsync(CombatHit(b.PlayerId, 3));
await SpinUntil(() => Stat(s.Server, "packetsRejected") > rejectedBefore, 2000);
True(Stat(s.Server, "packetsRejected") > rejectedBefore, "replayed combat sequence was rejected");
@@ -357,7 +327,7 @@ internal static class Program
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000CB03", "0000CB03", 0, 0, "spawn"));
await b.SendAsync(TransformPacket("0000CB04", "0000CB04", 1000, 0, "spawn")); // distinct scope
await b.SendAsync(TransformPacket("0000CB04", "0000CB04", 1000, 0, "spawn"));
await Drain(a, 300); await Drain(b, 300);
var rejectedBefore = Stat(s.Server, "packetsRejected");
var routedBefore = Stat(s.Server, "combatHitsRouted");
@@ -381,7 +351,6 @@ internal static class Program
await using var s = new TestServer();
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
// a and b spawn into distinct, non-overlapping scopes
await a.SendAsync(TransformPacket("0000A111", "0000A111", 0, 0, "spawn"));
await b.SendAsync(TransformPacket("0000B222", "0000B222", 5_000_000, 0, "spawn"));
@@ -405,7 +374,6 @@ internal static class Program
await Drain(a, 200);
var rejectsBefore = Stat(s.Server, "npcAuthorityRejects");
// submit npcState claiming a wrong (future) authority epoch
await a.SendAsync(new JsonObject
{
["type"] = "npcState",
@@ -422,16 +390,15 @@ internal static class Program
private static async Task AuthorityHandoffOnDisconnect()
{
await using var s = new TestServer();
var a = await Connect(s.Port); // connects first -> lower id -> initial authority
var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000A0FF", "0000A0FF", 0, 0, "spawn"));
var grantA = await ReceiveUntil(a, p => Type(p) == "npcAuthority" && UInt(p["authorityPlayerId"]) == a.PlayerId, 3000);
NotNull(grantA, "a is the initial authority for the shared scope");
var epochA = UInt(grantA!["authorityEpoch"]);
await b.SendAsync(TransformPacket("0000A0FF", "0000A0FF", 0, 0, "spawn")); // b joins the same scope
await b.SendAsync(TransformPacket("0000A0FF", "0000A0FF", 0, 0, "spawn"));
await Drain(a, 300); await Drain(b, 300);
// a leaves; authority must deterministically hand off to b with a newer epoch
await a.DisposeAsync();
var grantB = await ReceiveUntil(b, p => Type(p) == "npcAuthority" && UInt(p["authorityPlayerId"]) == b.PlayerId, 4000);
NotNull(grantB, "authority handed off to b after a disconnected");
@@ -443,20 +410,18 @@ internal static class Program
await using var s = new TestServer();
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000CE10", "0000CE10", 0, 0, "spawn")); // a authority for scope X
await a.SendAsync(TransformPacket("0000CE10", "0000CE10", 0, 0, "spawn"));
var grantA = await ReceiveUntil(a, p => Type(p) == "npcAuthority" && UInt(p["authorityPlayerId"]) == a.PlayerId, 3000);
NotNull(grantA, "a is authority for scope X");
var epochX1 = UInt(grantA!["authorityEpoch"]);
await b.SendAsync(TransformPacket("0000CE10", "0000CE10", 0, 0, "spawn")); // b joins scope X
await b.SendAsync(TransformPacket("0000CE10", "0000CE10", 0, 0, "spawn"));
await Drain(a, 300); await Drain(b, 300);
// a transitions out to a different cell; scope X must hand off to b
await a.SendAsync(TransformPacket("0000CE20", "0000CE20", 0, 0, "cell_change"));
var grantB = await ReceiveUntil(b, p => Type(p) == "npcAuthority" && UInt(p["authorityPlayerId"]) == b.PlayerId, 4000);
NotNull(grantB, "scope X handed off to b after a transitioned away");
True(UInt(grantB!["authorityEpoch"]) > epochX1, "handoff carries a newer epoch");
// the previous owner can no longer submit npcState for the reassigned scope
await Drain(a, 200);
var rejectsBefore = Stat(s.Server, "npcAuthorityRejects");
await a.SendAsync(new JsonObject
@@ -471,6 +436,94 @@ internal static class Program
True(Stat(s.Server, "npcAuthorityRejects") > rejectsBefore, "previous owner cannot submit for the reassigned scope");
}
// ---- load / fault -------------------------------------------------------
private static async Task BurstTransformTraffic()
{
await using var s = new TestServer();
await using var a = await Connect(s.Port);
await using var b = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000B057", "0000B057", 5, 5, "spawn"));
await b.SendAsync(TransformPacket("0000B057", "0000B057", 5, 5, "spawn"));
await Drain(a, 300); await Drain(b, 300);
var receivedBefore = Stat(s.Server, "transformPacketsReceived");
for (var i = 0; i < 400; i++)
{
try { await a.SendAsync(TransformPacket("0000B057", "0000B057", 5, 5, "normal")); }
catch { break; }
}
await SpinUntil(() => Stat(s.Server, "rateLimitedPackets") > 0, 3000);
True(Stat(s.Server, "rateLimitedPackets") > 0, "the burst engaged the rate limiter");
True(Stat(s.Server, "transformPacketsReceived") > receivedBefore, "the server processed transforms during the burst");
Equal(2u, Stat(s.Server, "connectedClients"), "both sessions survived the burst");
await Drain(b, 300);
var relayed = false;
for (var attempt = 0; attempt < 5 && !relayed; attempt++)
relayed = await RelayWorks(a, b);
True(relayed, "server relays a fresh transform after the burst");
}
private static async Task<bool> RelayWorks(SyntheticProtocolClient a, SyntheticProtocolClient b)
{
await a.SendAsync(TransformPacket("0000B057", "0000B057", 6, 6, "normal"));
return await ReceiveUntil(b, p => Type(p) == "transform" && UInt(p["playerId"]) == a.PlayerId, 800) is not null;
}
private static async Task ReconnectChurn()
{
await using var s = new TestServer(tuning: new ServerTuning { MaxConnectAttempts = 128 });
uint lastId = 0;
for (var k = 0; k < 12; k++)
{
var c = await Connect(s.Port);
True(c.PlayerId > lastId, "server-owned ids are monotonic and never reused across reconnects");
lastId = c.PlayerId;
await c.SendAsync(TransformPacket("0000C401", "0000C401", 0, 0, "spawn"));
await SpinUntil(() => Stat(s.Server, "connectedClients") == 1, 3000);
await c.DisposeAsync();
await SpinUntil(() => Stat(s.Server, "connectedClients") == 0, 3000);
Equal(0u, Stat(s.Server, "connectedClients"), "each disconnect leaves no stale active session");
}
await using var final = await Connect(s.Port);
True(final.PlayerId > lastId, "server still assigns fresh ids after churn");
}
private static async Task AuthorityHandoffUnderLoad()
{
const int n = 6;
await using var s = new TestServer(tuning: new ServerTuning { MaxConnectAttempts = 64 });
var clients = new List<SyntheticProtocolClient>();
try
{
for (var i = 0; i < n; i++)
{
var c = await Connect(s.Port);
clients.Add(c);
await c.SendAsync(TransformPacket("0000A11D", "0000A11D", 0, 0, "spawn"));
}
var watcher = clients[n - 1];
await Drain(watcher, 300);
uint lastEpoch = 0;
for (var i = 0; i < 3; i++)
{
await clients[i].DisposeAsync();
var expected = clients[i + 1].PlayerId;
var grant = await ReceiveUntil(watcher, p => Type(p) == "npcAuthority" && UInt(p["authorityPlayerId"]) == expected, 4000);
NotNull(grant, $"authority handed off to the next owner after retiring holder {i}");
var epoch = UInt(grant!["authorityEpoch"]);
True(epoch > lastEpoch, "each handoff carries a strictly newer epoch");
lastEpoch = epoch;
}
}
finally
{
foreach (var c in clients) await c.DisposeAsync();
}
}
// ---- session teardown ---------------------------------------------------
private static async Task HandshakeTimeoutCloses()
@@ -478,7 +531,6 @@ internal static class Program
await using var s = new TestServer();
using var raw = await RawClient.Connect(s.Port);
await raw.ExpectType("welcome", 2000);
// deliberately never send hello; the reaper must close the pending session
Equal(0u, Stat(s.Server, "connectedClients"), "pending session is not counted as active");
True(await raw.WaitClosed(14000), "server reaped the unactivated session after the handshake timeout");
Equal(0u, Stat(s.Server, "connectedClients"), "still no active clients after reap");
@@ -486,15 +538,12 @@ internal static class Program
private static async Task IdleTimeoutCloses()
{
// Short idle timeout so the reaper (500ms tick) fires quickly; handshake
// timeout keeps its default so the client can activate first.
await using var s = new TestServer(tuning: new ServerTuning { ClientIdleTimeoutSeconds = 2.0 });
await using var a = await Connect(s.Port);
await a.SendAsync(TransformPacket("0000171E", "0000171E", 0, 0, "spawn"));
await SpinUntil(() => Stat(s.Server, "connectedClients") == 1, 3000);
Equal(1u, Stat(s.Server, "connectedClients"), "client is active before going idle");
// stay idle past the 2s idle timeout; the reaper must close the session
await SpinUntil(() => Stat(s.Server, "connectedClients") == 0, 6000);
Equal(0u, Stat(s.Server, "connectedClients"), "idle active session was reaped");
}
@@ -625,8 +674,6 @@ internal static class Program
}
}
// Minimal raw client for cases the framed SyntheticProtocolClient can't express
// (oversize input, and connecting without completing the handshake).
private sealed class RawClient : IDisposable
{
private readonly TcpClient _client;
@@ -690,11 +737,11 @@ internal static class Program
while (true)
{
var n = await _stream.ReadAsync(one, cts.Token);
if (n == 0) return true; // clean EOF -> server closed
if (n == 0) return true;
}
}
catch (OperationCanceledException) { return false; }
catch (IOException) { return true; } // reset also counts as closed
catch (IOException) { return true; }
catch (SocketException) { return true; }
}