diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a88bb6d0b4..2ec2725a76 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -188,6 +188,10 @@ jobs: API_KEY: ${{ secrets.API_KEY }} ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }} CLUSTER_JSON: ${{ vars.CLUSTER_JSON }} + # "api" makes servers take their drain state from the API's + # registry (docs/MultiServer.md, "Server list v2"); unset keeps + # today's apex colour poll. + CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }} TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }} SERVER_HOST_MASTERS: ${{ secrets.SERVER_HOST_MASTERS }} SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3b7f5cb14..e9d4913f41 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,6 +102,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }} CLUSTER_JSON: ${{ vars.CLUSTER_JSON }} + CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }} TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }} SERVER_HOST_STAGING: ${{ secrets.SERVER_HOST_STAGING }} SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }} @@ -163,6 +164,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }} CLUSTER_JSON: ${{ vars.CLUSTER_JSON }} + CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }} TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }} SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }} SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }} @@ -238,6 +240,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }} CLUSTER_JSON: ${{ vars.CLUSTER_JSON }} + CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }} TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }} SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }} SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }} @@ -308,6 +311,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }} CLUSTER_JSON: ${{ vars.CLUSTER_JSON }} + CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }} TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }} SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }} SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }} diff --git a/deploy.sh b/deploy.sh index 92ff7cad91..95f8c1ee41 100755 --- a/deploy.sh +++ b/deploy.sh @@ -252,6 +252,7 @@ SUBDOMAIN=$SUBDOMAIN SITE_HOST=$SITE_HOST CDN_BASE=$CDN_BASE CLUSTER_JSON=$CLUSTER_JSON +CLUSTER_STATE_SOURCE=$CLUSTER_STATE_SOURCE TURNSTILE_SITE_KEY=$TURNSTILE_SITE_KEY OTEL_EXPORTER_OTLP_ENDPOINT=$OTEL_EXPORTER_OTLP_ENDPOINT OTEL_AUTH_HEADER=$OTEL_AUTH_HEADER diff --git a/src/server/ClusterCheckin.ts b/src/server/ClusterCheckin.ts new file mode 100644 index 0000000000..6a1e5c99cf --- /dev/null +++ b/src/server/ClusterCheckin.ts @@ -0,0 +1,110 @@ +import { z } from "zod"; +import { ServerEnv } from "./ServerEnv"; + +// Multi-server v2 (docs/MultiServer.md, "Server list v2"): every server +// tells the API on boot and every CHECKIN_INTERVAL_MS who it is and what it +// runs, and the API answers with whether this server should take new games. +// The API's list is what clients read (src/client/ServerList.ts), so a +// server that is not checking in is not offered to anyone; a deploy that +// fails halfway can't leave the list claiming servers that aren't there. +// +// The reply is only obeyed when CLUSTER_STATE_SOURCE=api. Until then the +// drain decision stays with today's apex colour poll (ActiveDeployment.ts), +// so this can ship before the API serves the registry. + +export const CHECKIN_INTERVAL_MS = 10_000; +const CHECKIN_TIMEOUT_MS = 8_000; + +// The states the API assigns (infra #700), the same vocabulary the client +// reads from GET /cluster.json: +// open: runs the site's `latest` and isn't fenced, so it takes new games. +// draining: on its way out (an older version, a deploy moving off it); +// existing games and rejoins still work, it just gets no new ones. +// fenced: deliberately held out of rotation by an operator. Same effect +// here as draining, but the API keeps them apart so the list can say +// why. Only "open" takes new games, so a state we fail to recognise +// must never be read as open (see sendCheckin: it returns null). +export const ServerStateSchema = z.enum(["open", "draining", "fenced"]); +export type ServerState = z.infer; + +export type ClusterStateSource = "apex" | "api"; + +export interface CheckinBody { + // The hostname players load the page from: the apex behind a load + // balancer (SITE_HOST), else this deployment's own host, so beta, + // nightly, alpha and branch previews each register under themselves. + // Lists are keyed by it. Mirrors, such as the openfront.dev apex + // serving nightly, are an alias table in the API, never something a + // server reports about itself. + site: string; + letter: string; + host: string; + // GIT_COMMIT, the full sha. Clients compare it prefix-tolerantly. + version: string; + numWorkers: number; + liveGames: number; +} + +const CheckinReplySchema = z.object({ state: ServerStateSchema }); + +/** + * What this server reports, or null under local development (`npm run dev`: + * no SUBDOMAIN, so no public host), where there is nothing to register. + * Every deployed host has one and registers under its own site. + */ +export function checkinBody(liveGames: number): CheckinBody | null { + const host = ServerEnv.publicHost(); + if (host === undefined) return null; + const { letter, entry } = ServerEnv.clusterSelf(); + return { + site: ServerEnv.siteHost() ?? host, + letter, + host, + version: ServerEnv.gitCommit(), + numWorkers: entry.numWorkers, + liveGames, + }; +} + +/** + * One check-in. Returns the state the API assigned, or null when the answer + * is unusable (the API predates the registry, a bot challenge, a network + * error). Callers must treat null as "no change", never as "drain": the + * failure mode of an unreachable API is the status quo. + */ +export async function sendCheckin( + body: CheckinBody, + fetchFn: typeof fetch = fetch, +): Promise { + try { + const res = await fetchFn(`${ServerEnv.jwtIssuer()}/cluster/checkin`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": ServerEnv.apiKey(), + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), + }); + if (!res.ok) return null; + const parsed = CheckinReplySchema.safeParse(await res.json()); + return parsed.success ? parsed.data.state : null; + } catch { + return null; + } +} + +/** + * Turn a check-in reply into the lobby service's active flag, but only when + * the API is the configured source of that decision. Active means "open"; + * draining and fenced both stop new games. Pure, so the switch is testable + * without booting the master. + */ +export function applyCheckinState( + state: ServerState | null, + source: ClusterStateSource, + setActive: (active: boolean) => void, +): void { + if (state === null || source !== "api") return; + setActive(state === "open"); +} diff --git a/src/server/IPCBridgeSchema.ts b/src/server/IPCBridgeSchema.ts index c188e912e2..76a349e43b 100644 --- a/src/server/IPCBridgeSchema.ts +++ b/src/server/IPCBridgeSchema.ts @@ -46,6 +46,11 @@ export const InternalPublicGamesSchema = z.object({ const WorkerLobbyListSchema = z.object({ type: z.literal("lobbyList"), lobbies: z.array(z.unknown()), + // Games this worker is running, lobbies included. The master sums them + // for the cluster check-in (ClusterCheckin.ts), so an operator can tell + // when a draining server is empty. Optional for a worker build that + // predates it; absent counts as zero. + liveGames: z.number().int().min(0).optional(), }); const WorkerReadySchema = z.object({ diff --git a/src/server/Master.ts b/src/server/Master.ts index 2a1a001b70..a0bc0cddd2 100644 --- a/src/server/Master.ts +++ b/src/server/Master.ts @@ -7,6 +7,12 @@ import path from "path"; import { fileURLToPath } from "url"; import { GameEnv } from "../core/configuration/Config"; import { fetchSiteColor } from "./ActiveDeployment"; +import { + applyCheckinState, + CHECKIN_INTERVAL_MS, + checkinBody, + sendCheckin, +} from "./ClusterCheckin"; import { getDescriptor } from "./DesktopRelease"; import { logger } from "./Logger"; import { MapPlaylist } from "./MapPlaylist"; @@ -176,15 +182,43 @@ export async function startMaster() { log.info(`Master HTTP server listening on port ${PORT}`); }); + // Register with the API and keep checking in (docs/MultiServer.md, + // "Server list v2"): the API's list is what clients read to find a + // server, so a server that isn't checking in isn't offered to anyone. + // The reply carries this server's state; it is obeyed only when + // CLUSTER_STATE_SOURCE=api, otherwise the apex colour poll below still + // decides. Local development (`npm run dev`, no SUBDOMAIN) has no public + // host and registers nowhere; every deployed host registers under its own + // site. + const stateSource = ServerEnv.clusterStateSource(); + if (checkinBody(0) !== null) { + log.info( + `Checking in with ${ServerEnv.jwtIssuer()}/cluster/checkin every ${CHECKIN_INTERVAL_MS / 1000}s (state source: ${stateSource})`, + ); + startPolling(async () => { + const body = checkinBody(lobbyService.liveGames()); + if (body === null) return; + const state = await sendCheckin(body); + applyCheckinState(state, stateSource, (active) => + lobbyService.setActive(active), + ); + }, CHECKIN_INTERVAL_MS); + } + // Behind a load balancer (blue/green), only the color the balancer // currently routes to should schedule public lobbies. The balancer's // /api/health reports the COLOR of whichever deployment answered; colors // are deployment-wide, so with several machines per color the poll // reaching a sibling — same color, different instanceId — still counts as // "the live color is mine". A standalone deployment (no SITE_HOST, or - // SITE_HOST is our own host) is always active. + // SITE_HOST is our own host) is always active. Not started when the API + // is the state source: two deciders would fight over setActive. const siteHost = ServerEnv.siteHost(); - if (siteHost !== undefined && siteHost !== ServerEnv.publicHost()) { + if ( + stateSource === "apex" && + siteHost !== undefined && + siteHost !== ServerEnv.publicHost() + ) { log.info(`Polling https://${siteHost}/api/health for active deployment`); // 5s: this latency is the window after a flip where the newly-active // deployment isn't creating public lobbies yet (and the draining one diff --git a/src/server/MasterLobbyService.ts b/src/server/MasterLobbyService.ts index ac509b85bc..9608375814 100644 --- a/src/server/MasterLobbyService.ts +++ b/src/server/MasterLobbyService.ts @@ -34,6 +34,8 @@ export class MasterLobbyService { private readonly workers = new Map(); // Worker id => the lobbies it owns. private readonly workerLobbies = new Map(); + // Worker id => games it last reported running (lobbies included). + private readonly workerLiveGames = new Map(); private readonly readyWorkers = new Set(); // gameID => consecutive broadcast cycles a hosted lobby has lost the // per-creator dedup or overflowed the cluster-wide cap. Losing once can be @@ -69,6 +71,7 @@ export class MasterLobbyService { break; case "lobbyList": this.workerLobbies.set(workerId, this.validLobbies(msg.lobbies)); + this.workerLiveGames.set(workerId, msg.liveGames ?? 0); break; } }); @@ -95,9 +98,18 @@ export class MasterLobbyService { removeWorker(workerId: number) { this.workers.delete(workerId); this.workerLobbies.delete(workerId); + this.workerLiveGames.delete(workerId); this.readyWorkers.delete(workerId); } + // Games running on this server, summed over the workers' last reports. + // Reported to the API at check-in (ClusterCheckin.ts). + liveGames(): number { + let total = 0; + for (const n of this.workerLiveGames.values()) total += n; + return total; + } + isHealthy(): boolean { // We consider the lobby service healthy if at least half of the workers are ready. // This allows for some leeway if a worker crashes. diff --git a/src/server/ServerEnv.ts b/src/server/ServerEnv.ts index c593c74b9d..8e909e6a61 100644 --- a/src/server/ServerEnv.ts +++ b/src/server/ServerEnv.ts @@ -243,6 +243,14 @@ export class ServerEnv { const v = process.env.SITE_HOST; return v && v.length > 0 ? v : undefined; } + // Where the drain decision comes from (docs/MultiServer.md, "Server list + // v2"): "apex" is today's /api/health colour poll of the site host; "api" + // obeys the state the API assigns at check-in (ClusterCheckin.ts). Any + // other value, or none, means apex, so a deploy that doesn't set it is + // unchanged. + static clusterStateSource(): "apex" | "api" { + return process.env.CLUSTER_STATE_SOURCE === "api" ? "api" : "apex"; + } static otelEnabled(): boolean { return ( ServerEnv.gameEnv !== GameEnv.Dev && diff --git a/src/server/WorkerLobbyService.ts b/src/server/WorkerLobbyService.ts index 22bbee52b5..a96da0133d 100644 --- a/src/server/WorkerLobbyService.ts +++ b/src/server/WorkerLobbyService.ts @@ -186,6 +186,7 @@ export class WorkerLobbyService { this.sendToMaster({ type: "lobbyList", lobbies: [...publicLobbies, ...hostedLobbies], + liveGames: this.gm.activeGames(), } satisfies WorkerLobbyList); } diff --git a/tests/server/ClusterCheckin.test.ts b/tests/server/ClusterCheckin.test.ts new file mode 100644 index 0000000000..2d19698854 --- /dev/null +++ b/tests/server/ClusterCheckin.test.ts @@ -0,0 +1,177 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { + applyCheckinState, + checkinBody, + sendCheckin, +} from "../../src/server/ClusterCheckin"; + +// Multi-server v2, priority 3 (docs/MultiServer.md, "Server list v2"): every +// server tells the API who it is and what it runs, and the API replies with +// whether it should take new games. Until CLUSTER_STATE_SOURCE=api the reply +// is recorded but not obeyed, so a deploy without the API is unchanged. +const CLUSTER = JSON.stringify({ + a: { host: "blue.openfront.io", color: "blue", numWorkers: 4 }, + b: { host: "green.openfront.io", color: "green", numWorkers: 4 }, +}); + +function fetchReturning(body: unknown, status = 200) { + return vi.fn( + async () => new Response(JSON.stringify(body), { status }), + ) as unknown as typeof fetch; +} + +describe("checkinBody", () => { + beforeEach(() => { + vi.stubEnv("CLUSTER_JSON", CLUSTER); + vi.stubEnv("DOMAIN", "openfront.io"); + vi.stubEnv("SUBDOMAIN", "blue"); + vi.stubEnv("GIT_COMMIT", "bfd5563a11111111111111111111111111111111"); + }); + afterEach(() => vi.unstubAllEnvs()); + + test("registers under the apex when the deployment sits behind one", () => { + vi.stubEnv("SITE_HOST", "openfront.io"); + expect(checkinBody(7)).toEqual({ + site: "openfront.io", + letter: "a", + host: "blue.openfront.io", + version: "bfd5563a11111111111111111111111111111111", + numWorkers: 4, + liveGames: 7, + }); + }); + + // Every deployed host that isn't behind the apex load balancer is its own + // site. Mirrors (the openfront.dev apex serving nightly) are aliased in + // the API, so nothing here reports them. + test.each([ + { + what: "a branch preview", + domain: "openfront.dev", + subdomain: "my-branch", + numWorkers: 2, + }, + { what: "beta", domain: "openfront.io", subdomain: "beta", numWorkers: 4 }, + { + what: "nightly", + domain: "openfront.dev", + subdomain: "nightly", + numWorkers: 3, + }, + ])( + "registers under its own host when standalone ($what)", + ({ domain, subdomain, numWorkers }) => { + const host = `${subdomain}.${domain}`; + vi.stubEnv("SITE_HOST", ""); + vi.stubEnv("DOMAIN", domain); + vi.stubEnv("SUBDOMAIN", subdomain); + vi.stubEnv( + "CLUSTER_JSON", + JSON.stringify({ a: { host, color: "blue", numWorkers } }), + ); + expect(checkinBody(0)).toMatchObject({ + site: host, + host, + letter: "a", + numWorkers, + }); + }, + ); + + test("does not check in from local development (npm run dev, no SUBDOMAIN)", () => { + vi.stubEnv("SITE_HOST", ""); + vi.stubEnv("SUBDOMAIN", ""); + vi.stubEnv("DOMAIN", "localhost"); + vi.stubEnv("GAME_ENV", "dev"); + expect(checkinBody(0)).toBeNull(); + }); +}); + +describe("sendCheckin", () => { + const body = { + site: "openfront.io", + letter: "a", + host: "blue.openfront.io", + version: "bfd5563a11111111111111111111111111111111", + numWorkers: 4, + liveGames: 7, + }; + + beforeEach(() => { + vi.stubEnv("DOMAIN", "openfront.io"); + vi.stubEnv("API_KEY", "secret"); + }); + afterEach(() => vi.unstubAllEnvs()); + + test("posts the body to the API with the deploy key and returns the state", async () => { + const fetchFn = fetchReturning({ state: "draining" }); + await expect(sendCheckin(body, fetchFn)).resolves.toBe("draining"); + expect(fetchFn).toHaveBeenCalledTimes(1); + const [url, init] = vi.mocked(fetchFn).mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + expect(url).toBe("https://api.openfront.io/cluster/checkin"); + expect(init.method).toBe("POST"); + expect((init.headers as Record)["x-api-key"]).toBe( + "secret", + ); + expect(JSON.parse(String(init.body))).toEqual(body); + expect(init.signal).toBeInstanceOf(AbortSignal); + }); + + test("parses a fenced reply instead of discarding it as unknown", async () => { + await expect( + sendCheckin(body, fetchReturning({ state: "fenced" })), + ).resolves.toBe("fenced"); + }); + + test.each([ + ["a 404 (API without the registry yet)", fetchReturning({}, 404)], + ["a state outside the vocabulary", fetchReturning({ state: "retired" })], + [ + "a non-JSON body", + vi.fn( + async () => new Response("", { status: 200 }), + ) as unknown as typeof fetch, + ], + [ + "a network error", + vi.fn(async () => { + throw new Error("ECONNRESET"); + }) as unknown as typeof fetch, + ], + ])("returns null on %s", async (_name, fetchFn) => { + await expect(sendCheckin(body, fetchFn)).resolves.toBeNull(); + }); +}); + +describe("applyCheckinState", () => { + test("obeys the API only when it is the configured state source", () => { + const setActive = vi.fn(); + applyCheckinState("draining", "api", setActive); + expect(setActive).toHaveBeenLastCalledWith(false); + applyCheckinState("open", "api", setActive); + expect(setActive).toHaveBeenLastCalledWith(true); + }); + + // A fence is an operator holding this server out of rotation. It has to + // stop new games like a drain does; only "open" is active. + test("a fenced server takes no new games", () => { + const setActive = vi.fn(); + applyCheckinState("fenced", "api", setActive); + expect(setActive).toHaveBeenLastCalledWith(false); + }); + + test("records but never applies the state under the apex source", () => { + const setActive = vi.fn(); + applyCheckinState("draining", "apex", setActive); + expect(setActive).not.toHaveBeenCalled(); + }); + + test("a failed check-in never drains: null means no change", () => { + const setActive = vi.fn(); + applyCheckinState(null, "api", setActive); + expect(setActive).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/server/HostedLobbyListing.test.ts b/tests/server/HostedLobbyListing.test.ts index ee34c91051..9b238c26a4 100644 --- a/tests/server/HostedLobbyListing.test.ts +++ b/tests/server/HostedLobbyListing.test.ts @@ -712,6 +712,7 @@ describe("WorkerLobbyService hosted lobbies", () => { publicLobbies: vi.fn().mockReturnValue([]), listedLobbies: vi.fn().mockReturnValue([]), game: vi.fn().mockReturnValue(null), + activeGames: vi.fn().mockReturnValue(0), }; const server = new EventEmitter(); service = new WorkerLobbyService( @@ -805,6 +806,32 @@ describe("WorkerLobbyService hosted lobbies", () => { expect(lobbyList.lobbies.map((l: any) => l.gameID)).toEqual(["ffa-g1"]); }); + it("reports the game manager's live game count to the master", () => { + gm.activeGames.mockReturnValue(7); + // Drop the stub so the real sendToMaster runs: assert on the message that + // actually leaves the worker. Anything that is not ours is forwarded + // untouched, since vitest's fork pool talks to its parent on this channel. + delete (service as any).sendToMaster; + const realSend = process.send; + const sent: any[] = []; + process.send = ((msg: any, ...rest: any[]) => { + if (msg?.type === "lobbyList" || msg?.type === "workerReady") { + sent.push(msg); + return true; + } + return (realSend as any)?.apply(process, [msg, ...rest]) ?? true; + }) as any; + try { + emitBroadcast({ ffa: [], team: [], special: [], hosted: [] }); + } finally { + process.send = realSend; + } + + const lobbyList = sent.find((m) => m.type === "lobbyList"); + expect(lobbyList).toBeDefined(); + expect(lobbyList.liveGames).toBe(7); + }); + it("strips creatorID from broadcasts and primed snapshots sent to clients", () => { const ws = connectClient(); emitBroadcast({ diff --git a/tests/server/MasterLobbyServiceLiveGames.test.ts b/tests/server/MasterLobbyServiceLiveGames.test.ts new file mode 100644 index 0000000000..82ba554270 --- /dev/null +++ b/tests/server/MasterLobbyServiceLiveGames.test.ts @@ -0,0 +1,64 @@ +import EventEmitter from "events"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { MasterLobbyService } from "../../src/server/MasterLobbyService"; +import { ServerEnv } from "../../src/server/ServerEnv"; + +vi.mock("../../src/server/Logger", () => ({ + logger: { child: () => ({ error: vi.fn(), info: vi.fn() }) }, +})); +vi.mock("../../src/server/PollingLoop", () => ({ startPolling: vi.fn() })); + +function createMockWorker() { + const emitter = new EventEmitter() as EventEmitter & { + send: ReturnType; + }; + emitter.send = vi.fn(); + return emitter; +} + +// The cluster check-in (ClusterCheckin.ts) reports how many games this +// server is running, so an operator can see when a draining server is empty +// and safe to stop. Each worker reports its own count with its lobby list; +// the master sums them. +describe("MasterLobbyService.liveGames", () => { + let service: MasterLobbyService; + + beforeEach(() => { + vi.stubEnv("DOMAIN", "localhost"); + vi.spyOn(ServerEnv, "numWorkers").mockReturnValue(2); + const playlist = { gameConfig: vi.fn(async () => ({})) }; + const log = { info: vi.fn(), error: vi.fn() } as any; + service = new MasterLobbyService(playlist as any, log); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("sums the counts every worker last reported", () => { + const w0 = createMockWorker(); + const w1 = createMockWorker(); + service.registerWorker(0, w0 as any); + service.registerWorker(1, w1 as any); + expect(service.liveGames()).toBe(0); + + w0.emit("message", { type: "lobbyList", lobbies: [], liveGames: 3 }); + w1.emit("message", { type: "lobbyList", lobbies: [], liveGames: 4 }); + expect(service.liveGames()).toBe(7); + + w1.emit("message", { type: "lobbyList", lobbies: [], liveGames: 1 }); + expect(service.liveGames()).toBe(4); + }); + + it("treats a report without a count (an older worker build) as zero, and forgets a dead worker", () => { + const w0 = createMockWorker(); + service.registerWorker(0, w0 as any); + w0.emit("message", { type: "lobbyList", lobbies: [] }); + expect(service.liveGames()).toBe(0); + w0.emit("message", { type: "lobbyList", lobbies: [], liveGames: 5 }); + expect(service.liveGames()).toBe(5); + service.removeWorker(0); + expect(service.liveGames()).toBe(0); + }); +}); diff --git a/tests/server/WorkerLobbyServiceDrain.test.ts b/tests/server/WorkerLobbyServiceDrain.test.ts index 2ef23d929c..a77d7f71c1 100644 --- a/tests/server/WorkerLobbyServiceDrain.test.ts +++ b/tests/server/WorkerLobbyServiceDrain.test.ts @@ -18,6 +18,7 @@ describe("WorkerLobbyService deployment drain flag", () => { publicLobbies: vi.fn().mockReturnValue([]), listedLobbies: vi.fn().mockReturnValue([]), game: vi.fn().mockReturnValue(null), + activeGames: vi.fn().mockReturnValue(0), }; const server = new EventEmitter(); service = new WorkerLobbyService(