Game server checks in with the API; drain state from it when enabled (OPE-432)

.github/workflows/deploy.yml

@@ -188,6 +188,10 @@ jobs:
           API_KEY: ${{ secrets.API_KEY }}
           ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }}
           CLUSTER_JSON: ${{ vars.CLUSTER_JSON }}
+          # "api" makes servers take their drain state from the API's
+          # registry (docs/MultiServer.md, "Server list v2"); unset keeps
+          # today's apex colour poll.
+          CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }}
           TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }}
           SERVER_HOST_MASTERS: ${{ secrets.SERVER_HOST_MASTERS }}
           SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }}

.github/workflows/release.yml

@@ -102,6 +102,7 @@ jobs:
           API_KEY: ${{ secrets.API_KEY }}
           ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }}
           CLUSTER_JSON: ${{ vars.CLUSTER_JSON }}
+          CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }}
           TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }}
           SERVER_HOST_STAGING: ${{ secrets.SERVER_HOST_STAGING }}
           SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }}
@@ -163,6 +164,7 @@ jobs:
           API_KEY: ${{ secrets.API_KEY }}
           ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }}
           CLUSTER_JSON: ${{ vars.CLUSTER_JSON }}
+          CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }}
           TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }}
           SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }}
           SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }}
@@ -238,6 +240,7 @@ jobs:
           API_KEY: ${{ secrets.API_KEY }}
           ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }}
           CLUSTER_JSON: ${{ vars.CLUSTER_JSON }}
+          CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }}
           TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }}
           SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }}
           SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }}
@@ -308,6 +311,7 @@ jobs:
           API_KEY: ${{ secrets.API_KEY }}
           ADMIN_BOT_API_KEY: ${{ secrets.ADMIN_BOT_API_KEY }}
           CLUSTER_JSON: ${{ vars.CLUSTER_JSON }}
+          CLUSTER_STATE_SOURCE: ${{ vars.CLUSTER_STATE_SOURCE }}
           TURNSTILE_SITE_KEY: ${{ vars.TURNSTILE_SITE_KEY }}
           SERVER_HOST_FALK2: ${{ secrets.SERVER_HOST_FALK2 }}
           SERVER_HOSTS_JSON: ${{ secrets.SERVER_HOSTS_JSON }}

deploy.sh

@@ -252,6 +252,7 @@ SUBDOMAIN=$SUBDOMAIN
 SITE_HOST=$SITE_HOST
 CDN_BASE=$CDN_BASE
 CLUSTER_JSON=$CLUSTER_JSON
+CLUSTER_STATE_SOURCE=$CLUSTER_STATE_SOURCE
 TURNSTILE_SITE_KEY=$TURNSTILE_SITE_KEY
 OTEL_EXPORTER_OTLP_ENDPOINT=$OTEL_EXPORTER_OTLP_ENDPOINT
 OTEL_AUTH_HEADER=$OTEL_AUTH_HEADER

src/server/ClusterCheckin.ts

@@ -0,0 +1,103 @@
+import { z } from "zod";
+import { ServerEnv } from "./ServerEnv";
+
+// Multi-server v2 (docs/MultiServer.md, "Server list v2"): every server
+// tells the API on boot and every CHECKIN_INTERVAL_MS who it is and what it
+// runs, and the API answers with whether this server should take new games.
+// The API's list is what clients read (src/client/ServerList.ts), so a
+// server that is not checking in is not offered to anyone; a deploy that
+// fails halfway can't leave the list claiming servers that aren't there.
+//
+// The reply is only obeyed when CLUSTER_STATE_SOURCE=api. Until then the
+// drain decision stays with today's apex colour poll (ActiveDeployment.ts),
+// so this can ship before the API serves the registry.
+
+export const CHECKIN_INTERVAL_MS = 10_000;
+const CHECKIN_TIMEOUT_MS = 8_000;
+
+// open: runs the site's `latest` and isn't fenced, so it takes new games.
+// draining: anything else; existing games and rejoins still work. The same
+// vocabulary the client reads from GET /cluster.json.
+export const ServerStateSchema = z.enum(["open", "draining"]);
+export type ServerState = z.infer<typeof ServerStateSchema>;
+
+export type ClusterStateSource = "apex" | "api";
+
+export interface CheckinBody {
+  // The hostname players load the page from: the apex behind a load
+  // balancer (SITE_HOST), else this deployment's own host, so beta,
+  // nightly, alpha and branch previews each register under themselves.
+  // Lists are keyed by it. Mirrors, such as the openfront.dev apex
+  // serving nightly, are an alias table in the API, never something a
+  // server reports about itself.
+  site: string;
+  letter: string;
+  host: string;
+  // GIT_COMMIT, the full sha. Clients compare it prefix-tolerantly.
+  version: string;
+  numWorkers: number;
+  liveGames: number;
+}
+
+const CheckinReplySchema = z.object({ state: ServerStateSchema });
+
+/**
+ * What this server reports, or null under local development (`npm run dev`:
+ * no SUBDOMAIN, so no public host), where there is nothing to register.
+ * Every deployed host has one and registers under its own site.
+ */
+export function checkinBody(liveGames: number): CheckinBody | null {
+  const host = ServerEnv.publicHost();
+  if (host === undefined) return null;
+  const { letter, entry } = ServerEnv.clusterSelf();
+  return {
+    site: ServerEnv.siteHost() ?? host,
+    letter,
+    host,
+    version: ServerEnv.gitCommit(),
+    numWorkers: entry.numWorkers,
+    liveGames,
+  };
+}
+
+/**
+ * One check-in. Returns the state the API assigned, or null when the answer
+ * is unusable (the API predates the registry, a bot challenge, a network
+ * error). Callers must treat null as "no change", never as "drain": the
+ * failure mode of an unreachable API is the status quo.
+ */
+export async function sendCheckin(
+  body: CheckinBody,
+  fetchFn: typeof fetch = fetch,
+): Promise<ServerState | null> {
+  try {
+    const res = await fetchFn(`${ServerEnv.jwtIssuer()}/cluster/checkin`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": ServerEnv.apiKey(),
+      },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS),
+    });
+    if (!res.ok) return null;
+    const parsed = CheckinReplySchema.safeParse(await res.json());
+    return parsed.success ? parsed.data.state : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Turn a check-in reply into the lobby service's active flag, but only when
+ * the API is the configured source of that decision. Pure, so the switch is
+ * testable without booting the master.
+ */
+export function applyCheckinState(
+  state: ServerState | null,
+  source: ClusterStateSource,
+  setActive: (active: boolean) => void,
+): void {
+  if (state === null || source !== "api") return;
+  setActive(state === "open");
+}

src/server/IPCBridgeSchema.ts

@@ -46,6 +46,11 @@ export const InternalPublicGamesSchema = z.object({
 const WorkerLobbyListSchema = z.object({
   type: z.literal("lobbyList"),
   lobbies: z.array(z.unknown()),
+  // Games this worker is running, lobbies included. The master sums them
+  // for the cluster check-in (ClusterCheckin.ts), so an operator can tell
+  // when a draining server is empty. Optional for a worker build that
+  // predates it; absent counts as zero.
+  liveGames: z.number().int().min(0).optional(),
 });
 
 const WorkerReadySchema = z.object({

src/server/Master.ts

@@ -7,6 +7,12 @@ import path from "path";
 import { fileURLToPath } from "url";
 import { GameEnv } from "../core/configuration/Config";
 import { fetchSiteColor } from "./ActiveDeployment";
+import {
+  applyCheckinState,
+  CHECKIN_INTERVAL_MS,
+  checkinBody,
+  sendCheckin,
+} from "./ClusterCheckin";
 import { getDescriptor } from "./DesktopRelease";
 import { logger } from "./Logger";
 import { MapPlaylist } from "./MapPlaylist";
@@ -176,15 +182,43 @@ export async function startMaster() {
     log.info(`Master HTTP server listening on port ${PORT}`);
   });
 
+  // Register with the API and keep checking in (docs/MultiServer.md,
+  // "Server list v2"): the API's list is what clients read to find a
+  // server, so a server that isn't checking in isn't offered to anyone.
+  // The reply carries this server's state; it is obeyed only when
+  // CLUSTER_STATE_SOURCE=api, otherwise the apex colour poll below still
+  // decides. Local development (`npm run dev`, no SUBDOMAIN) has no public
+  // host and registers nowhere; every deployed host registers under its own
+  // site.
+  const stateSource = ServerEnv.clusterStateSource();
+  if (checkinBody(0) !== null) {
+    log.info(
+      `Checking in with ${ServerEnv.jwtIssuer()}/cluster/checkin every ${CHECKIN_INTERVAL_MS / 1000}s (state source: ${stateSource})`,
+    );
+    startPolling(async () => {
+      const body = checkinBody(lobbyService.liveGames());
+      if (body === null) return;
+      const state = await sendCheckin(body);
+      applyCheckinState(state, stateSource, (active) =>
+        lobbyService.setActive(active),
+      );
+    }, CHECKIN_INTERVAL_MS);
+  }
+
   // Behind a load balancer (blue/green), only the color the balancer
   // currently routes to should schedule public lobbies. The balancer's
   // /api/health reports the COLOR of whichever deployment answered; colors
   // are deployment-wide, so with several machines per color the poll
   // reaching a sibling — same color, different instanceId — still counts as
   // "the live color is mine". A standalone deployment (no SITE_HOST, or
-  // SITE_HOST is our own host) is always active.
+  // SITE_HOST is our own host) is always active. Not started when the API
+  // is the state source: two deciders would fight over setActive.
   const siteHost = ServerEnv.siteHost();
-  if (siteHost !== undefined && siteHost !== ServerEnv.publicHost()) {
+  if (
+    stateSource === "apex" &&
+    siteHost !== undefined &&
+    siteHost !== ServerEnv.publicHost()
+  ) {
     log.info(`Polling https://${siteHost}/api/health for active deployment`);
     // 5s: this latency is the window after a flip where the newly-active
     // deployment isn't creating public lobbies yet (and the draining one

src/server/MasterLobbyService.ts

@@ -34,6 +34,8 @@ export class MasterLobbyService {
   private readonly workers = new Map<number, Worker>();
   // Worker id => the lobbies it owns.
   private readonly workerLobbies = new Map<number, InternalGameInfo[]>();
+  // Worker id => games it last reported running (lobbies included).
+  private readonly workerLiveGames = new Map<number, number>();
   private readonly readyWorkers = new Set<number>();
   // gameID => consecutive broadcast cycles a hosted lobby has lost the
   // per-creator dedup or overflowed the cluster-wide cap. Losing once can be
@@ -69,6 +71,7 @@ export class MasterLobbyService {
           break;
         case "lobbyList":
           this.workerLobbies.set(workerId, this.validLobbies(msg.lobbies));
+          this.workerLiveGames.set(workerId, msg.liveGames ?? 0);
           break;
       }
     });
@@ -95,9 +98,18 @@ export class MasterLobbyService {
   removeWorker(workerId: number) {
     this.workers.delete(workerId);
     this.workerLobbies.delete(workerId);
+    this.workerLiveGames.delete(workerId);
     this.readyWorkers.delete(workerId);
   }
 
+  // Games running on this server, summed over the workers' last reports.
+  // Reported to the API at check-in (ClusterCheckin.ts).
+  liveGames(): number {
+    let total = 0;
+    for (const n of this.workerLiveGames.values()) total += n;
+    return total;
+  }
+
   isHealthy(): boolean {
     // We consider the lobby service healthy if at least half of the workers are ready.
     // This allows for some leeway if a worker crashes.

src/server/ServerEnv.ts

@@ -243,6 +243,14 @@ export class ServerEnv {
     const v = process.env.SITE_HOST;
     return v && v.length > 0 ? v : undefined;
   }
+  // Where the drain decision comes from (docs/MultiServer.md, "Server list
+  // v2"): "apex" is today's /api/health colour poll of the site host; "api"
+  // obeys the state the API assigns at check-in (ClusterCheckin.ts). Any
+  // other value, or none, means apex, so a deploy that doesn't set it is
+  // unchanged.
+  static clusterStateSource(): "apex" | "api" {
+    return process.env.CLUSTER_STATE_SOURCE === "api" ? "api" : "apex";
+  }
   static otelEnabled(): boolean {
     return (
       ServerEnv.gameEnv !== GameEnv.Dev &&

src/server/WorkerLobbyService.ts

@@ -186,6 +186,7 @@ export class WorkerLobbyService {
     this.sendToMaster({
       type: "lobbyList",
       lobbies: [...publicLobbies, ...hostedLobbies],
+      liveGames: this.gm.activeGames(),
     } satisfies WorkerLobbyList);
   }