Skip to content

Commit 041ccaf

Browse files
authored
Merge pull request #737 from Mikewaves7/security/issue-603-prewave-be-008-add-resumable-websocket-streams
feat: add resumable WebSocket streams with backfill tokens
2 parents fc253e6 + 432c5f8 commit 041ccaf

8 files changed

Lines changed: 904 additions & 20 deletions

File tree

‎docs/FINALIZED_CURSOR.md‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Canonical Finalized-Block Ingestion Cursor (PREWAVE-BE-001)
22

3-
`_src/indexer/finalized-cursor/` persists one monotonic cursor per
4-
``(chain_id, release_id, projection_id)`` in `v2_finalized_cursors`
3+
`src/indexer/finalized-cursor/` persists one monotonic cursor per
4+
`(chain_id, release_id, projection_id)` in `v2_finalized_cursors`
55
(migration `1725000002000-CreateV2FinalizedCursors`).
66

77
## Boundaries
@@ -56,10 +56,10 @@ Once a binding is configured, `/health/ready` gains a critical
5656
Prometheus gauges are labelled only by `chain_id`, `release_id`, `projection_id` and a fixed
5757
`boundary` or `kind`:
5858

59-
- indexer_finalized_cursor_block{boundary="observed|safe|finalized|indexed"}
60-
- indexer_finalized_cursor_lag_blocks_blocks`k{kind="finality|index"}
61-
- `indexer_finalized_cursor_head_age_seconds`
62-
- `indexer_finalized_cursor_ready`
59+
- "indexer_finalized_cursor_block" {boundary="observed|safe|finalized|indexed"}
60+
- "indexer_finalized_cursor_lag_blocks" {kind="finality|index"}
61+
- "indexer_finalized_cursor_head_age_seconds"
62+
- "indexer_finalized_cursor_ready"
6363

6464
## Configuration
6565

@@ -143,6 +143,7 @@ shift already-returned pages. A cursor whose position is ahead of the current
143143

144144
```bash
145145
n pjest finalized-cursor
146+
OP_SEPOLIA_RPC_URL=https://sepolia.optimism.io n pjest finalized-cursor.op-sepolia
146147
```
147148

148149
The cursor suite covers tampering, replay across endpoints, filter change,

‎prisma/schema.prisma‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,113 @@ model ChainEvent {
377377
@@index([decoderVersion])
378378
}
379379

380+
// ---------------------------------------------------------------------------
381+
// PREWAVE-BE-008: Resumable WebSocket streams with backfill tokens.
382+
//
383+
// A StreamSession is opened by a client subscribing to a projection stream.
384+
// Each delivered envelope is persisted as a StreamEvent so that a reconnect
385+
// can be resumed from an opaque resume token. Resume tokens are stored
386+
// hashed (never the raw token) and are bound to chain/release/block/
387+
// projection version so that a wrong-release or wrong-chain resume fails
388+
// closed. Backfill is served from StreamEvent rows within a bounded window.
389+
//
390+
// Security: envelopes carry only routing metadata and a projection payload
391+
// reference — no secrets, authorization material, or protected rationale.
392+
// ---------------------------------------------------------------------------
393+
394+
model StreamSession {
395+
id String @id @default(uuid())
396+
397+
// Chain-derived binding — every session is pinned to a single chain,
398+
// release, deployment block, and artifact checksum.
399+
chainId Int
400+
releaseId String
401+
deploymentBlock BigInt
402+
artifactChecksum String
403+
404+
// Logical projection stream, e.g. "claims", "rewards"
405+
projection String
406+
projectionVersion Int
407+
408+
// Opaque resume token, stored hashed. Raw token is only returned to client.
409+
resumeTokenHash String @unique
410+
// When the current resume token expires; expired tokens force bounded resync.
411+
resumeTokenExpiresAt DateTime
412+
413+
// Last event successfully acknowledged by the client (cursor).
414+
lastAckedEventId String?
415+
lastAckedCursor BigInt @default(0)
416+
417+
// Lifecycle: ACTIVE | CLOSED | EXPIRED | RESYNC_REQUIRED
418+
status String @default("ACTIVE")
419+
420+
createdAt DateTime @default(now())
421+
updatedAt DateTime @updatedAt
422+
423+
events StreamEvent[]
424+
425+
@@index([chainId, releaseId, projection])
426+
@@index([status, resumeTokenExpiresAt])
427+
@@index([resumeTokenHash])
428+
}
429+
430+
model StreamEvent {
431+
id String @id @default(uuid())
432+
433+
sessionId String
434+
session StreamSession @relation(fields: [sessionId], references: [id], onDelete: Cascade)
435+
436+
// Monotonic per-session cursor used for gap/duplicate detection.
437+
cursor BigInt
438+
439+
// Versioned envelope fields (see stream-envelope.types.ts).
440+
envelopeVersion Int
441+
eventId String
442+
chainId Int
443+
releaseId String
444+
blockNumber BigInt
445+
blockHash String
446+
freshness String // "observed" | "safe" | "finalized"
447+
projectionVersion Int
448+
449+
// Reference to the canonical read-model row this envelope projects.
450+
// The actual payload is resolved from the canonical read model on
451+
// delivery — never stored inline here.
452+
projectionRef String
453+
454+
// JSON-encoded routing metadata only (no protected payload).
455+
metadata String @default("{}")
456+
457+
// Delivery state for backfill and idempotency.
458+
deliveredAt DateTime?
459+
createdAt DateTime @default(now())
460+
461+
@@unique([sessionId, cursor])
462+
@@unique([sessionId, eventId])
463+
@@index([sessionId, cursor])
464+
@@index([chainId, releaseId, blockNumber])
465+
}
466+
467+
model StreamBackfillRequest {
468+
id String @id @default(uuid())
469+
470+
sessionId String
471+
472+
// Bounded backfill window requested by the client.
473+
fromCursor BigInt
474+
toCursor BigInt
475+
476+
// Outcome: PENDING | SERVED | REJECTED_GAP | REJECTED_EXPIRED | REJECTED_RELEASE
477+
status String @default("PENDING")
478+
reason String?
479+
480+
requestedAt DateTime @default(now())
481+
servedAt DateTime?
482+
483+
@@index([sessionId, requestedAt])
484+
@@index([status])
485+
}
486+
380487
// ---------------------------------------------------------------------------
381488
// Feature flags & configuration rollback (PREWAVE-BE-034)
382489
// ---------------------------------------------------------------------------

‎src/blockchain/blockchain.controller.ts‎

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@ import {
44
Post,
55
Body,
66
Param,
7+
Query,
78
Headers,
89
Req,
910
HttpCode,
1011
HttpException,
12+
HttpStatus,
1113
UnauthorizedException,
1214
} from '@nestjs/common';
1315
import { ApiTags } from '@nestjs/swagger';
@@ -27,6 +29,14 @@ import {
2729
DeadLetterBatchControlRequest,
2830
DeadLetterBatchControlResult,
2931
} from './types';
32+
import {
33+
StreamEnvelope,
34+
StreamEnvelopeVersion,
35+
StreamResumeTokenPayload,
36+
StreamBackfillResponse,
37+
StreamReconciliationRequirement,
38+
} from './stream.types';
39+
import { StreamResumeService } from './stream-resume.service';
3040
import { WebhookSignatureService } from './webhook-signature.service';
3141

3242
/**
@@ -47,6 +57,7 @@ export class BlockchainController {
4757
private reconciliation: ReconciliationService,
4858
private stateService: BlockchainStateService,
4959
private voteResolver: WeightedVoteResolutionService,
60+
private streamResume: StreamResumeService,
5061
private readinessService: ReadinessService,
5162
private replayService: DeadLetterReplayService,
5263
private webhookSignatureService: WebhookSignatureService,
@@ -294,6 +305,94 @@ export class BlockchainController {
294305
};
295306
}
296307

308+
/**
309+
* Get the current stream envelope metadata and a fresh resume token.
310+
* GET /api/v1/blockchain/streams/envelope
311+
*/
312+
@Get('streams/envelope')
313+
async getStreamEnvelope(): Promise<StreamEnvelope> {
314+
return this.streamResume.getCurrentEnvelope();
315+
}
316+
317+
/**
318+
* Resume a stream from an opaque resume token.
319+
* POST /api/v1/blockchain/streams/resume
320+
*/
321+
@Post('streams/resume')
322+
async resumeStream(
323+
@Body() body: { resumeToken: string },
324+
): Promise<StreamBackfillResponse> {
325+
if (!body || typeof body.resumeToken !== 'string' || body.resumeToken.length === 0) {
326+
throw new HttpException(
327+
{
328+
error: 'invalid_resume_token',
329+
message: 'resumeToken is required',
330+
},
331+
HttpStatus.BAD_REQUEST,
332+
);
333+
}
334+
335+
const result = await this.streamResume.resume(body.resumeToken);
336+
if (!result.ok) {
337+
throw new HttpException(result.body, result.status);
338+
}
339+
return result.body;
340+
}
341+
342+
/**
343+
* Bounded HTTP backfill from a cursor.
344+
* GET /api/v1/blockchain/streams/backfill?cursor=...&limit=...
345+
*/
346+
@Get('streams/backfill')
347+
async backfillStream(
348+
@Query('cursor') cursor: string,
349+
@Query('limit') limit?: string,
350+
@Headers('if-none-match') etag?: string,
351+
): Promise<StreamBackfillResponse> {
352+
if (!cursor || cursor.length === 0) {
353+
throw new HttpException(
354+
{ error: 'invalid_cursor', message: 'cursor is required' },
355+
HttpStatus.BAD_REQUEST,
356+
);
357+
}
358+
359+
const parsedLimit = limit === undefined ? undefined : Number(limit);
360+
if (
361+
parsedLimit !== undefined &&
362+
(!Number.isFinite(parsedLimit) || parsedLimit <= 0)
363+
) {
364+
throw new HttpException(
365+
{ error: 'invalid_limit', message: 'limit must be a positive integer' },
366+
HttpStatus.BAD_REQUEST,
367+
);
368+
}
369+
370+
const result = await this.streamResume.backfill(cursor, parsedLimit, etag);
371+
if (!result.ok) {
372+
throw new HttpException(result.body, result.status);
373+
}
374+
return result.body;
375+
}
376+
377+
/**
378+
* Decode a resume token for observability / client reconciliation.
379+
* Tokens are opaque to clients but can be inspected to determine whether a
380+
* canonical resync is required. No protected payload is returned.
381+
* GET /api/v1/blockchain/streams/token/inspect?token=...
382+
*/
383+
@Get('streams/token/inspect')
384+
async inspectResumeToken(
385+
@Query('token') token: string,
386+
): Promise<StreamReconciliationRequirement> {
387+
if (!token || token.length === 0) {
388+
throw new HttpException(
389+
{ error: 'invalid_resume_token', message: 'token is required' },
390+
HttpStatus.BAD_REQUEST,
391+
);
392+
}
393+
return this.streamResume.inspectToken(token);
394+
}
395+
297396
/**
298397
* Replay a single dead-letter message.
299398
*
@@ -422,7 +521,7 @@ export class BlockchainController {
422521
{
423522
...baseVote,
424523
userId: 'user2',
425-
verdict: 'FALSE',
524+
verdict: 'ALSE',
426525
userReputation: 70,
427526
stakeAmount: '100',
428527
},

‎src/blockchain/blockchain.module.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { Module } from '@nestjs/common';
2-
import { TypeOrmModule } from '@nestjs/typeorm';
1+
import { Module } from '@intest/common';
2+
import { TypeOrmModule } from '@intest/typeorm';
33
import { BlockchainIndexerService } from './blockchain-indexer.service';
44
import {
55
ProcessedEvent,
@@ -16,6 +16,12 @@ import { WeightedVoteResolutionService } from './weighted-vote-resolution.servic
1616
import { SybilResistanceModule } from '../sybil-resistance/sybil-resistance.module';
1717
import { StartupValidationService } from './startup-validation.service';
1818
import { DeterministicRpcFailoverService } from './deterministic-rpc-failover.service';
19+
import { StreamEnvelopeService } from './stream-envelope.service';
20+
import { ResumeTokenService } from './resume-token.service';
21+
import { StreamBackfillService } from './stream-backfill.service';
22+
import { StreamGapDetectorService } from './stream-gap-detector.service';
23+
import { StreamSessionManagerService } from './stream-session-manager.service';
24+
import { StreamController } from './stream.controller';
1925
import { LeaderElectionService } from './leader-election.service';
2026
import { Lease } from './lease.entity';
2127
import { FencingToken } from './fencing-token.entity';
@@ -52,13 +58,18 @@ import { WebhookController } from './webhook.controller';
5258
EventIndexingService,
5359
StartupValidationService,
5460
DeterministicRpcFailoverService,
61+
StreamEnvelopeService,
62+
ResumeTokenService,
63+
StreamBackfillService,
64+
StreamGapDetectorService,
65+
StreamSessionManagerService,
5566
LeaderElectionService,
5667
DeadLetterReplayService,
5768
WeightedVoteResolutionService,
5869
WebhookSignatureService,
5970
WebhookReplayGuardService,
6071
],
61-
controllers: [BlockchainController, WebhookController],
72+
controllers: [BlockchainController, StreamController, WebhookController],
6273
exports: [
6374
BlockchainIndexerService,
6475
BlockchainStateService,
@@ -68,6 +79,11 @@ import { WebhookController } from './webhook.controller';
6879
ReconciliationService,
6980
EventIndexingService,
7081
DeterministicRpcFailoverService,
82+
StreamEnvelopeService,
83+
ResumeTokenService,
84+
StreamBackfillService,
85+
StreamGapDetectorService,
86+
StreamSessionManagerService,
7187
LeaderElectionService,
7288
DeadLetterReplayService,
7389
WeightedVoteResolutionService,

‎src/blockchain/entities/processed-event.entity.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,16 @@ export class ProcessedEvent {
5050
* state mutation this event applied (e.g. the Transfer amounts). Nullable for
5151
* backwards compatibility with rows written before this column existed.
5252
*/
53-
@Column({name: 'payload', type: 'simple-json', nullable: true})
54-
payload: Record<string, any> | null;
53+
@Column({ name: 'payload', type: 'simple-json', nullable: true })
54+
payload: Record<string, unknown> | null;
55+
56+
/**
57+
* Monotonic cursor assigned at ingestion time. Used to build opaque resume
58+
* tokens and to detect gaps/duplicates on WebSocket reconnect.
59+
*/
60+
@Column({ name: 'cursor', type: 'bigint', nullable: true })
61+
@Index()
62+
cursor: number | null;
5563

5664
/**
5765
* Set when the chain reports this log as removed (e.g. a reorg or a filter delivering
@@ -67,4 +75,4 @@ export class ProcessedEvent {
6775
default: () => 'CURRENT_TIMESTAMP',
6876
})
6977
processedAt: Date;
70-
}
78+
}

0 commit comments

Comments
 (0)