Add stratified key-size sampler with validation harness

apps/api/package.json

@@ -40,7 +40,9 @@
     "docs:index:valkey": "SOURCE=valkey ts-node -r tsconfig-paths/register scripts/index-docs.ts",
     "docs:index:redis": "SOURCE=redis ts-node -r tsconfig-paths/register scripts/index-docs.ts",
     "docs:clean": "rm -rf data/docs data/chunks data/vector-store",
-    "seed:demo": "ts-node -r tsconfig-paths/register scripts/seed-demo-data.ts"
+    "seed:demo": "ts-node -r tsconfig-paths/register scripts/seed-demo-data.ts",
+    "cache-sim:seed": "ts-node -r tsconfig-paths/register scripts/cache-simulator/seed.ts",
+    "cache-sim:validate": "ts-node -r tsconfig-paths/register scripts/cache-simulator/validate.ts"
   },
   "dependencies": {
     "@betterdb/agent-memory": "workspace:*",

apps/api/scripts/cache-simulator/scenarios.ts

@@ -0,0 +1,97 @@
+export interface SeedCommand {
+  type: 'string' | 'hash' | 'list' | 'set' | 'zset';
+  key: string;
+  value: string | string[];
+  ttlSeconds?: number;
+}
+
+export interface Scenario {
+  name: string;
+  tolerance: number;
+  runs: number;
+  generate(seed: number): Iterable<SeedCommand>;
+}
+
+export function mulberry32(seed: number): () => number {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6d2b79f5) >>> 0;
+    let t = state;
+    t = Math.imul(t ^ (t >>> 15), t | 1);
+    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+const between = (random: () => number, min: number, max: number): number => min + Math.floor(random() * (max - min + 1));
+const text = (length: number): string => 'x'.repeat(length);
+const items = (count: number, length: number, prefix: string): string[] => Array.from({ length: count }, (_, index) => `${prefix}${index}:${text(length)}`);
+
+function* mixed(random: () => number, prefix: string, counts: { string: number; hash: number; list: number; set: number; zset: number }): Iterable<SeedCommand> {
+  for (let index = 0; index < counts.string; index++) yield { type: 'string', key: `${prefix}:s:${index}`, value: text(between(random, 32, 512)) };
+  for (let index = 0; index < counts.hash; index++) yield { type: 'hash', key: `${prefix}:h:${index}`, value: items(between(random, 10, 50), 32, 'f') };
+  for (let index = 0; index < counts.list; index++) yield { type: 'list', key: `${prefix}:l:${index}`, value: items(between(random, 20, 200), 64, 'i') };
+  for (let index = 0; index < counts.set; index++) yield { type: 'set', key: `${prefix}:t:${index}`, value: items(between(random, 10, 100), 32, 'm') };
+  for (let index = 0; index < counts.zset; index++) yield { type: 'zset', key: `${prefix}:z:${index}`, value: items(between(random, 10, 100), 32, 'm') };
+}
+
+export const SCENARIOS: Scenario[] = [
+  {
+    name: 'homogeneous-small',
+    tolerance: 0.05,
+    runs: 10,
+    *generate(seed) {
+      for (let index = 0; index < 100_000; index++) yield { type: 'string', key: `hs:${seed}:${index}`, value: text(64) };
+    },
+  },
+  {
+    name: 'homogeneous-large',
+    tolerance: 0.05,
+    runs: 10,
+    *generate(seed) {
+      for (let index = 0; index < 20_000; index++) yield { type: 'string', key: `hl:${seed}:${index}`, value: text(16 * 1024) };
+    },
+  },
+  {
+    name: 'mixed',
+    tolerance: 0.05,
+    runs: 10,
+    generate: (seed) => mixed(mulberry32(seed), `mx:${seed}`, { string: 50_000, hash: 10_000, list: 5_000, set: 5_000, zset: 5_000 }),
+  },
+  {
+    name: 'pareto',
+    tolerance: 0.1,
+    runs: 10,
+    *generate(seed) {
+      const random = mulberry32(seed);
+      for (let index = 0; index < 50_000; index++) {
+        const u = 1 - random();
+        yield { type: 'string', key: `pa:${seed}:${index}`, value: text(Math.min(1024 * 1024, Math.floor(100 / u ** (1 / 1.5)))) };
+      }
+    },
+  },
+  {
+    name: 'ttl-mix',
+    tolerance: 0.05,
+    runs: 10,
+    *generate(seed) {
+      const random = mulberry32(seed);
+      for (let index = 0; index < 50_000; index++) {
+        yield { type: 'string', key: `tt:${seed}:${index}`, value: text(256), ...(random() < 0.5 ? { ttlSeconds: 3600 } : {}) };
+      }
+    },
+  },
+  {
+    name: 'scale-500k',
+    tolerance: 0.05,
+    runs: 3,
+    *generate(seed) {
+      const random = mulberry32(seed);
+      for (let index = 0; index < 500_000; index++) {
+        yield random() < 0.8
+          ? { type: 'string', key: `sc:${seed}:${index}`, value: text(between(random, 100, 300)) }
+          : { type: 'hash', key: `sc:${seed}:${index}`, value: items(between(random, 5, 20), 32, 'f') };
+      }
+    },
+  },
+];

apps/api/scripts/cache-simulator/seed.ts

@@ -0,0 +1,72 @@
+import Valkey from 'iovalkey';
+import { SCENARIOS, type SeedCommand } from './scenarios';
+
+export interface Target {
+  url: string;
+  db: number;
+}
+
+export function parseTarget(argv: string[]): Target & { scenario?: string; seed: number } {
+  const value = (flag: string): string | undefined => {
+    const index = argv.indexOf(flag);
+    return index >= 0 ? argv[index + 1] : undefined;
+  };
+  const url = value('--url');
+  const db = Number(value('--db'));
+  if (!url || !Number.isInteger(db) || db <= 0) {
+    throw new Error('Pass --url <redis-url> and --db <index greater than 0>; the target database must be empty');
+  }
+  return { url, db, scenario: value('--scenario'), seed: Number(value('--seed') ?? 1) };
+}
+
+export async function connect(target: Target): Promise<Valkey> {
+  const client = new Valkey(target.url, { db: target.db });
+  if ((await client.dbsize()) !== 0) {
+    client.disconnect();
+    throw new Error(`Database ${target.db} is not empty; refusing to seed`);
+  }
+  return client;
+}
+
+export async function seed(client: Valkey, commands: Iterable<SeedCommand>): Promise<number> {
+  let pipeline = client.pipeline();
+  let queued = 0;
+  let total = 0;
+  for (const command of commands) {
+    if (command.type === 'string') pipeline.set(command.key, command.value as string);
+    if (command.type === 'hash') pipeline.hset(command.key, Object.fromEntries((command.value as string[]).map((field) => [field, field])));
+    if (command.type === 'list') pipeline.rpush(command.key, ...(command.value as string[]));
+    if (command.type === 'set') pipeline.sadd(command.key, ...(command.value as string[]));
+    if (command.type === 'zset') pipeline.zadd(command.key, ...(command.value as string[]).flatMap((member, index) => [String(index), member]));
+    if (command.ttlSeconds !== undefined) pipeline.expire(command.key, command.ttlSeconds);
+    queued += 1;
+    total += 1;
+    if (queued === 1000) {
+      await pipeline.exec();
+      pipeline = client.pipeline();
+      queued = 0;
+    }
+  }
+  if (queued > 0) await pipeline.exec();
+  return total;
+}
+
+async function main(): Promise<void> {
+  const target = parseTarget(process.argv.slice(2));
+  const scenario = SCENARIOS.find((candidate) => candidate.name === target.scenario);
+  if (!scenario) throw new Error(`Pass --scenario with one of: ${SCENARIOS.map((candidate) => candidate.name).join(', ')}`);
+  const client = await connect(target);
+  try {
+    const count = await seed(client, scenario.generate(target.seed));
+    console.log(`Seeded ${count} keys for ${scenario.name} into db ${target.db}`);
+  } finally {
+    client.disconnect();
+  }
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error instanceof Error ? error.message : error);
+    process.exit(1);
+  });
+}

apps/api/scripts/cache-simulator/validate.ts

@@ -0,0 +1,60 @@
+import type Valkey from 'iovalkey';
+import { estimate } from '../../../../proprietary/cache-simulator/size-estimator';
+import { sampleNodes, SAMPLE_BUDGET, SAMPLE_DEADLINE_MS, SAMPLE_HARD_CAP } from '../../../../proprietary/cache-simulator/stratified-sampler';
+import type { SamplerClient } from '../../../../proprietary/cache-simulator/sample-types';
+import { SCENARIOS } from './scenarios';
+import { connect, parseTarget, seed } from './seed';
+
+async function groundTruth(client: Valkey): Promise<number> {
+  let cursor = '0';
+  let total = 0;
+  do {
+    const [next, keys] = await client.scan(cursor, 'COUNT', 1000);
+    cursor = next;
+    const pipeline = client.pipeline();
+    keys.forEach((key) => pipeline.call('MEMORY', 'USAGE', key, 'SAMPLES', '0'));
+    for (const [error, size] of (await pipeline.exec()) ?? []) if (!error && typeof size === 'number') total += size;
+  } while (cursor !== '0');
+  return total;
+}
+
+function adapter(client: Valkey): SamplerClient {
+  return {
+    call: (command: string, args: string[]) => client.call(command, ...args),
+    getClient: () => client,
+    getDbSize: () => client.dbsize(),
+  } as unknown as SamplerClient;
+}
+
+async function main(): Promise<void> {
+  const target = parseTarget(process.argv.slice(2));
+  const client = await connect(target);
+  let failed = false;
+  try {
+    for (const scenario of SCENARIOS.filter((candidate) => !target.scenario || candidate.name === target.scenario)) {
+      let covered = 0;
+      let worst = 0;
+      for (let run = 1; run <= scenario.runs; run++) {
+        await seed(client, scenario.generate(run));
+        const truth = await groundTruth(client);
+        const result = estimate(await sampleNodes([{ name: 'validate', client: adapter(client) }], { budget: SAMPLE_BUDGET, deadlineMs: SAMPLE_DEADLINE_MS, hardCap: SAMPLE_HARD_CAP }), Date.now());
+        const error = Math.abs(result.totalBytes - truth) / truth;
+        worst = Math.max(worst, error);
+        if (Math.abs(result.totalBytes - truth) <= result.ci95Bytes + 0.001 * truth) covered += 1;
+        await client.flushdb();
+      }
+      const required = scenario.runs - Math.floor(scenario.runs / 10);
+      const pass = worst <= scenario.tolerance && covered >= required;
+      failed ||= !pass;
+      console.log(`${pass ? 'PASS' : 'FAIL'} ${scenario.name}: worst error ${(worst * 100).toFixed(2)}% (limit ${scenario.tolerance * 100}%), CI coverage ${covered}/${scenario.runs} (need ${required})`);
+    }
+  } finally {
+    client.disconnect();
+  }
+  if (failed) process.exit(1);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : error);
+  process.exit(1);
+});

packages/shared/src/index.ts

@@ -32,3 +32,5 @@ export * from './types/otel-traces';
 export * from './types/cve';
 export * from './utils/cve-modules';
 export * from './types/workspace';
+export * from './types/cache-simulator.types';
+export * from './utils/cache-simulator';

packages/shared/src/types/cache-simulator.types.ts

@@ -0,0 +1,42 @@
+export type CacheSimulatorStatus = 'sampling' | 'ready' | 'failed';
+
+export interface CacheSimulatorInstant {
+  usedMemory: number;
+  maxmemory: number;
+  maxmemoryPolicy: string;
+  totalKeys: number;
+}
+
+export interface CacheSimulatorTypeEstimate {
+  type: string;
+  keys: number;
+  avgBytes: number;
+  stdBytes: number;
+  sampled: number;
+}
+
+export interface CacheSimulatorEstimate {
+  totalBytes: number;
+  ci95Bytes: number;
+  avgKeyBytes: number;
+  stdKeyBytes: number;
+  sampleCount: number;
+  byType: CacheSimulatorTypeEstimate[];
+  sampledAt: number;
+  partial: boolean;
+}
+
+export interface CacheSimulatorGrowth {
+  keysPerDay: number;
+  bytesPerDay: number;
+  days: number;
+}
+
+export interface CacheSimulatorSnapshot {
+  status: CacheSimulatorStatus;
+  progress: { sampled: number; target: number } | null;
+  instant: CacheSimulatorInstant;
+  estimate: CacheSimulatorEstimate | null;
+  growth: CacheSimulatorGrowth | null;
+  error?: string;
+}

packages/shared/src/utils/cache-simulator/eviction-copy.ts

@@ -0,0 +1,20 @@
+const COPY: Record<string, string> = {
+  noeviction: 'Writes start failing with OOM errors once memory is full; reads keep working.',
+  'allkeys-lru': 'The least recently used keys are evicted to make room, whether or not they have a TTL.',
+  'allkeys-lfu': 'The least frequently used keys are evicted to make room, whether or not they have a TTL.',
+  'allkeys-random': 'Random keys are evicted to make room, whether or not they have a TTL.',
+  'allkeys-lrm': 'The least recently modified keys are evicted to make room, whether or not they have a TTL.',
+  'volatile-lru': 'The least recently used keys that have a TTL are evicted; once none are left, writes fail with OOM errors.',
+  'volatile-lfu': 'The least frequently used keys that have a TTL are evicted; once none are left, writes fail with OOM errors.',
+  'volatile-random': 'Random keys that have a TTL are evicted; once none are left, writes fail with OOM errors.',
+  'volatile-ttl': 'Keys closest to expiring are evicted first; once no keys with a TTL are left, writes fail with OOM errors.',
+  'volatile-lrm': 'The least recently modified keys that have a TTL are evicted; once none are left, writes fail with OOM errors.',
+};
+
+export const EVICTION_POLICIES = Object.keys(COPY);
+
+export const UNKNOWN_EVICTION_COPY = 'This eviction policy is not recognised, so its behaviour at maxmemory cannot be described.';
+
+export function evictionConsequence(policy: string): string {
+  return COPY[policy.toLowerCase()] ?? UNKNOWN_EVICTION_COPY;
+}

packages/shared/src/utils/cache-simulator/index.ts

@@ -0,0 +1,3 @@
+export * from './projection';
+export * from './node-catalog';
+export * from './eviction-copy';