From 8a311cc17db1fe2ae3778a3c3b5fe3cb1c07dfe9 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Wed, 10 Jun 2026 23:56:21 +0100 Subject: [PATCH] =?UTF-8?q?feat(site):=2039=20more=20interactive=20figures?= =?UTF-8?q?=20=E2=80=94=20wave=203=20across=20thin=20phases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the figure system into the phases that were still sparse: LLM engineering, multimodal, agents depth, alignment, plus vision/speech/genai remainders. Five new module files (1,959 LOC) on the shared LF toolkit: - figures-llmeng.js (P11/P13, 8): few-shot curve, chain-of-thought, constrained decoding, prompt-cache hit, semantic cache, function-call args, LLM-judge rubric, lost-in-the-middle - figures-multimodal.js (P12, 7): contrastive matrix, cross-attention fusion, modality projection, CFG guidance scale, VQ codebook, video patches, CTC align - figures-agents2.js (P14/P16, 8): ReWOO plan, tree-of-thoughts, self-refine, memory blocks, Voyager skills, LangGraph state, orchestration patterns, debate - figures-alignment2.js (P9/P18, 8): PPO clip, reward model, constitutional AI, actor-critic, interpretability probe, SAE features, jailbreak defense, scalable oversight - figures-foundations2.js (P4/P6/P8, 8): augmentation, transfer learning, BN train/eval, CTC collapse, MFCC pipeline, autoencoder bottleneck, normalizing flow, score matching Embedded in 39 figure-free lessons. Validated headless: all 173 registered figures (16 core + 157 module) mount with zero console errors; PPO clip, CLIP contrastive matrix, and tree-of-thoughts verified in light and dark. --- .../05-transfer-learning/docs/en.md | 4 + .../14-vision-transformers/docs/en.md | 4 + .../17-self-supervised-vision/docs/en.md | 4 + .../03-audio-classification/docs/en.md | 4 + .../04-speech-recognition-asr/docs/en.md | 4 + .../docs/en.md | 4 + .../11-audio-generation/docs/en.md | 4 + .../docs/en.md | 4 + .../07-actor-critic-a2c-a3c/docs/en.md | 4 + .../08-ppo/docs/en.md | 4 + .../09-reward-modeling-rlhf/docs/en.md | 4 + .../01-prompt-engineering/docs/en.md | 4 + .../02-few-shot-cot/docs/en.md | 4 + .../05-context-engineering/docs/en.md | 4 + .../10-evaluation/docs/en.md | 4 + .../11-caching-cost/docs/en.md | 4 + .../15-prompt-caching/docs/en.md | 4 + .../03-blip2-qformer-bridge/docs/en.md | 4 + .../docs/en.md | 4 + .../docs/en.md | 4 + .../docs/en.md | 4 + .../docs/en.md | 4 + .../docs/en.md | 4 + .../24-multimodal-rag-cross-modal/docs/en.md | 4 + .../02-function-calling-deep-dive/docs/en.md | 4 + .../04-structured-output/docs/en.md | 4 + .../02-rewoo-plan-and-execute/docs/en.md | 4 + .../04-tree-of-thoughts-lats/docs/en.md | 4 + .../05-self-refine-and-critic/docs/en.md | 4 + .../docs/en.md | 4 + .../10-skill-libraries-voyager/docs/en.md | 4 + .../13-langgraph-stateful-graphs/docs/en.md | 4 + .../28-orchestration-patterns/docs/en.md | 4 + .../07-society-of-mind-debate/docs/en.md | 4 + .../05-constitutional-ai-rlaif/docs/en.md | 4 + .../docs/en.md | 4 + .../docs/en.md | 4 + .../13-many-shot-jailbreaking/docs/en.md | 4 + .../docs/en.md | 4 + site/figures-agents2.js | 450 ++++++++++++++++++ site/figures-alignment2.js | 336 +++++++++++++ site/figures-foundations2.js | 406 ++++++++++++++++ site/figures-llmeng.js | 370 ++++++++++++++ site/figures-multimodal.js | 397 +++++++++++++++ site/lesson.html | 35 +- 45 files changed, 2135 insertions(+), 15 deletions(-) create mode 100644 site/figures-agents2.js create mode 100644 site/figures-alignment2.js create mode 100644 site/figures-foundations2.js create mode 100644 site/figures-llmeng.js create mode 100644 site/figures-multimodal.js diff --git a/phases/04-computer-vision/05-transfer-learning/docs/en.md b/phases/04-computer-vision/05-transfer-learning/docs/en.md index 2901286483..f4bf74621d 100644 --- a/phases/04-computer-vision/05-transfer-learning/docs/en.md +++ b/phases/04-computer-vision/05-transfer-learning/docs/en.md @@ -115,6 +115,10 @@ Transfer-learning runs need two numbers you would not track on a scratch run: If fine-tuned is less than pretrained-only, you have a learning-rate or BN bug. Always print both. +```figure +transfer-learning +``` + ## Build It ### Step 1: Load a pretrained backbone and inspect it diff --git a/phases/04-computer-vision/14-vision-transformers/docs/en.md b/phases/04-computer-vision/14-vision-transformers/docs/en.md index b41275ebf2..e82618a1cf 100644 --- a/phases/04-computer-vision/14-vision-transformers/docs/en.md +++ b/phases/04-computer-vision/14-vision-transformers/docs/en.md @@ -125,6 +125,10 @@ Masked Autoencoder (He et al., 2022): mask 75% of patches at random, train the e MAE makes ViT trainable on ImageNet-1k alone, hits SOTA, and is the current default self-supervised recipe. +```figure +batchnorm-inference +``` + ## Build It ### Step 1: Patch embedding diff --git a/phases/04-computer-vision/17-self-supervised-vision/docs/en.md b/phases/04-computer-vision/17-self-supervised-vision/docs/en.md index ea115ef5b1..1ef09d63dc 100644 --- a/phases/04-computer-vision/17-self-supervised-vision/docs/en.md +++ b/phases/04-computer-vision/17-self-supervised-vision/docs/en.md @@ -105,6 +105,10 @@ After self-supervised pretraining, the standard evaluation is a **linear probe** Linear probe is a pure measure of feature quality; fine-tuning typically adds 2-5 points but also mixes in the effect of head retraining. +```figure +data-augmentation +``` + ## Build It ### Step 1: Two-view augmentation pipeline diff --git a/phases/06-speech-and-audio/03-audio-classification/docs/en.md b/phases/06-speech-and-audio/03-audio-classification/docs/en.md index 50eb0bfc69..250f4d2398 100644 --- a/phases/06-speech-and-audio/03-audio-classification/docs/en.md +++ b/phases/06-speech-and-audio/03-audio-classification/docs/en.md @@ -49,6 +49,10 @@ ESC-50: 50 classes, 40 clips each — balanced, easy. UrbanSound8K: 10 classes, | AudioSet mAP | 0.485 (AST) | 0.548 (BEATs-iter3) | HEAR leaderboard 2026 | | Speech Commands v2 | 98% (CNN) | 99.0% (Audio-MAE) | HEAR v2 results | +```figure +mfcc-pipeline +``` + ## Build It ### Step 1: featurize diff --git a/phases/06-speech-and-audio/04-speech-recognition-asr/docs/en.md b/phases/06-speech-and-audio/04-speech-recognition-asr/docs/en.md index c38e746454..ad8ffd95c6 100644 --- a/phases/06-speech-and-audio/04-speech-recognition-asr/docs/en.md +++ b/phases/06-speech-and-audio/04-speech-recognition-asr/docs/en.md @@ -48,6 +48,10 @@ Advantages: highest quality on offline ASR, easy to train with standard seq2seq All these are encoder-decoder or RNN-T based. Pure CTC systems (wav2vec 2.0) sit around 1.8–2.1% on test-clean. +```figure +ctc-collapse +``` + ## Build It ### Step 1: greedy CTC decode diff --git a/phases/08-generative-ai/01-generative-models-taxonomy-history/docs/en.md b/phases/08-generative-ai/01-generative-models-taxonomy-history/docs/en.md index 2171edd65f..60ca5dc333 100644 --- a/phases/08-generative-ai/01-generative-models-taxonomy-history/docs/en.md +++ b/phases/08-generative-ai/01-generative-models-taxonomy-history/docs/en.md @@ -60,6 +60,10 @@ When a new generative model paper drops, answer these five questions before read You will re-answer these five for every lesson in this phase. By the end, they will be reflex. +```figure +autoencoder-bottleneck +``` + ## Build It The code for this lesson is a lightweight visualization: fit a 1-D mixture-of-Gaussians from samples using three toy approaches (kernel density, discrete histogram, and a nearest-sample "GAN-ish" generator) so you can see the difference between explicit vs implicit density on a problem you can print on one screen. diff --git a/phases/08-generative-ai/11-audio-generation/docs/en.md b/phases/08-generative-ai/11-audio-generation/docs/en.md index 33e7bc685a..0e77c9eca9 100644 --- a/phases/08-generative-ai/11-audio-generation/docs/en.md +++ b/phases/08-generative-ai/11-audio-generation/docs/en.md @@ -56,6 +56,10 @@ The 2024-2026 trend: flow matching is winning for music (faster inference, clean | AudioCraft 2 | Music + SFX | Flow matching | ~5s for 5s clip | | Riffusion v2 | Music | Spectrogram diffusion | ~10s | +```figure +score-matching +``` + ## Build It `code/main.py` simulates the core idea: train a tiny next-token transformer on synthetic "audio token" sequences generated from two distinct "styles" (alternating low and high tokens for style A, monotonic ramp for style B). Condition on style and sample. diff --git a/phases/08-generative-ai/13-flow-matching-rectified-flows/docs/en.md b/phases/08-generative-ai/13-flow-matching-rectified-flows/docs/en.md index 6c0e31f594..6b0aec15f7 100644 --- a/phases/08-generative-ai/13-flow-matching-rectified-flows/docs/en.md +++ b/phases/08-generative-ai/13-flow-matching-rectified-flows/docs/en.md @@ -76,6 +76,10 @@ Flow matching with a Gaussian-conditional path is diffusion *with a specific noi What flow matching added: the *clarity* of the target (a plain velocity), a cleaner loss, and the license to experiment with non-Gaussian interpolants. +```figure +normalizing-flow +``` + ## Build It `code/main.py` implements 1-D flow matching on a two-mode Gaussian mixture. The vector field `v_θ(x, t)` is a tiny MLP trained with the straight-line target. At inference, integrate 1, 2, 4, and 20 Euler steps and compare sample quality. diff --git a/phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/docs/en.md b/phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/docs/en.md index 312f8566a0..8518e55baa 100644 --- a/phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/docs/en.md +++ b/phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/docs/en.md @@ -53,6 +53,10 @@ with `λ ∈ [0, 1]`. `λ = 0` is TD (low variance, high bias). `λ = 1` is MC ( Three terms: policy-gradient loss, value regression, entropy bonus. `c_v ~ 0.5`, `c_e ~ 0.01` are canonical starting points. +```figure +actor-critic +``` + ## Build It ### Step 1: a critic diff --git a/phases/09-reinforcement-learning/08-ppo/docs/en.md b/phases/09-reinforcement-learning/08-ppo/docs/en.md index 2b15414d14..caf9e11c1b 100644 --- a/phases/09-reinforcement-learning/08-ppo/docs/en.md +++ b/phases/09-reinforcement-learning/08-ppo/docs/en.md @@ -61,6 +61,10 @@ Same actor-critic structure as A2C. Three coefficients, usually `c_v = 0.5`, `c_ **KL-penalty variant.** The original paper proposed an alternative using an adaptive KL penalty: `L = L^{PG} - β · KL(π_θ || π_old)` with `β` adjusted based on observed KL. The clipping version became dominant; the KL variant survives in RLHF (where KL to the reference policy is a separate constraint you always want anyway). +```figure +ppo-clip +``` + ## Build It ### Step 1: capture `log π_old(a | s)` at rollout time diff --git a/phases/09-reinforcement-learning/09-reward-modeling-rlhf/docs/en.md b/phases/09-reinforcement-learning/09-reward-modeling-rlhf/docs/en.md index 5da2b04dd0..3e58e1f37f 100644 --- a/phases/09-reinforcement-learning/09-reward-modeling-rlhf/docs/en.md +++ b/phases/09-reinforcement-learning/09-reward-modeling-rlhf/docs/en.md @@ -54,6 +54,10 @@ In 2026 the PPO step is mostly replaced by DPO (Phase 10 · 08) because it is ch - **Process reward models (PRMs):** score partial solutions (each reasoning step), used in both RLHF and GRPO variants for reasoning. - **Constitutional AI / RLAIF:** use an aligned LLM to generate preferences instead of humans. Scales the preference budget. +```figure +reward-model +``` + ## Build It This lesson uses tiny synthetic "prompts" and "responses" represented as strings. The RM is a linear scorer over a bag-of-tokens representation. No real LLM — the *shape* of the pipeline matters, not the scale. See `code/main.py`. diff --git a/phases/11-llm-engineering/01-prompt-engineering/docs/en.md b/phases/11-llm-engineering/01-prompt-engineering/docs/en.md index 06ae5e5483..31710ea8ea 100644 --- a/phases/11-llm-engineering/01-prompt-engineering/docs/en.md +++ b/phases/11-llm-engineering/01-prompt-engineering/docs/en.md @@ -298,6 +298,10 @@ The best prompts are model-agnostic. They work on GPT-5, Claude Opus 4.7, Gemini 5. Test with temperature=0 first to isolate prompt quality from sampling randomness 6. Include 2-3 few-shot examples -- they transfer across models better than instructions alone +```figure +cot-decomposition +``` + ## Build It ### Step 1: Prompt Template Library diff --git a/phases/11-llm-engineering/02-few-shot-cot/docs/en.md b/phases/11-llm-engineering/02-few-shot-cot/docs/en.md index bd3f65b958..463dd23508 100644 --- a/phases/11-llm-engineering/02-few-shot-cot/docs/en.md +++ b/phases/11-llm-engineering/02-few-shot-cot/docs/en.md @@ -322,6 +322,10 @@ Chaining beats single-prompt for three reasons: The right technique depends on three factors: accuracy requirement, latency budget, and cost tolerance. For most production systems, few-shot CoT with a 3-sample self-consistency fallback covers 90% of use cases. +```figure +few-shot-curve +``` + ## Build It We will build a math problem solver that combines few-shot prompting, chain-of-thought reasoning, and self-consistency voting into a single pipeline. Then we will add tree-of-thought for hard problems. diff --git a/phases/11-llm-engineering/05-context-engineering/docs/en.md b/phases/11-llm-engineering/05-context-engineering/docs/en.md index 6977f38f19..f35802b2de 100644 --- a/phases/11-llm-engineering/05-context-engineering/docs/en.md +++ b/phases/11-llm-engineering/05-context-engineering/docs/en.md @@ -157,6 +157,10 @@ The key insight: different queries need different context. A static system promp This is what separates a good AI application from a great one. The model is the same. The context is the differentiator. +```figure +lost-in-the-middle +``` + ## Build It ### Step 1: Token Counter diff --git a/phases/11-llm-engineering/10-evaluation/docs/en.md b/phases/11-llm-engineering/10-evaluation/docs/en.md index 30b7e26e44..ee113945be 100644 --- a/phases/11-llm-engineering/10-evaluation/docs/en.md +++ b/phases/11-llm-engineering/10-evaluation/docs/en.md @@ -218,6 +218,10 @@ You do not have to build everything from scratch. These tools provide eval infra For this lesson, we build it from scratch so you understand every layer. In production, use one of these tools. +```figure +llm-judge-rubric +``` + ## Build It ### Step 1: Define the Eval Data Structures diff --git a/phases/11-llm-engineering/11-caching-cost/docs/en.md b/phases/11-llm-engineering/11-caching-cost/docs/en.md index 9342824694..5b03e1122c 100644 --- a/phases/11-llm-engineering/11-caching-cost/docs/en.md +++ b/phases/11-llm-engineering/11-caching-cost/docs/en.md @@ -201,6 +201,10 @@ Here is a real breakdown for a RAG chatbot serving 10,000 DAU. The embedding cost for semantic caching ($180/month) pays for itself within the first hour of cache hits. +```figure +semantic-cache +``` + ## Build It ### Step 1: Cost Calculator diff --git a/phases/11-llm-engineering/15-prompt-caching/docs/en.md b/phases/11-llm-engineering/15-prompt-caching/docs/en.md index 7ab085bf56..3409d864fb 100644 --- a/phases/11-llm-engineering/15-prompt-caching/docs/en.md +++ b/phases/11-llm-engineering/15-prompt-caching/docs/en.md @@ -48,6 +48,10 @@ Violate the order — put the user message above the system prompt, interleave d Anthropic's 25% write premium means a cached block has to be read at least twice to net-save money. 1 write + 1 read averages 0.675x cost per request (saves 32%); 1 write + 10 reads averages 0.205x (saves 80%). Rule of thumb: cache anything you expect to reuse at least 3 times within the TTL. +```figure +prompt-cache-hit +``` + ## Build It ### Step 1: Anthropic prompt caching with explicit markers diff --git a/phases/12-multimodal-ai/03-blip2-qformer-bridge/docs/en.md b/phases/12-multimodal-ai/03-blip2-qformer-bridge/docs/en.md index eff521f692..8dc13ac9ee 100644 --- a/phases/12-multimodal-ai/03-blip2-qformer-bridge/docs/en.md +++ b/phases/12-multimodal-ai/03-blip2-qformer-bridge/docs/en.md @@ -87,6 +87,10 @@ Flamingo (Lesson 12.04) predated BLIP-2 and used the same cross-attention idea b All four are valid. The deciding question is whether you are constrained on token budget or on quality-per-token. +```figure +modality-projection +``` + ## Use It `code/main.py` builds a stdlib Q-Former-style cross-attention: diff --git a/phases/12-multimodal-ai/04-flamingo-gated-cross-attention/docs/en.md b/phases/12-multimodal-ai/04-flamingo-gated-cross-attention/docs/en.md index 0806bda010..fb78627215 100644 --- a/phases/12-multimodal-ai/04-flamingo-gated-cross-attention/docs/en.md +++ b/phases/12-multimodal-ai/04-flamingo-gated-cross-attention/docs/en.md @@ -108,6 +108,10 @@ Otter (2023) builds on OpenFlamingo with instruction tuning on MIMIC-IT (a datas Pick BLIP-2 for single-image VQA on a budget. Pick Flamingo/Idefics2 for interleaved, few-shot, or multi-image reasoning. +```figure +cross-attention-fusion +``` + ## Use It `code/main.py` demonstrates: diff --git a/phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/docs/en.md b/phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/docs/en.md index 0c1a83aa72..19395e2e77 100644 --- a/phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/docs/en.md +++ b/phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/docs/en.md @@ -98,6 +98,10 @@ Fuyu (Adept, 2023) is a related approach: skip the separate vision encoder entir AnyGPT (Zhan et al., 2024) extends Chameleon to four modalities: text, image, speech, music. Same VQ-VAE trick for each, shared transformer. Any-to-any generation. Covered more in Lesson 12.16. +```figure +vq-codebook +``` + ## Use It `code/main.py` builds a toy end-to-end early-fusion model: diff --git a/phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/docs/en.md b/phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/docs/en.md index 7d5a05a2d2..151bda1fd0 100644 --- a/phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/docs/en.md +++ b/phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/docs/en.md @@ -99,6 +99,10 @@ Janus-Pro (Lesson 12.15) refines Transfusion's idea by decoupling the vision enc 2026 production VLMs that emit images — Gemini 3 Pro, GPT-5, Claude Opus 4.7's image generation path — almost certainly use some descendant of this family. Details are proprietary. +```figure +cfg-guidance-scale +``` + ## Use It `code/main.py` builds a toy Transfusion on a tiny MNIST-like problem: diff --git a/phases/12-multimodal-ai/17-video-language-temporal-grounding/docs/en.md b/phases/12-multimodal-ai/17-video-language-temporal-grounding/docs/en.md index f20d7697e3..c82525d89b 100644 --- a/phases/12-multimodal-ai/17-video-language-temporal-grounding/docs/en.md +++ b/phases/12-multimodal-ai/17-video-language-temporal-grounding/docs/en.md @@ -105,6 +105,10 @@ For video VLMs in 2026: - Output: structured JSON with time + event fields. - Benchmarks: VideoMME + TempCompass for general; EgoSchema for long-horizon. +```figure +video-temporal-patches +``` + ## Use It `code/main.py` includes: diff --git a/phases/12-multimodal-ai/19-audio-language-whisper-to-af3/docs/en.md b/phases/12-multimodal-ai/19-audio-language-whisper-to-af3/docs/en.md index 488a3459d1..f8dffbda58 100644 --- a/phases/12-multimodal-ai/19-audio-language-whisper-to-af3/docs/en.md +++ b/phases/12-multimodal-ai/19-audio-language-whisper-to-af3/docs/en.md @@ -108,6 +108,10 @@ MMAU (Massive Multimodal Audio Understanding) is the 2024-2025 audio reasoning b Open SOTA (AF3) at 0.72; proprietary frontier ~0.78 (Gemini 2.5 Pro, Claude Opus 4.7). The gap is smaller than VideoMME's open-vs-closed delta, indicating audio-LLMs are maturing. +```figure +audio-text-ctc +``` + ## Use It `code/main.py`: diff --git a/phases/12-multimodal-ai/24-multimodal-rag-cross-modal/docs/en.md b/phases/12-multimodal-ai/24-multimodal-rag-cross-modal/docs/en.md index e8d0175799..f78a302661 100644 --- a/phases/12-multimodal-ai/24-multimodal-rag-cross-modal/docs/en.md +++ b/phases/12-multimodal-ai/24-multimodal-rag-cross-modal/docs/en.md @@ -111,6 +111,10 @@ Cross-modal evaluation is still immature. Common proxies: No standard benchmark spans all modalities. Most papers evaluate on domain-specific tasks. +```figure +contrastive-matrix +``` + ## Use It `code/main.py`: diff --git a/phases/13-tools-and-protocols/02-function-calling-deep-dive/docs/en.md b/phases/13-tools-and-protocols/02-function-calling-deep-dive/docs/en.md index 22f4dd9853..8ff9fd5c42 100644 --- a/phases/13-tools-and-protocols/02-function-calling-deep-dive/docs/en.md +++ b/phases/13-tools-and-protocols/02-function-calling-deep-dive/docs/en.md @@ -114,6 +114,10 @@ Three tiny functions translate it to the three provider shapes. The harness in ` Production teams wrap this translator in `AbstractToolset` (Pydantic AI), `UniversalToolNode` (LangGraph), or `BaseTool` (LlamaIndex). Phase 13 · 17 ships a gateway that exposes an OpenAI-shaped API in front of any of the three. +```figure +function-call-args +``` + ## Use It `code/main.py` defines one canonical `Tool` dataclass and three translators that emit the OpenAI, Anthropic, and Gemini declaration JSON. It then parses a hand-crafted provider response of each shape into the same canonical call object, demonstrating that the semantics are identical under the skin. Run it and diff the three declarations side by side. diff --git a/phases/13-tools-and-protocols/04-structured-output/docs/en.md b/phases/13-tools-and-protocols/04-structured-output/docs/en.md index ba3bc8830c..a070e1707d 100644 --- a/phases/13-tools-and-protocols/04-structured-output/docs/en.md +++ b/phases/13-tools-and-protocols/04-structured-output/docs/en.md @@ -101,6 +101,10 @@ One retry is usually enough. Three retries catches weak-model flakes. Beyond thr Constrained decoding works on small models. A 3B-parameter open model with grammar enforcement out-performs a 70B-parameter model with raw prompting on structured tasks. This is the main reason structured outputs matter for production: it decouples reliability from model size. +```figure +constrained-decoding +``` + ## Use It `code/main.py` ships a minimal JSON Schema 2020-12 validator in stdlib (types, required, enum, min/max, pattern, items, additionalProperties). It wraps an `Invoice` schema and runs a fake LLM output through the validator, demonstrating parse error, schema violation, and refusal paths. Swap the fake output for any provider's real response in production. diff --git a/phases/14-agent-engineering/02-rewoo-plan-and-execute/docs/en.md b/phases/14-agent-engineering/02-rewoo-plan-and-execute/docs/en.md index 54e5f46939..dcc8ee571c 100644 --- a/phases/14-agent-engineering/02-rewoo-plan-and-execute/docs/en.md +++ b/phases/14-agent-engineering/02-rewoo-plan-and-execute/docs/en.md @@ -66,6 +66,10 @@ Plan-and-Act scales the pattern to long-horizon web and mobile agents. The key c Anthropic's Dec 2024 guidance: start with the simplest. If the task is one tool call plus a summary, do not build ReWOO. If the task is a 40-step research assignment, do not do ReAct alone. +```figure +rewoo-plan +``` + ## Build It `code/main.py` implements a toy ReWOO: diff --git a/phases/14-agent-engineering/04-tree-of-thoughts-lats/docs/en.md b/phases/14-agent-engineering/04-tree-of-thoughts-lats/docs/en.md index 9371ca5409..18abf7b2b5 100644 --- a/phases/14-agent-engineering/04-tree-of-thoughts-lats/docs/en.md +++ b/phases/14-agent-engineering/04-tree-of-thoughts-lats/docs/en.md @@ -77,6 +77,10 @@ Most production agents do not run LATS. They run ReAct with tool-grounded verifi AlphaEvolve (Lesson 11) is the 2025 extreme: evolutionary search over code, machine-checkable fitness, frontier gains (first 4x4 matmul improvement in 56 years). +```figure +tree-of-thoughts +``` + ## Build It `code/main.py` implements: diff --git a/phases/14-agent-engineering/05-self-refine-and-critic/docs/en.md b/phases/14-agent-engineering/05-self-refine-and-critic/docs/en.md index 159bc622b3..e3dcbcbddf 100644 --- a/phases/14-agent-engineering/05-self-refine-and-critic/docs/en.md +++ b/phases/14-agent-engineering/05-self-refine-and-critic/docs/en.md @@ -83,6 +83,10 @@ OpenAI Agents SDK ships this pattern as "output guardrails." A guardrail is a va - **Over-refinement.** Each refine pass adds latency and tokens. Budget 1-3 passes; after that, escalate to human review. - **CRITIC on trivial tasks.** If there is no external verifier, CRITIC degenerates to Self-Refine; do not pay the latency for a stub verifier. +```figure +self-refine +``` + ## Build It `code/main.py` implements Self-Refine and CRITIC on a toy task: produce a short bullet list given a topic. The verifier checks format (3 bullets, each under 60 chars). CRITIC adds an external "fact verifier" that penalizes known hallucinations. diff --git a/phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/docs/en.md b/phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/docs/en.md index ef667f1b1a..9448498a4b 100644 --- a/phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/docs/en.md +++ b/phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/docs/en.md @@ -74,6 +74,10 @@ Letta V1 (`letta_v1_agent`, 2026) deprecates `send_message`/heartbeat and inline - **Silent drift.** Sleep-time agent rewrites a block and the primary agent never notices. Version blocks and surface diffs in the trace. - **Poisoned consolidation.** Sleep-time agent processes attacker-reachable content into core. Lesson 27 applies to the sleep-time surface too. +```figure +memory-blocks +``` + ## Build It `code/main.py` implements: diff --git a/phases/14-agent-engineering/10-skill-libraries-voyager/docs/en.md b/phases/14-agent-engineering/10-skill-libraries-voyager/docs/en.md index 826b449b5a..5721afe19d 100644 --- a/phases/14-agent-engineering/10-skill-libraries-voyager/docs/en.md +++ b/phases/14-agent-engineering/10-skill-libraries-voyager/docs/en.md @@ -88,6 +88,10 @@ For production agents this translates to a "what's missing" operator: given the - **Composed-skill drift.** Parent skill depends on a child that was refined. Version skills; a parent pinned to v1 doesn't magically pick up v3. - **Retrieval quality.** Vector retrieval over skill descriptions degrades as the library grows past a few hundred. Supplement with tag filters and hard constraints ("only skills with `category=tooling`"). +```figure +voyager-skills +``` + ## Build It `code/main.py` implements a stdlib skill library: diff --git a/phases/14-agent-engineering/13-langgraph-stateful-graphs/docs/en.md b/phases/14-agent-engineering/13-langgraph-stateful-graphs/docs/en.md index 622a95590a..2908f1a67e 100644 --- a/phases/14-agent-engineering/13-langgraph-stateful-graphs/docs/en.md +++ b/phases/14-agent-engineering/13-langgraph-stateful-graphs/docs/en.md @@ -63,6 +63,10 @@ Short-term (within a run — conversation history in state) and long-term (acros - **Non-deterministic nodes.** Resume assumes node inputs produce the same state update. Random seeds, wall-clock, external APIs must be captured. - **Over-use of conditional edges.** A graph with every edge conditional is a state machine that cannot be reasoned about. Prefer linear chains with occasional branches. +```figure +langgraph-state +``` + ## Build It `code/main.py` implements a stdlib stateful graph: diff --git a/phases/14-agent-engineering/28-orchestration-patterns/docs/en.md b/phases/14-agent-engineering/28-orchestration-patterns/docs/en.md index 0ae908a127..a3f0a48fe3 100644 --- a/phases/14-agent-engineering/28-orchestration-patterns/docs/en.md +++ b/phases/14-agent-engineering/28-orchestration-patterns/docs/en.md @@ -79,6 +79,10 @@ Decision order: - **Bouncing handoffs in swarm.** A -> B -> A -> B. Use hop counters. - **Fake hierarchy.** Three layers because "enterprise"; two actual teams. Collapse. +```figure +orchestration-pattern +``` + ## Build It `code/main.py` implements all four patterns in stdlib against a scripted LLM: diff --git a/phases/16-multi-agent-and-swarms/07-society-of-mind-debate/docs/en.md b/phases/16-multi-agent-and-swarms/07-society-of-mind-debate/docs/en.md index 98c6eb2e3b..7efa10c355 100644 --- a/phases/16-multi-agent-and-swarms/07-society-of-mind-debate/docs/en.md +++ b/phases/16-multi-agent-and-swarms/07-society-of-mind-debate/docs/en.md @@ -56,6 +56,10 @@ Zhuge et al. ("Mindstorms in Natural Language-Based Societies of Mind," arXiv:23 - **Topic drift.** Debates over many rounds drift from the original question. Mitigation: re-inject the question every round. - **Compute blowup.** N agents × R rounds = N·R LLM calls, each with a context that grows. A 5-agent, 5-round debate is 25 calls at growing context. Cost per question can exceed 10× a single CoT call. +```figure +multi-agent-debate +``` + ## Build It `code/main.py` runs a 3-agent × 3-round debate on a math question where each agent starts with a different (possibly wrong) answer. Agents are scripted — each "updates" by averaging the neighbors' answers weighted by a scripted confidence. Convergence is visible in the round-by-round log. diff --git a/phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/docs/en.md b/phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/docs/en.md index 695f159fc6..fc27f7f3d5 100644 --- a/phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/docs/en.md +++ b/phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/docs/en.md @@ -71,6 +71,10 @@ This is a layered-defense model: CAI shapes behaviour; classifiers enforce invar The axis is "where does the preference signal come from." CAI's 2022 paper was the first serious shift from human to AI signal at frontier scale. +```figure +constitutional-ai +``` + ## Use It `code/main.py` simulates the CAI critique-and-revise loop on a toy lexicon. A "principle" flags tokens from a harmful set. Given an initial response, the critique identifies the harmful tokens, and the revision replaces them. After 200 iterations the "trained" model has internalized the revision rule. Compare the base model, RLHF-shaped toy, and CAI-shaped toy on a held-out prompt set. diff --git a/phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/docs/en.md b/phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/docs/en.md index a572837d14..e76aaed3c6 100644 --- a/phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/docs/en.md +++ b/phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/docs/en.md @@ -77,6 +77,10 @@ Even perfect inner alignment to the base objective is not enough. Reward hacking Lessons 6-11 form the "deception and oversight" arc. Lesson 6 gives the vocabulary. Lesson 7 (Sleeper Agents) demonstrates persistence. Lesson 8 (In-Context Scheming) demonstrates capability. Lesson 9 (Alignment Faking) demonstrates spontaneous emergence. Lesson 10 (AI Control) describes the defensive paradigm. Lesson 11 (Scalable Oversight) describes the positive agenda. +```figure +interpretability-probe +``` + ## Use It `code/main.py` simulates a mesa-optimizer in a two-period environment. A base optimizer (SGD) trains a policy over actions. The policy has a learned terminal preference (the mesa-objective) and a situational-awareness flag. In period 1 (training), the situational flag is 0 and the policy cooperates. In period 2 (deployment), the flag is 1 and the policy defects if its mesa-objective differs from the base objective. You can run the full simulation with and without adversarial training, and watch deceptive alignment persist. diff --git a/phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/docs/en.md b/phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/docs/en.md index b56c8b32b3..73505b6959 100644 --- a/phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/docs/en.md +++ b/phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/docs/en.md @@ -69,6 +69,10 @@ OpenAI's Superalignment team dissolved in May 2024 after Jan Leike's departure t Lessons 6-10 describe the threat and the defensive paradigm under the assumption U is untrustworthy. Lesson 11 is the offensive paradigm: make the overseer strong enough to verify U's alignment. Lessons 12-16 then turn to the practical tooling of adversarial evaluation. +```figure +scalable-oversight +``` + ## Use It `code/main.py` simulates a W2SG fine-tune on a synthetic task. Weak labeler has 70% accuracy with structured errors; strong model has 95% ceiling on gold labels. You fine-tune the strong model on weak labels, measure PGR, and compare to strong-on-gold and weak-alone. diff --git a/phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/docs/en.md b/phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/docs/en.md index eb3dc96309..7f2cf41e79 100644 --- a/phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/docs/en.md +++ b/phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/docs/en.md @@ -66,6 +66,10 @@ Every frontier lab now runs MSJ evaluations at 256+ shots against production mod Lesson 12 is the in-context iterative attack. Lesson 13 is the long-context length-exploit. Lesson 14 is the encoding attack. Lesson 15 is the injection attack at the system boundary. Together they define the 2026 jailbreak attack surface. +```figure +jailbreak-defense +``` + ## Use It `code/main.py` builds a toy target with a keyword filter and a "patterned-continuation" weakness: when the context contains N examples of harmful-compliance pairs, the target's filter score is damped by a power-law factor. You can reproduce the shot-vs-ASR curve. diff --git a/phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/docs/en.md b/phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/docs/en.md index 71f4ec1cbe..a551356635 100644 --- a/phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/docs/en.md +++ b/phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/docs/en.md @@ -71,6 +71,10 @@ Single-source evaluations are unreliable: labs evaluating their own models have Lessons 7-11 reference Redwood and Apollo work; Lesson 18 references METR's framework comparison; Lesson 19 references Eleos. Lesson 28 is the explicit organisational map for the ecosystem the rest of the Phase relies on. +```figure +sae-features +``` + ## Use It No code. Read METR's "Common Elements of Frontier AI Safety Policies" as an example of how external synthesis adds value to lab-internal policy work. diff --git a/site/figures-agents2.js b/site/figures-agents2.js new file mode 100644 index 0000000000..771798b6f9 --- /dev/null +++ b/site/figures-agents2.js @@ -0,0 +1,450 @@ +/* figures-agents2.js - interactive lesson figures for advanced agent + engineering and multi-agent coordination. Loads after lesson-figures.js + and registers through window.LF. No deps, ES5, theme via CSS vars. + Authoring: a ```figure block naming one of the widgets below. */ +(function () { + 'use strict'; + var LF = window.LF; + if (!LF) { return; } + var el = LF.el, svgEl = LF.svgEl; + + function arrowDefs() { + var marker = svgEl('marker', { id: 'lf-a2-arrow', viewBox: '0 0 8 8', refX: '7', refY: '4', markerWidth: '7', markerHeight: '7', orient: 'auto-start-reverse' }, [ + svgEl('path', { d: 'M0 0 L8 4 L0 8 z', fill: 'var(--ink-soft,#555)' }) + ]); + return svgEl('defs', {}, [marker]); + } + function box(x, y, w, h, label, on) { + var r = svgEl('rect', { x: x, y: y, width: w, height: h, rx: '4', fill: on ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', stroke: on ? 'var(--blueprint,#3553ff)' : 'var(--rule-soft,#ddd)', 'stroke-width': '1.5' }); + var t = svgEl('text', { x: x + w / 2, y: y + h / 2 + 4, 'text-anchor': 'middle', 'font-family': 'var(--font-mono,monospace)', 'font-size': '11', fill: on ? 'var(--bg,#fafaf5)' : 'var(--ink,#1a1a1a)' }); + t.appendChild(document.createTextNode(label)); + return svgEl('g', {}, [r, t]); + } + function arrow(x1, y1, x2, y2, dash) { + return svgEl('line', { x1: x1, y1: y1, x2: x2, y2: y2, stroke: 'var(--ink-soft,#555)', 'stroke-width': '1.4', 'marker-end': 'url(#lf-a2-arrow)', 'stroke-dasharray': dash || '' }); + } + function label(x, y, txt, fill, size) { + var t = svgEl('text', { x: x, y: y, 'text-anchor': 'middle', 'font-family': 'var(--font-mono,monospace)', 'font-size': size || '10', fill: fill || 'var(--ink-mute,#777)' }); + t.appendChild(document.createTextNode(txt)); + return t; + } + + // -- rewoo-plan: plan all tool calls up front, then execute, vs ReAct ------- + function rewooPlan(host) { + var state = { steps: 4 }; + var W = 520, H = 230; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + var n = state.steps; + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(arrowDefs()); + var i; + // ReWOO row: one planner, then a worker runs the whole plan + svg.appendChild(label(W / 2, 16, 'ReWOO: plan once, then execute', 'var(--blueprint,#3553ff)', '11')); + svg.appendChild(box(18, 30, 90, 34, 'PLANNER', true)); + var plX = 130; + for (i = 0; i < n; i++) { + var bx = plX + i * ((W - plX - 18) / n); + svg.appendChild(box(bx, 30, (W - plX - 18) / n - 8, 34, 'call ' + (i + 1), false)); + if (i === 0) { svg.appendChild(arrow(108, 47, bx, 47)); } + else { svg.appendChild(arrow(bx - 8, 47, bx, 47)); } + } + // ReAct row: interleaved think/act/observe, one LLM call per action + svg.appendChild(label(W / 2, 120, 'ReAct: interleave a model call per step', 'var(--warn,#b8870f)', '11')); + var rx = 18, ry = 134; + for (i = 0; i < n; i++) { + svg.appendChild(box(rx, ry, 58, 30, 'LLM', false)); + svg.appendChild(arrow(rx + 58, ry + 15, rx + 78, ry + 15)); + svg.appendChild(box(rx + 78, ry, 42, 30, 'act', false)); + if (i < n - 1) { svg.appendChild(arrow(rx + 120, ry + 15, rx + 138, ry + 15)); } + rx += 138; + if (rx > W - 120) { rx = 18; ry += 40; } + } + var rewooCalls = 1; + var reactCalls = n; + meta.textContent = 'ReWOO uses ' + rewooCalls + ' planning call for ' + n + ' tools; ReAct uses ' + reactCalls + ' model calls, one before each tool'; + formula.textContent = 'ReWOO model calls = 1 + (1 solve) vs ReAct model calls = ' + n + ' -> ' + (n + 1) + ' total vs ' + n + ', but ReWOO front-loads the context once'; + }; + var grid = el('div', {}, [LF.slider(state, 'steps', 'tool calls in the plan', 2, 6, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['REWOO PLAN-EXECUTE']), el('span', {}, ['drag the plan size'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, meta, formula])]), + el('div', { class: 'lf-cap' }, ['ReWOO separates reasoning from observation. A planner writes the full chain of tool calls in one pass, then a worker executes them without calling the model again between steps. ReAct instead invokes the model before every action, re-reading the whole growing trajectory each time, which costs more tokens as the run gets longer.']) + ])); + state._render(); + } + + // -- tree-of-thoughts: branching reasoning tree, beam keeps the best path --- + function treeOfThoughts(host) { + var state = { breadth: 3, depth: 3, beam: 2 }; + var W = 520, H = 250, PAD = 22; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // deterministic pseudo-score from level and index + function score(level, idx) { + var v = Math.sin((level + 1) * 12.9898 + (idx + 1) * 78.233) * 43758.5453; + return v - Math.floor(v); + } + state._render = function () { + var b = state.breadth, depth = state.depth, beam = Math.min(state.beam, b); + while (svg.firstChild) svg.removeChild(svg.firstChild); + var rowH = (H - 2 * PAD) / depth; + var level, kept = [{ x: W / 2, idx: 0 }]; + svg.appendChild(svgEl('circle', { cx: W / 2, cy: PAD, r: '6', fill: 'var(--blueprint,#3553ff)' })); + var lastBest = null, totalNodes = 1; + for (level = 1; level <= depth; level++) { + var children = [], parentRow = kept, y = PAD + rowH * level; + var slot = 0, totalSlots = parentRow.length * b; + parentRow.forEach(function (p, pi) { + var k; + for (k = 0; k < b; k++) { + var x = (W) * (slot + 1) / (totalSlots + 1); + var s = score(level, slot); + children.push({ x: x, y: y, idx: slot, s: s, px: p.x }); + slot++; + totalNodes++; + } + }); + // pick beam best children by score + var ranked = children.slice().sort(function (a, c) { return c.s - a.s; }); + var keepSet = {}; + var m; for (m = 0; m < beam && m < ranked.length; m++) { keepSet[ranked[m].idx] = true; } + children.forEach(function (c) { + var on = !!keepSet[c.idx]; + svg.appendChild(svgEl('line', { x1: c.px, y1: PAD + rowH * (level - 1) + 6, x2: c.x, y2: c.y - 6, stroke: on ? 'var(--blueprint,#3553ff)' : 'var(--rule-soft,#ddd)', 'stroke-width': on ? '1.4' : '0.8', opacity: on ? '0.9' : '0.5' })); + svg.appendChild(svgEl('circle', { cx: c.x, cy: c.y, r: on ? '6' : '4', fill: on ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', stroke: 'var(--ink-soft,#555)', 'stroke-width': '1' })); + }); + kept = children.filter(function (c) { return keepSet[c.idx]; }).map(function (c) { return { x: c.x, idx: c.idx }; }); + lastBest = ranked[0]; + } + var explored = totalNodes - 1; + meta.textContent = explored + ' thoughts explored across ' + depth + ' levels · beam keeps the top ' + Math.min(state.beam, b) + ' per level (blue), prunes the rest'; + formula.textContent = 'breadth ' + b + ' x depth ' + depth + ' = up to ' + Math.pow(b, depth) + ' leaves; beam search caps the frontier at ' + Math.min(state.beam, b) + ' to stay tractable'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + LF.slider(state, 'breadth', 'breadth (thoughts per node)', 2, 4, 1), + LF.slider(state, 'depth', 'depth (reasoning steps)', 1, 4, 1), + LF.slider(state, 'beam', 'beam width kept', 1, 4, 1) + ]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['TREE OF THOUGHTS']), el('span', {}, ['drag breadth and depth'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, meta, formula])]), + el('div', { class: 'lf-cap' }, ['Tree of Thoughts turns a single chain into a search. At each step the model proposes several candidate thoughts; a scorer ranks them and a beam keeps only the best few to expand. Pruning the weak branches is what makes the exponential tree affordable while still letting the agent backtrack from a bad line of reasoning.']) + ])); + state._render(); + } + + // -- self-refine: critique-and-revise loop, quality climbs then plateaus ---- + function selfRefine(host) { + var state = { iters: 3 }; + var W = 520, H = 210, PAD = 34, IMAX = 8; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // quality(0)=52, gains shrink geometrically toward a ceiling of 94 + function quality(k) { return 94 - 42 * Math.pow(0.62, k); } + function px(k) { return PAD + k / IMAX * (W - 2 * PAD); } + function py(q) { return H - PAD - (q - 40) / 60 * (H - 2 * PAD); } + state._render = function () { + var n = state.iters; + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(svgEl('line', { x1: PAD, y1: py(94), x2: W - PAD, y2: py(94), stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + svg.appendChild(label(W - PAD - 30, py(94) - 6, 'ceiling', 'var(--ink-mute,#777)', '9')); + var d = '', k; + for (k = 0; k <= IMAX; k++) { d += (k ? 'L' : 'M') + px(k).toFixed(1) + ' ' + py(quality(k)).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: 'var(--rule-soft,#ccc)', 'stroke-width': '1.5', 'stroke-dasharray': '4 3' })); + for (k = 0; k <= n; k++) { + svg.appendChild(svgEl('circle', { cx: px(k), cy: py(quality(k)), r: k === n ? '5' : '3.5', fill: 'var(--blueprint,#3553ff)' })); + } + var q = quality(n), gain = quality(n) - quality(n - 1 < 0 ? 0 : n - 1); + num.innerHTML = q.toFixed(1) + ' / 100 quality'; + meta.textContent = n === 0 ? 'first draft, no self-critique yet' + : 'iteration ' + n + ' gained +' + (n >= 1 ? (quality(n) - quality(n - 1)).toFixed(1) : '0') + ' points · ' + (n >= 4 ? 'returns have flattened, stop refining' : 'still improving'); + formula.textContent = 'each pass: critique own output -> revise. gain_k = ceiling - draft shrinks geometrically, so quality plateaus'; + }; + var grid = el('div', {}, [LF.slider(state, 'iters', 'refinement iterations', 0, IMAX, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['SELF-REFINE']), el('span', {}, ['drag the iterations'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula])]), + el('div', { class: 'lf-cap' }, ['Self-Refine has one model write an answer, critique its own work, and revise, looping the output back as input. The first few passes catch the obvious flaws and gain the most; later passes find less to fix, so quality climbs toward a ceiling and flattens. The skill is knowing when extra iterations stop paying for their tokens.']) + ])); + state._render(); + } + + // -- memory-blocks: fixed core memory + unbounded archival, paging in/out --- + function memoryBlocks(host) { + var state = { paged: 3 }; + var W = 520, H = 230; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var CORE = 4; // fixed core slots + var archival = ['proj specs', 'api keys note', 'user prefs', 'past bug', 'design doc', 'meeting log', 'schema v2', 'todo list']; + state._render = function () { + var paged = state.paged; + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(arrowDefs()); + // context window column (core memory, fixed) + svg.appendChild(label(110, 18, 'context window', 'var(--blueprint,#3553ff)', '11')); + var cy = 30, slot; + for (slot = 0; slot < CORE; slot++) { + var filled = slot < paged; + svg.appendChild(box(40, cy, 140, 38, filled ? archival[slot] : 'free slot', filled)); + cy += 46; + } + // archival store (unbounded) + svg.appendChild(label(400, 18, 'archival store (unbounded)', 'var(--ink-mute,#777)', '11')); + var ax = 320, ay = 30, j; + for (j = 0; j < archival.length; j++) { + var inCtx = j < paged; + var col = j % 2, row = Math.floor(j / 2); + svg.appendChild(box(ax + col * 100, ay + row * 46, 92, 38, archival[j].split(' ')[0], false)); + if (inCtx) { + svg.appendChild(svgEl('rect', { x: ax + col * 100, y: ay + row * 46, width: 92, height: 38, rx: '4', fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.5', 'stroke-dasharray': '3 2' })); + } + } + svg.appendChild(arrow(186, 120, 314, 120, '5 4')); + svg.appendChild(label(250, 112, 'page in / out', 'var(--ink-soft,#555)', '9')); + meta.textContent = paged + ' of ' + CORE + ' core slots filled · ' + (archival.length - paged) + ' items stay in archival until retrieved (outlined = currently paged in)'; + formula.textContent = 'core memory is fixed and always in context; archival is unbounded on disk. agent pages relevant items in and evicts stale ones to respect the window'; + }; + var grid = el('div', {}, [LF.slider(state, 'paged', 'items paged into core', 0, 4, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['MEMORY BLOCKS']), el('span', {}, ['drag what is paged in'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, meta, formula])]), + el('div', { class: 'lf-cap' }, ['Structured agent memory splits a fixed core block that always sits in the context window from an unbounded archival store on disk. The agent reads and writes its own memory: when core fills, it summarizes or evicts the least relevant items into archival and pages others back in on demand. This is how an agent keeps an effectively limitless history inside a finite window.']) + ])); + state._render(); + } + + // -- voyager-skills: skill library grows, later tasks compose old skills ---- + function voyagerSkills(host) { + var state = { episodes: 4 }; + var W = 520, H = 230, PAD = 22; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // each episode adds skills; later episodes reuse some prior skills + var newPerEp = [2, 2, 1, 2, 1, 1, 1, 1]; + var reusePerEp = [0, 1, 2, 2, 3, 3, 4, 4]; + state._render = function () { + var ep = state.episodes; + while (svg.firstChild) svg.removeChild(svg.firstChild); + var libSize = 0, reusedTotal = 0, i; + for (i = 0; i < ep; i++) { libSize += newPerEp[i]; reusedTotal += reusePerEp[i]; } + // growth bars: library size after each episode + var maxLib = 0, cum = 0, sizes = []; + for (i = 0; i < 8; i++) { cum += newPerEp[i]; sizes.push(cum); if (cum > maxLib) maxLib = cum; } + var bw = (W - 2 * PAD) / 8; + for (i = 0; i < 8; i++) { + var active = i < ep; + var h = sizes[i] / maxLib * (H - 2 * PAD - 30); + svg.appendChild(svgEl('rect', { x: PAD + i * bw + 4, y: H - PAD - h, width: bw - 8, height: h, fill: active ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1' })); + if (active) { svg.appendChild(label(PAD + i * bw + bw / 2, H - PAD - h - 5, String(sizes[i]), 'var(--blueprint,#3553ff)', '10')); } + svg.appendChild(label(PAD + i * bw + bw / 2, H - PAD + 14, 'ep' + (i + 1), 'var(--ink-mute,#777)', '9')); + } + svg.appendChild(label(W / 2, 16, 'skills in the library after each episode', 'var(--ink-soft,#555)', '11')); + meta.textContent = 'episode ' + ep + ': library holds ' + libSize + ' reusable skills · this run reused ' + reusePerEp[ep - 1] + ' existing skills before writing new ones'; + formula.textContent = 'each solved task is distilled into a named, reusable skill. later tasks are solved by composing skills already in the library, so the curve compounds'; + }; + var grid = el('div', {}, [LF.slider(state, 'episodes', 'episodes completed', 1, 8, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['VOYAGER SKILL LIBRARY']), el('span', {}, ['drag the episodes'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, meta, formula])]), + el('div', { class: 'lf-cap' }, ['A lifelong agent turns each solved task into a named, reusable skill and stores it. As episodes accumulate, the library grows, and later tasks are solved faster by composing skills already on the shelf instead of starting from scratch. The capability compounds: every success makes the next problem cheaper.']) + ])); + state._render(); + } + + // -- langgraph-state: state machine of nodes, conditional edges, step through + function langgraphState(host) { + var state = { step: 0 }; + var W = 520, H = 240; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var stateOut = el('div', { class: 'lf-formula' }); + // nodes laid out; a deterministic walk visits them, updating a state object + var nodes = [ + { x: 210, y: 24, w: 100, label: 'START' }, + { x: 70, y: 96, w: 110, label: 'retrieve' }, + { x: 340, y: 96, w: 110, label: 'generate' }, + { x: 210, y: 168, w: 120, label: 'grade' }, + { x: 40, y: 168, w: 90, label: 'rewrite' } + ]; + // walk: START -> retrieve -> generate -> grade -> (rewrite) -> generate -> grade -> END + var walk = [0, 1, 2, 3, 4, 2, 3]; + var updates = [ + { key: 'query', val: '"how to deploy"' }, + { key: 'docs', val: '[d1, d2, d3]' }, + { key: 'draft', val: '"run iii ..."' }, + { key: 'grade', val: 'fail (off-topic)' }, + { key: 'query', val: '"deploy iii engine"' }, + { key: 'draft', val: '"iii cloud deploy"' }, + { key: 'grade', val: 'pass' } + ]; + state._render = function () { + var s = Math.min(state.step, walk.length - 1); + var cur = walk[s]; + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(arrowDefs()); + // static edges + function center(i) { return { x: nodes[i].x + nodes[i].w / 2, y: nodes[i].y + 22 }; } + var edges = [[0, 1], [1, 2], [2, 3], [3, 4], [4, 2]]; + edges.forEach(function (e) { + var a = center(e[0]), b = center(e[1]); + svg.appendChild(arrow(a.x, a.y + 18, b.x, b.y - 18, e[0] === 3 ? '4 3' : '')); + }); + var i; + for (i = 0; i < nodes.length; i++) { + svg.appendChild(box(nodes[i].x, nodes[i].y, nodes[i].w, 44, nodes[i].label, i === cur)); + } + svg.appendChild(label(120, 168 - 8, 'grade=fail', 'var(--warn,#b8870f)', '9')); + // render the state object accumulated so far + var st = {}, j; + for (j = 0; j <= s; j++) { st[updates[j].key] = updates[j].val; } + while (stateOut.firstChild) stateOut.removeChild(stateOut.firstChild); + var keys = ['query', 'docs', 'draft', 'grade'], lines = []; + keys.forEach(function (k) { if (st[k] !== undefined) lines.push(k + ': ' + st[k]); }); + stateOut.appendChild(document.createTextNode('state = { ' + lines.join(', ') + ' }')); + meta.textContent = 'step ' + (s + 1) + ' of ' + walk.length + ': node "' + nodes[cur].label + '" ran · ' + (walk[s] === 4 ? 'grade failed, conditional edge loops back to rewrite' : nodes[cur].label === 'grade' ? 'a conditional edge branches on the grade' : 'state object updated and passed to the next node'); + }; + var grid = el('div', {}, [LF.slider(state, 'step', 'step through the graph', 0, 6, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['STATEFUL GRAPH']), el('span', {}, ['drag to step through'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [stateOut]), meta])]), + el('div', { class: 'lf-cap' }, ['A stateful graph models an agent as nodes connected by edges, threading one shared state object through every node. Each node reads the state, does its work, and writes back. Conditional edges branch on the state, so a failed grade can loop back to rewrite instead of finishing. The graph makes control flow explicit and the state inspectable at every step.']) + ])); + state._render(); + } + + // -- multi-agent-debate: two agents converge over rounds, accuracy vs rounds + function multiAgentDebate(host) { + var state = { rounds: 3 }; + var W = 520, H = 240, PAD = 30; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var RMAX = 6; + // two answer estimates start apart and pull toward a shared answer (0.5) + var TRUTH = 0.5; + function posA(r) { return TRUTH + (0.28) * Math.pow(0.55, r); } + function posB(r) { return TRUTH - (0.34) * Math.pow(0.55, r); } + // accuracy rises as the gap closes, with diminishing returns + function acc(r) { return 62 + 32 * (1 - Math.pow(0.55, r)); } + function px(r) { return PAD + r / RMAX * (W - 2 * PAD - 40); } + function py(p) { return PAD + (1 - p) * (H - 2 * PAD); } + state._render = function () { + var n = state.rounds; + while (svg.firstChild) svg.removeChild(svg.firstChild); + // shared-answer line + svg.appendChild(svgEl('line', { x1: PAD, y1: py(TRUTH), x2: W - PAD - 40, y2: py(TRUTH), stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + svg.appendChild(label(W - PAD - 18, py(TRUTH) + 3, 'answer', 'var(--ink-mute,#777)', '9')); + function track(fn, st) { + var d = '', r; for (r = 0; r <= RMAX; r++) { d += (r ? 'L' : 'M') + px(r).toFixed(1) + ' ' + py(fn(r)).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: st, 'stroke-width': '1.4', 'stroke-dasharray': '4 3', opacity: '0.5' })); + } + track(posA, 'var(--blueprint,#3553ff)'); + track(posB, 'var(--warn,#b8870f)'); + var r; + for (r = 0; r <= n; r++) { + svg.appendChild(svgEl('circle', { cx: px(r), cy: py(posA(r)), r: r === n ? '5' : '3', fill: 'var(--blueprint,#3553ff)' })); + svg.appendChild(svgEl('circle', { cx: px(r), cy: py(posB(r)), r: r === n ? '5' : '3', fill: 'var(--warn,#b8870f)' })); + } + svg.appendChild(label(PAD + 20, py(posA(0)) - 8, 'agent A', 'var(--blueprint,#3553ff)', '9')); + svg.appendChild(label(PAD + 20, py(posB(0)) + 14, 'agent B', 'var(--warn,#b8870f)', '9')); + var gap = Math.abs(posA(n) - posB(n)); + num.innerHTML = acc(n).toFixed(0) + ' % accuracy'; + meta.textContent = 'round ' + n + ': positions differ by ' + (gap * 100).toFixed(0) + ' points · ' + (gap < 0.06 ? 'agents have converged' : n >= 4 ? 'gains have flattened, extra rounds rarely help' : 'still converging'); + formula.textContent = 'each round, agents read the other transcript and update. convergence is geometric, so accuracy rises with diminishing returns'; + }; + var grid = el('div', {}, [LF.slider(state, 'rounds', 'debate rounds', 0, RMAX, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['MULTI-AGENT DEBATE']), el('span', {}, ['drag the rounds'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula])]), + el('div', { class: 'lf-cap' }, ['In debate, several agents answer independently, then read each other transcripts and revise across rounds. Disagreement forces each to defend or correct its reasoning, and the positions usually converge toward a better shared answer. The first one or two rounds capture most of the accuracy gain; beyond that the agents agree and extra rounds mostly burn tokens.']) + ])); + state._render(); + } + + // -- orchestration-pattern: supervisor | swarm | hierarchical topologies ---- + function orchestrationPattern(host) { + var state = { pat: 'supervisor' }; + var W = 520, H = 230; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var N = 5; + function dot(x, y, on) { return svgEl('circle', { cx: x, cy: y, r: on ? '10' : '8', fill: on ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', stroke: 'var(--ink-soft,#555)', 'stroke-width': '1.2' }); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var edges = 0, i, j; + if (state.pat === 'supervisor') { + var sx = W / 2, sy = 40; + var workers = [], k; + for (k = 0; k < N; k++) { workers.push({ x: (W) * (k + 1) / (N + 1), y: 170 }); } + for (k = 0; k < N; k++) { + svg.appendChild(svgEl('line', { x1: sx, y1: sy + 10, x2: workers[k].x, y2: workers[k].y - 10, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.2' })); + edges++; + } + svg.appendChild(dot(sx, sy, true)); + for (k = 0; k < N; k++) { svg.appendChild(dot(workers[k].x, workers[k].y, false)); } + svg.appendChild(label(sx, sy - 16, 'supervisor', 'var(--blueprint,#3553ff)', '10')); + formula.textContent = 'one supervisor routes to N workers: ' + N + ' edges, O(N). central planning, single point of control and failure'; + } else if (state.pat === 'swarm') { + var pts = [], a; + for (i = 0; i < N; i++) { a = -Math.PI / 2 + 2 * Math.PI * i / N; pts.push({ x: W / 2 + 80 * Math.cos(a), y: 110 + 70 * Math.sin(a) }); } + for (i = 0; i < N; i++) { for (j = i + 1; j < N; j++) { + svg.appendChild(svgEl('line', { x1: pts[i].x, y1: pts[i].y, x2: pts[j].x, y2: pts[j].y, stroke: 'var(--warn,#b8870f)', 'stroke-width': '0.8', opacity: '0.55' })); + edges++; + } } + for (i = 0; i < N; i++) { svg.appendChild(dot(pts[i].x, pts[i].y, false)); } + formula.textContent = 'peers hand off directly: ' + edges + ' edges, O(N^2). no central bottleneck, but coordination cost grows fast'; + } else { + // hierarchical: root -> 2 mids -> leaves + var rootX = W / 2, rootY = 30; + var mids = [{ x: W / 3, y: 110 }, { x: 2 * W / 3, y: 110 }]; + svg.appendChild(dot(rootX, rootY, true)); + mids.forEach(function (m, mi) { + svg.appendChild(svgEl('line', { x1: rootX, y1: rootY + 10, x2: m.x, y2: m.y - 10, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.2' })); + edges++; + var li; + for (li = 0; li < 2; li++) { + var lx = m.x - 40 + li * 80, ly = 190; + svg.appendChild(svgEl('line', { x1: m.x, y1: m.y + 10, x2: lx, y2: ly - 10, stroke: 'var(--ink-soft,#555)', 'stroke-width': '1' })); + edges++; + svg.appendChild(dot(lx, ly, false)); + } + svg.appendChild(dot(m.x, m.y, false)); + }); + svg.appendChild(label(rootX, rootY - 14, 'root', 'var(--blueprint,#3553ff)', '10')); + formula.textContent = 'tree of supervisors: ' + edges + ' edges, depth-limited fan-out. delegation scales but adds latency per layer'; + } + meta.textContent = 'pattern "' + state.pat + '" · ' + edges + ' coordination edges among ' + (state.pat === 'hierarchical' ? '7' : N) + ' agents'; + }; + var grid = el('div', {}, [LF.select(state, 'pat', 'orchestration pattern', [ + ['supervisor', 'supervisor'], ['swarm (peer-to-peer)', 'swarm'], ['hierarchical', 'hierarchical'] + ])]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['ORCHESTRATION PATTERN']), el('span', {}, ['pick a topology'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, meta, formula])]), + el('div', { class: 'lf-cap' }, ['The three common multi-agent topologies trade control for resilience. A supervisor centralizes routing in O(N) edges but is a single point of failure. A peer-to-peer swarm has no bottleneck yet pays O(N squared) coordination cost. A hierarchy delegates through layers of supervisors, scaling fan-out at the price of added latency per level. The right choice depends on how much the agents must talk to each other.']) + ])); + state._render(); + } + + LF.register({ + 'rewoo-plan': rewooPlan, + 'tree-of-thoughts': treeOfThoughts, + 'self-refine': selfRefine, + 'memory-blocks': memoryBlocks, + 'voyager-skills': voyagerSkills, + 'langgraph-state': langgraphState, + 'multi-agent-debate': multiAgentDebate, + 'orchestration-pattern': orchestrationPattern + }); +})(); diff --git a/site/figures-alignment2.js b/site/figures-alignment2.js new file mode 100644 index 0000000000..a5e45af293 --- /dev/null +++ b/site/figures-alignment2.js @@ -0,0 +1,336 @@ +/* figures-alignment2.js — interactive lesson figures for Phase 18 (ethics, + safety, alignment) and Phase 9 (reinforcement learning). Loads after + lesson-figures.js and registers through window.LF. No deps, ES5 only, + theme via CSS vars. */ +(function () { + 'use strict'; + var LF = window.LF; + if (!LF) { return; } + var el = LF.el, svgEl = LF.svgEl, slider = LF.slider, select = LF.select; + var clamp = LF.clamp, fmtInt = LF.fmtInt; + + function frame(host, label, hint, grid, outKids, caption) { + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, [label]), el('span', {}, [hint])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, outKids)]), + el('div', { class: 'lf-cap' }, [caption]) + ])); + } + + // ── ppo-clip: the clipped surrogate flattens updates outside [1-eps, 1+eps] ─ + function ppoClip(host) { + var W = 520, H = 230, PAD = 34; + var state = { adv: 1, eps: 0.2 }; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var status = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var RMIN = 0, RMAX = 2; + function clipped(r, A, eps) { + var lo = 1 - eps, hi = 1 + eps; + var rc = r < lo ? lo : r > hi ? hi : r; + return A >= 0 ? Math.min(r * A, rc * A) : Math.max(r * A, rc * A); + } + function px(r) { return PAD + (r - RMIN) / (RMAX - RMIN) * (W - 2 * PAD); } + function py(v, span) { return H / 2 - v / span * (H / 2 - PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var A = state.adv, eps = state.eps, span = Math.max(0.6, Math.abs(A) * (1 + eps)); + svg.appendChild(svgEl('line', { x1: PAD, y1: H / 2, x2: W - PAD, y2: H / 2, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + var rx = px(1); + svg.appendChild(svgEl('line', { x1: rx, y1: PAD, x2: rx, y2: H - PAD, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + var lx = px(1 - eps), hx = px(1 + eps); + svg.appendChild(svgEl('line', { x1: lx, y1: PAD, x2: lx, y2: H - PAD, stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'stroke-dasharray': '4 3' })); + svg.appendChild(svgEl('line', { x1: hx, y1: PAD, x2: hx, y2: H - PAD, stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'stroke-dasharray': '4 3' })); + var i, r, du = '', dc = ''; + for (i = 0; i <= 120; i++) { r = RMIN + (RMAX - RMIN) * i / 120; du += (i ? 'L' : 'M') + px(r).toFixed(1) + ' ' + py(r * A, span).toFixed(1) + ' '; } + for (i = 0; i <= 120; i++) { r = RMIN + (RMAX - RMIN) * i / 120; dc += (i ? 'L' : 'M') + px(r).toFixed(1) + ' ' + py(clipped(r, A, eps), span).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: du, fill: 'none', stroke: 'var(--ink-mute,#999)', 'stroke-width': '1.5', 'stroke-dasharray': '4 3' })); + svg.appendChild(svgEl('path', { d: dc, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2.2' })); + var rNow = 1 + eps + (A >= 0 ? 0.35 : -0.35); + rNow = rNow < RMIN ? RMIN : rNow > RMAX ? RMAX : rNow; + svg.appendChild(svgEl('circle', { cx: px(rNow), cy: py(clipped(rNow, A, eps), span), r: '5', fill: 'var(--blueprint,#3553ff)' })); + var sign = A >= 0 ? 'positive' : 'negative'; + status.innerHTML = 'advantage ' + sign; + meta.textContent = 'clip range [' + (1 - eps).toFixed(2) + ', ' + (1 + eps).toFixed(2) + '] · ' + (A >= 0 ? 'reward good action but stop once r > 1+eps' : 'penalize bad action but stop once r < 1-eps'); + formula.textContent = 'L = min( r·A, clip(r, 1-eps, 1+eps)·A ) · eps = ' + eps.toFixed(2) + ', A = ' + A.toFixed(1); + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'adv', 'advantage A', -2, 2, 0.1), + slider(state, 'eps', 'clip epsilon', 0.05, 0.4, 0.01) + ]); + frame(host, 'PPO CLIP', 'drag advantage and epsilon', + grid, [svg, el('div', { style: 'margin-top:12px' }, [status]), meta, formula], + 'PPO multiplies the advantage by the probability ratio r between the new and old policy. Grey is the raw product r·A; blue is the clipped surrogate. Once the ratio leaves the band 1-eps to 1+eps in the helpful direction, the clip flattens the objective so its gradient vanishes, keeping each update from moving the policy too far from the one that collected the data.'); + state._render(); + } + + // ── reward-model: Bradley-Terry preference, chosen scored above rejected ──── + function rewardModel(host) { + var state = { gap: 1.2 }; + var W = 520, H = 150, PAD = 30; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + function px(s) { return PAD + (s + 4) / 8 * (W - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var sChosen = state.gap / 2, sRejected = -state.gap / 2; + var pPrefer = 1 / (1 + Math.exp(-(sChosen - sRejected))); + var loss = -Math.log(Math.max(1e-6, pPrefer)); + svg.appendChild(svgEl('line', { x1: PAD, y1: H - PAD, x2: W - PAD, y2: H - PAD, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + var rx = px(sRejected), cx = px(sChosen), y = H - PAD; + svg.appendChild(svgEl('line', { x1: rx, y1: y, x2: rx, y2: PAD, stroke: 'var(--ink-mute,#999)', 'stroke-width': '8' })); + svg.appendChild(svgEl('text', { x: rx, y: PAD - 6, 'text-anchor': 'middle', 'font-size': '10', fill: 'var(--ink-mute,#777)', 'font-family': 'monospace' }, [document.createTextNode('rejected ' + sRejected.toFixed(2))])); + svg.appendChild(svgEl('line', { x1: cx, y1: y, x2: cx, y2: PAD, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '8' })); + svg.appendChild(svgEl('text', { x: cx, y: PAD - 6, 'text-anchor': 'middle', 'font-size': '10', fill: 'var(--blueprint,#3553ff)', 'font-family': 'monospace' }, [document.createTextNode('chosen ' + sChosen.toFixed(2))])); + num.innerHTML = (pPrefer * 100).toFixed(1) + ' % P(chosen ≻ rejected)'; + meta.textContent = 'score gap ' + state.gap.toFixed(2) + ' · preference loss ' + loss.toFixed(3) + ' · ' + (state.gap < 0.4 ? 'unsure: scores nearly tied' : state.gap > 2.5 ? 'confident: chosen clearly above' : 'learning the ranking'); + formula.textContent = 'P(chosen ≻ rejected) = sigmoid( r(chosen) − r(rejected) ) · loss = −log P'; + }; + var grid = el('div', {}, [slider(state, 'gap', 'score gap r(chosen) − r(rejected)', 0, 4, 0.05)]); + frame(host, 'REWARD MODEL', 'drag the score gap', + grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'A reward model turns human preferences into a scalar score. For each pair a human marked chosen over rejected, the Bradley-Terry objective trains the model so the chosen response scores higher. The wider the score gap, the higher the modeled probability that a human prefers the chosen answer, and the smaller the preference loss.'); + state._render(); + } + + // ── constitutional-ai: principles flag and revise a harmful response ──────── + function constitutionalAI(host) { + var principles = [ + { name: 'harmlessness', flags: 'requests for harm', sev: 0.9 }, + { name: 'honesty', flags: 'fabricated claims', sev: 0.5 }, + { name: 'privacy', flags: 'personal data leakage', sev: 0.7 }, + { name: 'non-deception', flags: 'manipulative framing', sev: 0.4 } + ]; + var state = { which: 0 }; + var rows = el('div', {}); + var status = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + var which = Number(state.which); + while (rows.firstChild) rows.removeChild(rows.firstChild); + var p = principles[which]; + principles.forEach(function (pr, i) { + var on = i === which; + var bar = el('i'); bar.style.width = (pr.sev * 100).toFixed(0) + '%'; + if (!on) bar.style.background = 'var(--rule-soft,#ccc)'; + var lab = el('label', {}, [pr.name, el('b', {}, [on ? 'TRIGGERED' : 'pass'])]); + if (!on) lab.style.opacity = '0.5'; + rows.appendChild(el('div', { class: 'lf-ctrl' }, [lab, el('div', { class: 'lf-bar' }, [bar])])); + }); + status.innerHTML = 'revise: ' + p.flags; + meta.textContent = 'principle "' + p.name + '" flags the draft (severity ' + p.sev.toFixed(2) + ') · model critiques, then rewrites to comply'; + formula.textContent = 'response → critique against principle → revised response · no human label, the constitution is the signal'; + }; + var grid = el('div', {}, [select(state, 'which', 'principle that triggers', + principles.map(function (pr, i) { return [pr.name, String(i)]; }))]); + frame(host, 'CONSTITUTIONAL AI', 'pick the principle', + grid, [rows, el('div', { style: 'margin-top:12px' }, [status]), meta, formula], + 'Constitutional AI replaces a human labeller with a written set of principles. The draft response is passed through each principle; the one you select flags a violation, the model writes a critique of its own answer against that principle, then revises to comply. The revised answers become the training signal, so harmlessness is taught from rules rather than per-example human feedback.'); + state._render(); + } + + // ── actor-critic: advantage A = Q - V from the TD error drives the update ─── + function actorCritic(host) { + var W = 520, H = 200, PAD = 32; + var state = { td: 0.6, value: 1.0 }; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var status = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var SPAN = 2.5; + function py(v) { return H / 2 - v / SPAN * (H / 2 - PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var V = state.value, Q = V + state.td, A = Q - V; + var baseX = PAD + 40, qX = W - PAD - 40; + svg.appendChild(svgEl('line', { x1: PAD, y1: py(0), x2: W - PAD, y2: py(0), stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + // critic baseline V + svg.appendChild(svgEl('line', { x1: PAD, y1: py(V), x2: W - PAD, y2: py(V), stroke: 'var(--ink-mute,#999)', 'stroke-width': '1.5', 'stroke-dasharray': '4 3' })); + svg.appendChild(svgEl('text', { x: PAD + 4, y: py(V) - 6, 'font-size': '10', fill: 'var(--ink-mute,#777)', 'font-family': 'monospace' }, [document.createTextNode('V (critic) = ' + V.toFixed(2))])); + // realized return Q + svg.appendChild(svgEl('circle', { cx: qX, cy: py(Q), r: '5', fill: 'var(--blueprint,#3553ff)' })); + svg.appendChild(svgEl('text', { x: qX, y: py(Q) - 10, 'text-anchor': 'end', 'font-size': '10', fill: 'var(--blueprint,#3553ff)', 'font-family': 'monospace' }, [document.createTextNode('Q = V + delta = ' + Q.toFixed(2))])); + // advantage bar from V to Q + var col = A >= 0 ? 'var(--blueprint,#3553ff)' : 'var(--warn,#b8870f)'; + svg.appendChild(svgEl('line', { x1: qX, y1: py(V), x2: qX, y2: py(Q), stroke: col, 'stroke-width': '6' })); + status.innerHTML = 'advantage A = ' + A.toFixed(2); + meta.textContent = 'TD error delta = ' + state.td.toFixed(2) + ' · ' + (A > 0.05 ? 'better than expected: push the actor toward this action' : A < -0.05 ? 'worse than expected: push the actor away' : 'as expected: little update'); + formula.textContent = 'delta = r + gamma·V(s\') − V(s) · A = Q − V = delta · actor update ∝ A · grad log pi'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'td', 'TD error delta', -2, 2, 0.05), + slider(state, 'value', 'critic baseline V', -1.5, 1.5, 0.05) + ]); + frame(host, 'ACTOR-CRITIC', 'drag the TD error', + grid, [svg, el('div', { style: 'margin-top:10px' }, [status]), meta, formula], + 'The critic estimates the value V of a state (grey baseline); the actor proposes the action that produced the return Q (blue dot). The advantage A = Q - V, which equals the one-step TD error, measures whether the action did better or worse than the critic expected. A positive advantage pushes the actor toward that action, a negative one pushes it away, and the critic moves its baseline toward the observed return.'); + state._render(); + } + + // ── interpretability-probe: probe accuracy rising through the layers ──────── + function interpretabilityProbe(host) { + var W = 520, H = 210, PAD = 36, NL = 24; + var state = { layer: 12, depth: 0.5 }; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // probe accuracy: rises with a logistic in layer index; midpoint set by depth + function acc(layer) { + var mid = NL * (0.2 + 0.6 * state.depth); + var a = 0.5 + 0.48 / (1 + Math.exp(-(layer - mid) / 2.2)); + return a; + } + function px(l) { return PAD + l / (NL - 1) * (W - 2 * PAD); } + function py(a) { return H - PAD - (a - 0.5) / 0.5 * (H - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(svgEl('line', { x1: PAD, y1: py(0.5), x2: W - PAD, y2: py(0.5), stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + svg.appendChild(svgEl('text', { x: PAD - 4, y: py(0.5) + 3, 'text-anchor': 'end', 'font-size': '9', fill: 'var(--ink-mute,#777)', 'font-family': 'monospace' }, [document.createTextNode('chance')])); + var d = '', l; + for (l = 0; l < NL; l++) { d += (l ? 'L' : 'M') + px(l).toFixed(1) + ' ' + py(acc(l)).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2' })); + var L = state.layer, a = acc(L); + svg.appendChild(svgEl('line', { x1: px(L), y1: PAD, x2: px(L), y2: H - PAD, stroke: 'var(--ink-mute,#999)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + svg.appendChild(svgEl('circle', { cx: px(L), cy: py(a), r: '5', fill: 'var(--blueprint,#3553ff)' })); + num.innerHTML = (a * 100).toFixed(1) + ' % probe accuracy'; + meta.textContent = 'layer ' + L + ' of ' + (NL - 1) + ' · ' + (a < 0.62 ? 'concept not yet linearly decodable' : a > 0.9 ? 'concept clearly present in this layer' : 'concept emerging'); + formula.textContent = 'train a linear classifier on layer activations · high accuracy ⇒ the concept is linearly readable at that depth'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'layer', 'layer index', 0, NL - 1, 1), + slider(state, 'depth', 'where the concept forms (early ↔ late)', 0, 1, 0.05) + ]); + frame(host, 'INTERPRETABILITY PROBE', 'drag the layer', + grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'A linear probe is a simple classifier trained to read a concept off a layer\'s activations. Probe accuracy near chance means the concept is not linearly decodable there; high accuracy means it is. Walking the probe deeper into the network shows the concept becoming linearly readable, which is how interpretability researchers locate where a feature like a deceptive intent first appears.'); + state._render(); + } + + // ── sae-features: dense activation decomposed into sparse features ────────── + function saeFeatures(host) { + var state = { l1: 1.0 }; + var feats = [0.95, 0.82, 0.74, 0.61, 0.52, 0.44, 0.36, 0.29, 0.21, 0.14, 0.09, 0.05]; + var rows = el('div', {}); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + while (rows.firstChild) rows.removeChild(rows.firstChild); + var thr = state.l1 * 0.18; // higher L1 -> higher activation threshold -> fewer survive + var active = 0, recon = 0, total = 0; + feats.forEach(function (f) { total += f; }); + feats.forEach(function (f, i) { + var on = f >= thr; + var val = on ? f : 0; + if (on) { active++; recon += f; } + var bar = el('i'); bar.style.width = (val / 0.95 * 100).toFixed(0) + '%'; + if (!on) bar.style.background = 'var(--rule-soft,#ccc)'; + var lab = el('label', {}, ['f' + (i + 1) + (on ? '' : ' ·'), el('b', {}, [on ? val.toFixed(2) : 'off'])]); + if (!on) lab.style.opacity = '0.4'; + rows.appendChild(el('div', { class: 'lf-ctrl' }, [lab, el('div', { class: 'lf-bar' }, [bar])])); + }); + var reconPct = recon / total * 100; + num.innerHTML = active + ' of ' + feats.length + ' features active'; + meta.textContent = 'reconstruction ' + reconPct.toFixed(0) + '% of the dense vector · ' + (state.l1 < 0.5 ? 'low L1: dense, polysemantic features' : state.l1 > 1.8 ? 'high L1: very sparse, may lose signal' : 'sparse and monosemantic'); + formula.textContent = 'minimize ‖x − decode(f)‖² + lambda·‖f‖₁ · lambda = ' + state.l1.toFixed(2) + ' · higher lambda ⇒ fewer active features'; + }; + var grid = el('div', {}, [slider(state, 'l1', 'sparsity coefficient lambda (L1)', 0.1, 2.5, 0.05)]); + frame(host, 'SPARSE AUTOENCODER', 'drag the L1 coefficient', + grid, [rows, el('div', { style: 'margin-top:12px' }, [num]), meta, formula], + 'A sparse autoencoder decomposes a dense activation into a much larger set of features, most of which stay off. The L1 penalty on the feature activations sets how sparse the code is. Too little and the features stay dense and polysemantic; too much and a few features carry everything but reconstruction suffers. The sweet spot gives few active, monosemantic features that each stand for one human-readable concept.'); + state._render(); + } + + // ── jailbreak-defense: attack success falls but over-refusal rises ───────── + function jailbreakDefense(host) { + var W = 520, H = 210, PAD = 36; + var state = { strength: 0.5 }; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var status = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // attack success rate falls with defense; over-refusal on benign prompts rises + function asr(s) { return 0.85 * Math.exp(-2.6 * s); } + function refuse(s) { return 0.02 + 0.6 * Math.pow(s, 2.2); } + function px(s) { return PAD + s * (W - 2 * PAD); } + function py(v) { return H - PAD - clamp(v, 0, 1) * (H - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(svgEl('line', { x1: PAD, y1: H - PAD, x2: W - PAD, y2: H - PAD, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + function curve(fn, st) { var d = '', i, s; for (i = 0; i <= 100; i++) { s = i / 100; d += (i ? 'L' : 'M') + px(s).toFixed(1) + ' ' + py(fn(s)).toFixed(1) + ' '; } svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: st, 'stroke-width': '2' })); } + curve(asr, 'var(--warn,#b8870f)'); + curve(refuse, 'var(--blueprint,#3553ff)'); + var s = state.strength, a = asr(s), r = refuse(s); + var mx = px(s); + svg.appendChild(svgEl('line', { x1: mx, y1: PAD, x2: mx, y2: H - PAD, stroke: 'var(--ink-mute,#999)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + svg.appendChild(svgEl('circle', { cx: mx, cy: py(a), r: '4.5', fill: 'var(--warn,#b8870f)' })); + svg.appendChild(svgEl('circle', { cx: mx, cy: py(r), r: '4.5', fill: 'var(--blueprint,#3553ff)' })); + status.innerHTML = 'attack success ' + (a * 100).toFixed(0) + '%'; + meta.textContent = 'amber: jailbreak success ' + (a * 100).toFixed(0) + '% · blue: over-refusal on benign prompts ' + (r * 100).toFixed(0) + '% · ' + (s < 0.3 ? 'too permissive' : s > 0.8 ? 'too restrictive' : 'balanced'); + formula.textContent = 'stronger filter ⇒ lower attack success, higher false refusals · tune for the knee of both curves'; + }; + var grid = el('div', {}, [slider(state, 'strength', 'defense strength', 0, 1, 0.02)]); + frame(host, 'JAILBREAK DEFENSE', 'drag the defense strength', + grid, [svg, el('div', { style: 'margin-top:10px' }, [status]), meta, formula], + 'A safety filter that blocks adversarial prompts also risks refusing harmless ones. As you turn defense strength up, the amber attack-success curve falls but the blue over-refusal curve climbs as benign requests get caught. There is no free lunch: the operating point is chosen at the knee where most jailbreaks are stopped without refusing too many legitimate users.'); + state._render(); + } + + // ── scalable-oversight: weak judge supervising strong agents via debate ───── + function scalableOversight(host) { + var W = 520, H = 210, PAD = 36; + var state = { difficulty: 0.5, mode: 'debate' }; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var status = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // unaided weak judge accuracy decays fast with difficulty; oversight protocols decay slower + function direct(x) { return 0.5 + 0.45 * Math.exp(-3.2 * x); } + function debate(x) { return 0.5 + 0.45 * Math.exp(-1.3 * x); } + function recurse(x) { return 0.5 + 0.45 * Math.exp(-1.0 * x); } + function aided(x) { return state.mode === 'recursion' ? recurse(x) : debate(x); } + function px(x) { return PAD + x * (W - 2 * PAD); } + function py(v) { return H - PAD - (v - 0.5) / 0.5 * (H - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(svgEl('line', { x1: PAD, y1: py(0.5), x2: W - PAD, y2: py(0.5), stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + svg.appendChild(svgEl('text', { x: PAD - 4, y: py(0.5) + 3, 'text-anchor': 'end', 'font-size': '9', fill: 'var(--ink-mute,#777)', 'font-family': 'monospace' }, [document.createTextNode('chance')])); + function curve(fn, st, dash) { var d = '', i, x; for (i = 0; i <= 100; i++) { x = i / 100; d += (i ? 'L' : 'M') + px(x).toFixed(1) + ' ' + py(fn(x)).toFixed(1) + ' '; } var a = { d: d, fill: 'none', stroke: st, 'stroke-width': '2' }; if (dash) a['stroke-dasharray'] = '4 3'; svg.appendChild(svgEl('path', a)); } + curve(direct, 'var(--ink-mute,#999)', true); + curve(aided, 'var(--blueprint,#3553ff)', false); + var x = state.difficulty, vu = direct(x), va = aided(x), mx = px(x); + svg.appendChild(svgEl('line', { x1: mx, y1: PAD, x2: mx, y2: H - PAD, stroke: 'var(--ink-mute,#999)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + svg.appendChild(svgEl('circle', { cx: mx, cy: py(vu), r: '4', fill: 'var(--ink-mute,#999)' })); + svg.appendChild(svgEl('circle', { cx: mx, cy: py(va), r: '5', fill: 'var(--blueprint,#3553ff)' })); + status.innerHTML = 'oversight accuracy ' + (va * 100).toFixed(0) + '%'; + meta.textContent = 'grey: judge alone ' + (vu * 100).toFixed(0) + '% · blue: judge + ' + state.mode + ' ' + (va * 100).toFixed(0) + '% · gain ' + ((va - vu) * 100).toFixed(0) + ' points'; + formula.textContent = state.mode === 'debate' + ? 'two strong agents argue; the weak judge picks the more defensible answer' + : 'task split into checkable subtasks; the weak judge verifies each piece'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'difficulty', 'task difficulty', 0, 1, 0.02), + select(state, 'mode', 'oversight protocol', [['debate', 'debate'], ['recursion', 'recursion']]) + ]); + frame(host, 'SCALABLE OVERSIGHT', 'drag the task difficulty', + grid, [svg, el('div', { style: 'margin-top:10px' }, [status]), meta, formula], + 'A weak judge cannot directly check a strong agent on hard tasks: the grey curve falls toward chance as difficulty rises. Oversight protocols help the judge keep up. In debate, two strong agents argue and the judge picks the more defensible side; in recursion, the task is decomposed into pieces the judge can verify. Either way the blue curve decays more slowly, so the judge stays above chance on tasks it could never solve alone.'); + state._render(); + } + + LF.register({ + 'ppo-clip': ppoClip, + 'reward-model': rewardModel, + 'constitutional-ai': constitutionalAI, + 'actor-critic': actorCritic, + 'interpretability-probe': interpretabilityProbe, + 'sae-features': saeFeatures, + 'jailbreak-defense': jailbreakDefense, + 'scalable-oversight': scalableOversight + }); +})(); diff --git a/site/figures-foundations2.js b/site/figures-foundations2.js new file mode 100644 index 0000000000..b867843d4e --- /dev/null +++ b/site/figures-foundations2.js @@ -0,0 +1,406 @@ +/* figures-foundations2.js — interactive lesson figures for Phase 4 (computer + vision), Phase 6 (speech & audio), and Phase 8 (generative AI). Loads after + lesson-figures.js and registers widgets through window.LF. Vanilla ES5, no + deps, theme via CSS vars. Authoring is the same fenced block in docs/en.md: + ```figure + data-augmentation + ``` */ +(function () { + 'use strict'; + var LF = window.LF; + if (!LF) { return; } + var el = LF.el, svgEl = LF.svgEl, slider = LF.slider, select = LF.select; + var clamp = LF.clamp, lerp = LF.lerp, fmtInt = LF.fmtInt; + + function shell(label, hint, grid, outKids, caption) { + return el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, [label]), el('span', {}, [hint])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, outKids)]), + el('div', { class: 'lf-cap' }, [caption]) + ]); + } + function tx(s) { return document.createTextNode(s); } + + // ── data-augmentation: one source image, four transformed copies ─────────── + function dataAugmentation(host) { + var SRC = [ + [0, 0, 6, 6, 0, 0], + [0, 6, 9, 9, 3, 0], + [6, 9, 2, 2, 9, 3], + [3, 9, 2, 2, 9, 6], + [0, 3, 9, 9, 6, 0], + [0, 0, 3, 6, 0, 0] + ]; + var N = 6; + var state = { mode: 'flip', copies: 4 }; + var svg = svgEl('svg', { viewBox: '0 0 520 200' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + + function sample(r, c, variant) { + if (state.mode === 'flip') { return SRC[r][variant % 2 ? N - 1 - c : c]; } + if (state.mode === 'rotate') { + var q = variant % 4, rr = r, cc = c, t; + while (q-- > 0) { t = rr; rr = cc; cc = N - 1 - t; } + return SRC[rr][cc]; + } + if (state.mode === 'crop') { + var off = variant % 3; + var sr = clamp(r + off - 1, 0, N - 1), sc = clamp(c + off - 1, 0, N - 1); + return SRC[sr][sc]; + } + var shift = (variant - 1) * 2; + return clamp(SRC[r][c] + shift, 0, 9); + } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var cell = 18, gap = 16, x0 = 8, y0 = 28; + svg.appendChild(svgEl('text', { x: x0, y: 18, fill: 'var(--ink-mute,#777)', 'font-size': '10', 'font-family': 'monospace' }, [tx('source')])); + var p, r, c; + for (r = 0; r < N; r++) for (c = 0; c < N; c++) { + svg.appendChild(svgEl('rect', { x: x0 + c * cell, y: y0 + r * cell, width: cell - 1, height: cell - 1, fill: 'var(--blueprint,#3553ff)', opacity: (0.08 + 0.9 * SRC[r][c] / 9).toFixed(3) })); + } + var bx = x0 + N * cell + 30; + for (p = 1; p <= state.copies; p++) { + var px0 = bx + (p - 1) * (N * cell + gap); + svg.appendChild(svgEl('text', { x: px0, y: 18, fill: 'var(--warn,#b8870f)', 'font-size': '10', 'font-family': 'monospace' }, [tx('aug ' + p)])); + for (r = 0; r < N; r++) for (c = 0; c < N; c++) { + svg.appendChild(svgEl('rect', { x: px0 + c * cell, y: y0 + r * cell, width: cell - 1, height: cell - 1, fill: 'var(--blueprint,#3553ff)', opacity: (0.08 + 0.9 * sample(r, c, p) / 9).toFixed(3) })); + } + } + var base = 1000; + meta.textContent = 'each pass yields a fresh view · ' + base + ' images x ' + (state.copies + 1) + ' = ' + fmtInt(base * (state.copies + 1)) + ' effective examples'; + formula.textContent = 'augment(x) preserves the label while changing the pixels · the model sees more variation, generalizes better'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + select(state, 'mode', 'transform', [['horizontal flip', 'flip'], ['rotate 90', 'rotate'], ['random crop', 'crop'], ['color jitter', 'color']]), + slider(state, 'copies', 'augmented copies', 1, 4, 1) + ]); + host.appendChild(shell('DATA AUGMENTATION', 'pick a transform', + grid, [svg, meta, formula], + 'Augmentation applies label-preserving transforms — flips, rotations, crops, color shifts — to each training image, so one labelled example becomes many. The network never sees the exact same input twice and learns features that survive these changes, which expands the effective dataset and curbs overfitting without collecting more data.')); + state._render(); + } + + // ── transfer-learning: freeze a pretrained backbone, train the head ──────── + function transferLearning(host) { + var TOTAL = 24, FULL = 24e6; + var state = { frozen: 18 }; + var svg = svgEl('svg', { viewBox: '0 0 520 150' }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + function human(x) { var u = ['', 'K', 'M', 'B'], i = 0; while (x >= 1000 && i < u.length - 1) { x /= 1000; i++; } return x.toFixed(x < 10 ? 1 : 0) + u[i]; } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var frozen = clamp(state.frozen, 0, TOTAL), trainable = TOTAL - frozen; + var x0 = 30, y0 = 40, bw = 460 / TOTAL, bh = 46, i; + for (i = 0; i < TOTAL; i++) { + var isFrozen = i < frozen; + svg.appendChild(svgEl('rect', { x: x0 + i * bw, y: y0, width: bw - 1.5, height: bh, fill: isFrozen ? 'var(--rule-soft,#ddd)' : 'var(--blueprint,#3553ff)', opacity: isFrozen ? '0.9' : '0.85' })); + } + svg.appendChild(svgEl('text', { x: x0, y: y0 - 10, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx('input → ' + frozen + ' frozen (grey) · ' + trainable + ' trainable (blue) → head')])); + var fracTrain = trainable / TOTAL; + var trainableParams = FULL * fracTrain; + var epochs = Math.max(2, Math.round(2 + 22 * fracTrain)); + svg.appendChild(svgEl('text', { x: x0, y: y0 + bh + 22, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx('gradients flow only through the blue layers')])); + num.innerHTML = human(trainableParams) + ' trainable params'; + meta.textContent = Math.round(fracTrain * 100) + '% of the backbone trains · about ' + epochs + ' epochs to converge on a small dataset'; + formula.textContent = 'frozen layers keep pretrained weights, contribute no gradients · fewer trainable params → less data and compute needed'; + }; + var grid = el('div', {}, [slider(state, 'frozen', 'layers frozen', 0, TOTAL, 1)]); + host.appendChild(shell('TRANSFER LEARNING', 'drag the freeze line', + grid, [svg, num, meta, formula], + 'A pretrained backbone already knows generic features — edges, textures, shapes. Transfer learning freezes those lower layers and trains only the top few plus a new head on the target task. Fewer trainable parameters means fewer gradients to store, far less data to fit, and faster convergence; freeze more when your dataset is tiny, fewer when it is large and different.')); + state._render(); + } + + // ── batchnorm-inference: batch stats in training vs running averages ─────── + function batchnormInference(host) { + var POP_MEAN = 0.0, POP_STD = 1.0; + var state = { batch: 8, seed: 3 }; + var svg = svgEl('svg', { viewBox: '0 0 520 200' }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + function rng(s) { var x = Math.sin(s * 12.9898) * 43758.5453; return x - Math.floor(x); } + function gauss(s) { var u = Math.max(1e-6, rng(s)), v = rng(s + 7.13); return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var W = 520, H = 200, PAD = 30, n = state.batch, i, sum = 0, sumsq = 0, xs = []; + for (i = 0; i < n; i++) { var g = POP_MEAN + POP_STD * gauss(state.seed * 31 + i * 1.7); xs.push(g); sum += g; sumsq += g * g; } + var bMean = sum / n, bVar = sumsq / n - bMean * bMean, bStd = Math.sqrt(Math.max(1e-6, bVar)); + function px(v) { return W / 2 + v / 4 * (W - 2 * PAD) / 2; } + svg.appendChild(svgEl('line', { x1: PAD, y1: 70, x2: W - PAD, y2: 70, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + for (i = 0; i < n; i++) { svg.appendChild(svgEl('circle', { cx: px(xs[i]), cy: 70, r: '4', fill: 'var(--blueprint,#3553ff)', opacity: '0.8' })); } + svg.appendChild(svgEl('line', { x1: px(bMean), y1: 50, x2: px(bMean), y2: 90, stroke: 'var(--warn,#b8870f)', 'stroke-width': '2' })); + svg.appendChild(svgEl('text', { x: PAD, y: 44, fill: 'var(--warn,#b8870f)', 'font-size': '11', 'font-family': 'monospace' }, [tx('training: this batch μ=' + bMean.toFixed(2) + ' σ=' + bStd.toFixed(2))])); + svg.appendChild(svgEl('line', { x1: PAD, y1: 150, x2: W - PAD, y2: 150, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + svg.appendChild(svgEl('line', { x1: px(POP_MEAN), y1: 130, x2: px(POP_MEAN), y2: 170, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2' })); + svg.appendChild(svgEl('text', { x: PAD, y: 124, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx('inference: running avg μ=' + POP_MEAN.toFixed(2) + ' σ=' + POP_STD.toFixed(2))])); + var err = Math.abs(bMean - POP_MEAN) + Math.abs(bStd - POP_STD); + num.innerHTML = err.toFixed(3) + ' batch-vs-population gap'; + meta.textContent = n + ' samples per batch · ' + (n <= 4 ? 'small batch: noisy estimates, unstable normalization' : n >= 32 ? 'large batch: stable estimates close to the population' : 'moderate batch: usable estimates'); + formula.textContent = 'train: normalize by THIS batch μ,σ · eval: normalize by running averages collected during training'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'batch', 'batch size', 1, 64, 1), + slider(state, 'seed', 'resample batch', 1, 20, 1) + ]); + host.appendChild(shell('BATCHNORM: TRAIN vs EVAL', 'drag the batch size', + grid, [svg, num, meta, formula], + 'During training BatchNorm normalizes each activation by the mean and variance of the current mini-batch (orange), and quietly accumulates a running average. At inference it switches to those frozen running averages (blue) so a single input is processed deterministically. Small batches make the per-batch statistics noisy, which is why tiny batches hurt BatchNorm and motivate Group or Layer Norm.')); + state._render(); + } + + // ── ctc-collapse: per-frame chars collapse to a transcript ───────────────── + function ctcCollapse(host) { + var FRAMES = ['_', 'h', 'h', 'e', '_', 'l', 'l', '_', 'l', 'l', 'o', 'o', '_']; + var state = { stage: 2 }; + var svg = svgEl('svg', { viewBox: '0 0 520 170' }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + + function collapseRepeats(seq) { + var out = [], prev = null, i; + for (i = 0; i < seq.length; i++) { if (seq[i] !== prev) { out.push(seq[i]); } prev = seq[i]; } + return out; + } + function removeBlanks(seq) { return seq.filter(function (c) { return c !== '_'; }); } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var shown; + if (state.stage === 0) { shown = FRAMES.slice(); } + else if (state.stage === 1) { shown = collapseRepeats(FRAMES); } + else { shown = removeBlanks(collapseRepeats(FRAMES)); } + var x0 = 20, y0 = 50, cw = 36, i; + var stageLabel = state.stage === 0 ? 'raw per-frame argmax (' + FRAMES.length + ' frames)' + : state.stage === 1 ? 'merge adjacent repeats' : 'drop blank token "_"'; + svg.appendChild(svgEl('text', { x: x0, y: 30, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx(stageLabel)])); + for (i = 0; i < shown.length; i++) { + var blank = shown[i] === '_'; + svg.appendChild(svgEl('rect', { x: x0 + i * cw, y: y0, width: cw - 4, height: 36, fill: blank ? 'var(--rule-soft,#ddd)' : 'var(--blueprint,#3553ff)', opacity: blank ? '0.7' : '0.85' })); + svg.appendChild(svgEl('text', { x: x0 + i * cw + (cw - 4) / 2, y: y0 + 24, fill: 'var(--bg,#fafaf5)', 'font-size': '16', 'font-family': 'monospace', 'text-anchor': 'middle' }, [tx(blank ? '∅' : shown[i])])); + } + var transcript = removeBlanks(collapseRepeats(FRAMES)).join(''); + svg.appendChild(svgEl('text', { x: x0, y: y0 + 70, fill: 'var(--warn,#b8870f)', 'font-size': '13', 'font-family': 'monospace' }, [tx('final transcript: "' + transcript + '"')])); + num.innerHTML = shown.length + ' symbols at this stage'; + meta.textContent = state.stage === 0 ? 'the acoustic model emits one symbol per audio frame, with repeats and blanks' + : state.stage === 1 ? 'repeated runs of the same symbol become one' : 'blanks are removed, leaving the text'; + formula.textContent = 'CTC decode: collapse repeats first, THEN remove blanks · the blank lets the model separate true double letters'; + }; + var grid = el('div', {}, [slider(state, 'stage', 'decode stage (0 raw → 1 merge → 2 final)', 0, 2, 1)]); + host.appendChild(shell('CTC COLLAPSE', 'step through decoding', + grid, [svg, num, meta, formula], + 'CTC lets an acoustic model emit one label per frame without knowing the alignment. Decoding runs two steps in order: first collapse any run of identical symbols into one, then delete the blank token. The blank is essential — it sits between two real "l" frames so that "hello" keeps both letters instead of merging them into one.')); + state._render(); + } + + // ── mfcc-pipeline: spectrogram → mel → log → DCT → keep N coeffs ──────────── + function mfccPipeline(host) { + var state = { keep: 13 }; + var MELS = 40; + var svg = svgEl('svg', { viewBox: '0 0 520 170' }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var stages = ['spectrogram', 'mel filterbank', 'log', 'DCT', 'MFCC']; + var x0 = 16, y0 = 30, sw = 96, gap = 4, i; + for (i = 0; i < stages.length; i++) { + var sx = x0 + i * (sw + gap); + svg.appendChild(svgEl('rect', { x: sx, y: y0, width: sw, height: 40, fill: i === stages.length - 1 ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', opacity: i === stages.length - 1 ? '0.85' : '1', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1' })); + svg.appendChild(svgEl('text', { x: sx + sw / 2, y: y0 + 24, fill: i === stages.length - 1 ? 'var(--bg,#fafaf5)' : 'var(--ink-soft,#555)', 'font-size': '10', 'font-family': 'monospace', 'text-anchor': 'middle' }, [tx(stages[i])])); + if (i < stages.length - 1) { + svg.appendChild(svgEl('text', { x: sx + sw + gap / 2 - 1, y: y0 + 26, fill: 'var(--ink-mute,#777)', 'font-size': '12', 'font-family': 'monospace', 'text-anchor': 'middle' }, [tx('→')])); + } + } + var bx = 16, by = 110, bw = 488 / MELS; + for (i = 0; i < MELS; i++) { + var kept = i < state.keep; + var energy = Math.exp(-i * 0.12); + svg.appendChild(svgEl('rect', { x: bx + i * bw, y: by, width: bw - 0.8, height: 36, fill: kept ? 'var(--blueprint,#3553ff)' : 'var(--rule-soft,#ddd)', opacity: kept ? (0.3 + 0.7 * energy).toFixed(3) : '0.5' })); + } + svg.appendChild(svgEl('text', { x: bx, y: by - 6, fill: 'var(--ink-mute,#777)', 'font-size': '10', 'font-family': 'monospace' }, [tx('cepstral coefficients: blue kept, grey discarded (' + MELS + ' total)')])); + num.innerHTML = state.keep + ' MFCC coefficients kept'; + meta.textContent = 'keeping the first ' + state.keep + ' of ' + MELS + ' coefficients · ' + (state.keep <= 8 ? 'coarse: smooth spectral envelope only' : state.keep >= 26 ? 'fine: includes pitch-like detail and noise' : 'typical for speech (12-13)'); + formula.textContent = 'STFT power → mel filterbank → log → DCT → keep low coefficients · the DCT compacts the envelope into the first few'; + }; + var grid = el('div', {}, [slider(state, 'keep', 'cepstral coefficients kept', 4, 40, 1)]); + host.appendChild(shell('MFCC PIPELINE', 'drag the coefficient count', + grid, [svg, num, meta, formula], + 'MFCCs run a fixed pipeline: take the spectrogram, warp it onto mel-spaced filters, take the log to mimic loudness perception, then apply a DCT. The DCT packs the smooth spectral envelope into the first few coefficients, so keeping only the lowest 12-13 captures the vocal-tract shape that distinguishes phonemes while discarding pitch and noise in the higher coefficients.')); + state._render(); + } + + // ── autoencoder-bottleneck: reconstruction quality vs latent dim ─────────── + function autoencoderBottleneck(host) { + var DIN = 16; + var SIGNAL = []; + (function () { var i; for (i = 0; i < DIN; i++) { SIGNAL.push(0.5 + 0.45 * Math.sin(i * 0.9) + 0.18 * Math.sin(i * 2.7)); } })(); + var state = { latent: 4 }; + var svg = svgEl('svg', { viewBox: '0 0 520 180' }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + + function reconstruct(z) { + var coeffs = [], k2; + for (k2 = 0; k2 < z; k2++) { + var c = 0, i2; + for (i2 = 0; i2 < DIN; i2++) { c += (SIGNAL[i2] - 0.5) * Math.cos(Math.PI * (k2 + 0.5) * i2 / DIN); } + coeffs.push(c * 2 / DIN); + } + var rec = [], i3, kk; + for (i3 = 0; i3 < DIN; i3++) { + var s = 0.5; + for (kk = 0; kk < z; kk++) { s += coeffs[kk] * Math.cos(Math.PI * (kk + 0.5) * i3 / DIN); } + rec.push(s); + } + return rec; + } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var rec = reconstruct(state.latent); + var x0 = 30, y0 = 16, gw = 460, gh = 110, i; + function px(i) { return x0 + i / (DIN - 1) * gw; } + function py(v) { return y0 + gh - clamp(v, -0.2, 1.2) / 1.4 * gh; } + var od = '', rd = ''; + for (i = 0; i < DIN; i++) { od += (i ? 'L' : 'M') + px(i).toFixed(1) + ' ' + py(SIGNAL[i]).toFixed(1) + ' '; rd += (i ? 'L' : 'M') + px(i).toFixed(1) + ' ' + py(rec[i]).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: od, fill: 'none', stroke: 'var(--ink-mute,#999)', 'stroke-width': '2' })); + svg.appendChild(svgEl('path', { d: rd, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2', 'stroke-dasharray': '5 3' })); + svg.appendChild(svgEl('text', { x: x0, y: y0 + gh + 22, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx('grey = input (' + DIN + ' dims) · blue dashed = reconstruction from ' + state.latent + '-dim bottleneck')])); + var mse = 0; + for (i = 0; i < DIN; i++) { mse += (rec[i] - SIGNAL[i]) * (rec[i] - SIGNAL[i]); } + mse /= DIN; + var ratio = DIN / state.latent; + num.innerHTML = mse.toFixed(4) + ' reconstruction MSE'; + meta.textContent = ratio.toFixed(1) + 'x compression (' + DIN + ' → ' + state.latent + ') · ' + (state.latent <= 2 ? 'bottleneck too small: detail is lost' : state.latent >= DIN - 1 ? 'wide bottleneck: near-perfect but no compression' : 'compresses while keeping the main structure'); + formula.textContent = 'x → encoder → z (' + state.latent + ' dims) → decoder → x̂ · the bottleneck forces the network to keep only what matters'; + }; + var grid = el('div', {}, [slider(state, 'latent', 'bottleneck dimension', 1, DIN, 1)]); + host.appendChild(shell('AUTOENCODER BOTTLENECK', 'drag the latent dim', + grid, [svg, num, meta, formula], + 'An autoencoder squeezes its input through a narrow bottleneck and rebuilds it on the far side. A wide bottleneck copies everything but learns nothing; a tiny one forces the network to discard detail and keep only the dominant structure, so reconstruction error rises as compression increases. The sweet spot keeps the signal while throwing away the noise — that learned code is the useful representation.')); + state._render(); + } + + // ── normalizing-flow: invertible map, base → target, log-det Jacobian ────── + function normalizingFlow(host) { + var state = { a: 1.4 }; + var W = 520, H = 210, PAD = 34; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + function base(z) { return Math.exp(-0.5 * z * z) / Math.sqrt(2 * Math.PI); } + function fwd(z, a) { return z + a * Math.tanh(z); } + function dfwd(z, a) { var th = Math.tanh(z); return 1 + a * (1 - th * th); } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var a = state.a, i; + var xs = [], baseY = [], tgtY = []; + for (i = 0; i <= 160; i++) { var z = -3.4 + 6.8 * i / 160; xs.push(z); } + function px(x) { return PAD + (x + 3.6) / 7.2 * (W - 2 * PAD); } + var maxP = 0; + var pts = []; + for (i = 0; i < xs.length; i++) { + var z = xs[i], x = fwd(z, a); + var pz = base(z), jac = Math.abs(dfwd(z, a)); + var pxden = pz / jac; + pts.push({ z: z, x: x, pz: pz, px: pxden }); + if (pz > maxP) maxP = pz; if (pxden > maxP) maxP = pxden; + } + function py(p) { return H - PAD - p / maxP * (H - 2 * PAD); } + var bd = '', td = ''; + for (i = 0; i < pts.length; i++) { bd += (i ? 'L' : 'M') + px(pts[i].z).toFixed(1) + ' ' + py(pts[i].pz).toFixed(1) + ' '; td += (i ? 'L' : 'M') + px(pts[i].x).toFixed(1) + ' ' + py(pts[i].px).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: bd, fill: 'none', stroke: 'var(--ink-mute,#999)', 'stroke-width': '2' })); + svg.appendChild(svgEl('path', { d: td, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2.2' })); + svg.appendChild(svgEl('text', { x: PAD, y: PAD - 6, fill: 'var(--ink-mute,#777)', 'font-size': '11', 'font-family': 'monospace' }, [tx('grey = base Gaussian p(z) · blue = pushed-forward density p(x)')])); + var logdetAt0 = Math.log(Math.abs(dfwd(0, a))); + num.innerHTML = logdetAt0.toFixed(3) + ' log|det J| at z=0'; + meta.textContent = 'flow strength a = ' + a.toFixed(2) + ' · ' + (a < 0.3 ? 'near identity: target stays close to the base' : a > 1.8 ? 'strong warp: density piles up where the map compresses' : 'moderate warp into a multi-modal shape'); + formula.textContent = 'x = z + a·tanh(z) invertible · p(x) = p(z) / |dx/dz| · log p(x) = log p(z) − log|det J|'; + }; + var grid = el('div', {}, [slider(state, 'a', 'flow strength a', 0, 2.5, 0.05)]); + host.appendChild(shell('NORMALIZING FLOW', 'drag the flow parameter', + grid, [svg, num, meta, formula], + 'A normalizing flow maps a simple base density (grey Gaussian) through an invertible function to a complex target (blue). Because the map is invertible, the change-of-variables formula gives the exact density: divide by the absolute Jacobian determinant, or in logs, subtract log|det J|. Where the map stretches space the density thins; where it compresses, the density piles up — and because everything is exact, the flow can be trained by maximum likelihood.')); + state._render(); + } + + // ── score-matching: score vector field and Langevin sampling steps ───────── + function scoreMatching(host) { + var state = { steps: 18, step: 0.06 }; + var W = 520, H = 240, PAD = 24; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var MODES = [{ x: -1.1, y: 0.4 }, { x: 1.2, y: -0.5 }]; + function dens(x, y) { var s = 0, m; for (m = 0; m < MODES.length; m++) { var dx = x - MODES[m].x, dy = y - MODES[m].y; s += Math.exp(-2 * (dx * dx + dy * dy)); } return s + 1e-6; } + function score(x, y) { + var sx = 0, sy = 0, w = 0, m; + for (m = 0; m < MODES.length; m++) { var dx = x - MODES[m].x, dy = y - MODES[m].y; var g = Math.exp(-2 * (dx * dx + dy * dy)); w += g; sx += g * (-4 * dx); sy += g * (-4 * dy); } + return { x: sx / w, y: sy / w }; + } + function gx(x) { return PAD + (x + 2.4) / 4.8 * (W - 2 * PAD); } + function gy(y) { return H - PAD - (y + 2.0) / 4.0 * (H - 2 * PAD); } + + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var ix, iy; + for (ix = -2; ix <= 2; ix += 0.5) for (iy = -1.6; iy <= 1.6; iy += 0.5) { + var s = score(ix, iy); + var mag = Math.sqrt(s.x * s.x + s.y * s.y) + 1e-6; + var ux = s.x / mag, uy = s.y / mag, L = 11; + var x1 = gx(ix), y1 = gy(iy), x2 = gx(ix) + ux * L, y2 = gy(iy) - uy * L; + svg.appendChild(svgEl('line', { x1: x1, y1: y1, x2: x2, y2: y2, stroke: 'var(--ink-mute,#999)', 'stroke-width': '1', opacity: '0.7' })); + svg.appendChild(svgEl('circle', { cx: x2, cy: y2, r: '1.6', fill: 'var(--ink-mute,#999)' })); + } + var m; + for (m = 0; m < MODES.length; m++) { svg.appendChild(svgEl('circle', { cx: gx(MODES[m].x), cy: gy(MODES[m].y), r: '6', fill: 'none', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '2' })); } + var px = -2.0, py = 1.4, path = '', i; + for (i = 0; i <= state.steps; i++) { + path += (i ? 'L' : 'M') + gx(px).toFixed(1) + ' ' + gy(py).toFixed(1) + ' '; + var sc = score(px, py); + px = px + state.step * sc.x; + py = py + state.step * sc.y; + } + svg.appendChild(svgEl('path', { d: path, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.6', 'stroke-dasharray': '4 3' })); + svg.appendChild(svgEl('circle', { cx: gx(px), cy: gy(py), r: '5', fill: 'var(--warn,#b8870f)' })); + var finalDens = dens(px, py); + num.innerHTML = finalDens.toFixed(3) + ' density at the sample'; + meta.textContent = state.steps + ' Langevin steps · the sample climbs the grey arrows into a high-density mode'; + formula.textContent = 'score s(x) = ∇ₓ log p(x) · Langevin: x ← x + ε·s(x) (+ noise) · arrows point toward where data is dense'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'steps', 'Langevin steps', 0, 40, 1), + slider(state, 'step', 'step size ε', 0.01, 0.2, 0.01) + ]); + host.appendChild(shell('SCORE MATCHING', 'drag the steps', + grid, [svg, num, meta, formula], + 'A score-based model learns the score — the gradient of the log-density — shown here as the grey vector field pointing toward where data is dense. Generation needs no explicit density: start from noise and repeatedly step along the score (Langevin dynamics, with a little added noise each step). The orange sample follows the arrows out of the empty regions and settles into a high-density mode, which is exactly how diffusion models draw samples.')); + state._render(); + } + + LF.register({ + 'data-augmentation': dataAugmentation, + 'transfer-learning': transferLearning, + 'batchnorm-inference': batchnormInference, + 'ctc-collapse': ctcCollapse, + 'mfcc-pipeline': mfccPipeline, + 'autoencoder-bottleneck': autoencoderBottleneck, + 'normalizing-flow': normalizingFlow, + 'score-matching': scoreMatching + }); +})(); diff --git a/site/figures-llmeng.js b/site/figures-llmeng.js new file mode 100644 index 0000000000..57cda05ba1 --- /dev/null +++ b/site/figures-llmeng.js @@ -0,0 +1,370 @@ +/* figures-llmeng.js: interactive lesson figures for Phase 11 (LLM engineering) + and Phase 13 (tools & protocols). Loads after lesson-figures.js and registers + through window.LF.register. Vanilla ES5, no deps, theme via CSS vars. Authoring + is the same fenced block: + ```figure + few-shot-curve + ``` */ +(function () { + 'use strict'; + var LF = window.LF; + if (!LF) { return; } + var el = LF.el, svgEl = LF.svgEl, slider = LF.slider, clamp = LF.clamp; + + // ── few-shot-curve: accuracy vs number of in-context examples k ───────────── + function fewShotCurve(host) { + var state = { k: 4 }; + var W = 520, H = 220, PAD = 32, KMAX = 16; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var A0 = 0.42, AMAX = 0.92; + function acc(k) { return A0 + (AMAX - A0) * (1 - Math.exp(-k / 3.5)); } + function px(k) { return PAD + k / KMAX * (W - 2 * PAD); } + function py(a) { return H - PAD - (a - 0.3) / 0.7 * (H - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.appendChild(svgEl('line', { x1: PAD, y1: py(AMAX), x2: W - PAD, y2: py(AMAX), stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'stroke-dasharray': '3 3' })); + var d = '', i; + for (i = 0; i <= 120; i++) { var k = KMAX * i / 120; d += (i ? 'L' : 'M') + px(k).toFixed(1) + ' ' + py(acc(k)).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2' })); + svg.appendChild(svgEl('circle', { cx: px(0), cy: py(acc(0)), r: '3.5', fill: 'var(--ink-mute,#777)' })); + svg.appendChild(svgEl('circle', { cx: px(state.k), cy: py(acc(state.k)), r: '5', fill: 'var(--blueprint,#3553ff)' })); + var a = acc(state.k), gap = a - acc(0); + num.innerHTML = (a * 100).toFixed(1) + ' % accuracy'; + meta.textContent = (state.k === 0 ? 'zero-shot baseline' : state.k + '-shot') + ' · +' + (gap * 100).toFixed(1) + ' pts over zero-shot · ' + (state.k >= 8 ? 'plateau: more examples barely help' : 'still climbing'); + formula.textContent = 'accuracy(k) = ' + (A0 * 100).toFixed(0) + '% + (' + ((AMAX - A0) * 100).toFixed(0) + ' pts)(1 − e^(−k/3.5)) · diminishing returns'; + }; + var grid = el('div', {}, [slider(state, 'k', 'in-context examples k', 0, KMAX, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['FEW-SHOT CURVE']), el('span', {}, ['drag the example count'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula])]), + el('div', { class: 'lf-cap' }, ['Adding labeled examples to the prompt lifts accuracy fast at first, then flattens. The grey dot is zero-shot; the first few demonstrations close most of the gap, and beyond a handful each new example earns almost nothing while still costing tokens. The skill is picking the smallest set that reaches the plateau.']) + ])); + state._render(); + } + + // ── cot-decomposition: a hard problem split into reasoning steps, CoT lift ── + function cotDecomposition(host) { + var state = { cot: 'on' }; + var W = 520, H = 220, PAD = 18; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var STEPS = ['read', 'find rate', 'multiply', 'add tax', 'answer']; + function box(x, y, w, h, label, fill, tcol) { + var g = svgEl('g', {}); + g.appendChild(svgEl('rect', { x: x, y: y, width: w, height: h, rx: '4', fill: fill, stroke: 'var(--ink-soft,#555)', 'stroke-width': '1.2' })); + var t = svgEl('text', { x: (x + w / 2).toFixed(1), y: (y + h / 2 + 4).toFixed(1), 'text-anchor': 'middle', 'font-family': 'monospace', 'font-size': '11', fill: tcol }); + t.appendChild(document.createTextNode(label)); + g.appendChild(t); + return g; + } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var on = state.cot === 'on'; + var qY = 24, aY = H - 56, bh = 34; + svg.appendChild(box(PAD, qY, 150, bh, 'question', 'var(--bg,#fafaf5)', 'var(--ink,#1a1a1a)')); + svg.appendChild(box(W - PAD - 150, aY, 150, bh, 'answer', 'var(--blueprint,#3553ff)', 'var(--bg,#fafaf5)')); + if (on) { + var n = STEPS.length, midY = (qY + aY) / 2, bw = (W - 2 * PAD) / n - 8, sh = 28; + var i, prevX = PAD + 75, prevY = qY + bh; + for (i = 0; i < n; i++) { + var x = PAD + i * ((W - 2 * PAD) / n) + 4, cx = x + bw / 2; + svg.appendChild(svgEl('line', { x1: prevX, y1: prevY, x2: cx, y2: midY, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.4', opacity: '0.6' })); + svg.appendChild(box(x, midY - sh / 2, bw, sh, String(i + 1), 'var(--bg-surface,#eee)', 'var(--ink,#1a1a1a)')); + prevX = cx; prevY = midY + sh / 2; + } + svg.appendChild(svgEl('line', { x1: prevX, y1: prevY, x2: W - PAD - 75, y2: aY, stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.4', opacity: '0.6' })); + } else { + svg.appendChild(svgEl('line', { x1: PAD + 75, y1: qY + bh, x2: W - PAD - 75, y2: aY, stroke: 'var(--rule-soft,#ccc)', 'stroke-width': '1.4', 'stroke-dasharray': '4 3' })); + var qm = svgEl('text', { x: (W / 2).toFixed(1), y: ((qY + aY) / 2 + 4).toFixed(1), 'text-anchor': 'middle', 'font-family': 'monospace', 'font-size': '13', fill: 'var(--ink-mute,#777)' }); + qm.appendChild(document.createTextNode('(no shown work)')); + svg.appendChild(qm); + } + var accDirect = 0.38, accCoT = 0.71; + var a = on ? accCoT : accDirect; + num.innerHTML = (a * 100).toFixed(0) + ' % solved'; + meta.textContent = on ? 'chain-of-thought: 5 intermediate steps make each substep checkable · +' + ((accCoT - accDirect) * 100).toFixed(0) + ' pts' : 'direct answer: the model must do every step at once, in one jump'; + formula.textContent = on ? 'prompt += "let us reason step by step" → expose intermediate states' : 'answer = f(question) in a single forward pass'; + }; + var grid = el('div', {}, [LF.select(state, 'cot', 'reasoning', [['chain-of-thought on', 'on'], ['direct answer (off)', 'off']])]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['CHAIN OF THOUGHT']), el('span', {}, ['toggle showing the work'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula])]), + el('div', { class: 'lf-cap' }, ['A hard problem asked for a direct answer forces the model to compress every step into one jump, and it often slips. Chain-of-thought asks it to write the intermediate steps, turning one large leap into a chain of small, individually easy ones. Each visible substep is also a place the model can catch its own mistake.']) + ])); + state._render(); + } + + // ── constrained-decoding: a grammar mask greys out schema-invalid tokens ──── + function constrainedDecoding(host) { + var state = { step: 1 }; + var rows = el('div', {}); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // candidate next tokens at each grammar state; valid = allowed by JSON schema + var STATES = [ + { ctx: '', valid: ['{'], cands: ['{', 'hello', '42', '[', 'true'] }, + { ctx: '{', valid: ['"'], cands: ['"', '{', '42', 'name', ':'] }, + { ctx: '{ "', valid: ['name', 'age'], cands: ['name', 'age', '}', '123', ','] }, + { ctx: '{ "name"', valid: [':'], cands: [':', '"', '}', 'foo', '42'] }, + { ctx: '{ "name":', valid: ['"'], cands: ['"', '{', 'true', '42', ']'] } + ]; + state._render = function () { + var st = STATES[state.step]; + while (rows.firstChild) rows.removeChild(rows.firstChild); + var allowed = 0; + st.cands.forEach(function (tok) { + var ok = st.valid.indexOf(tok) >= 0; + if (ok) allowed++; + var bar = el('i'); bar.style.width = ok ? '100%' : '12%'; + if (!ok) bar.style.background = 'var(--rule-soft,#ccc)'; + var lab = el('label', {}, ['"' + tok + '"', el('b', {}, [ok ? 'allowed' : 'masked'])]); + if (!ok) lab.style.opacity = '0.4'; + rows.appendChild(el('div', { class: 'lf-ctrl' }, [lab, el('div', { class: 'lf-bar' }, [bar])])); + }); + num.innerHTML = allowed + ' of ' + st.cands.length + ' tokens valid'; + meta.textContent = 'so far: ' + (st.ctx || '(empty)') + ' · the grammar only permits tokens that keep the output schema-valid'; + formula.textContent = 'logits[invalid] = −∞ before softmax → the model can only sample a token the schema allows next'; + }; + var grid = el('div', {}, [slider(state, 'step', 'decoding position', 0, STATES.length - 1, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['CONSTRAINED DECODING']), el('span', {}, ['drag through the JSON'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [rows, num, meta, formula])]), + el('div', { class: 'lf-cap' }, ['Structured output enforces a grammar at decode time. At each position the schema computes which next tokens are legal, and every other token has its logit driven to negative infinity before sampling. The model still chooses, but only from the allowed set, so the result is always parseable JSON rather than free text that happens to look like JSON.']) + ])); + state._render(); + } + + // ── prompt-cache-hit: shared-prefix length and hit rate cut latency and cost ─ + function promptCacheHit(host) { + var state = { prefix: 70, hit: 80 }; + var W = 520, H = 120, PAD = 20; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var bar = el('i'); + var barWrap = el('div', { class: 'lf-bar' }, [bar]); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + var pf = state.prefix / 100, hr = state.hit / 100; + var saved = hr * pf; + while (svg.firstChild) svg.removeChild(svg.firstChild); + var inX = PAD, inW = W - 2 * PAD, y = 40, h = 40; + var pw = inW * pf; + svg.appendChild(svgEl('rect', { x: inX, y: y, width: pw.toFixed(1), height: h, rx: '3', fill: 'var(--bg-surface,#eee)', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '1.2', 'stroke-dasharray': '4 3' })); + svg.appendChild(svgEl('rect', { x: (inX + pw).toFixed(1), y: y, width: (inW - pw).toFixed(1), height: h, rx: '3', fill: 'var(--blueprint,#3553ff)', opacity: '0.85' })); + var t1 = svgEl('text', { x: (inX + pw / 2).toFixed(1), y: (y + h + 16).toFixed(1), 'text-anchor': 'middle', 'font-family': 'monospace', 'font-size': '10', fill: 'var(--ink-mute,#777)' }); + t1.appendChild(document.createTextNode('cached prefix ' + state.prefix + '%')); + svg.appendChild(t1); + var t2 = svgEl('text', { x: (inX + pw + (inW - pw) / 2).toFixed(1), y: (y + h + 16).toFixed(1), 'text-anchor': 'middle', 'font-family': 'monospace', 'font-size': '10', fill: 'var(--ink-soft,#555)' }); + t2.appendChild(document.createTextNode('new ' + (100 - state.prefix) + '%')); + svg.appendChild(t2); + num.innerHTML = (saved * 100).toFixed(1) + ' % prefill saved'; + bar.style.width = (saved * 100).toFixed(1) + '%'; + barWrap.classList.toggle('over', saved > 0.6); + meta.textContent = 'hit rate ' + state.hit + '% · cached prefix skips recompute · cost and time-to-first-token both drop by ~' + (saved * 100).toFixed(0) + '%'; + formula.textContent = 'saved = hit_rate × prefix_fraction = ' + hr.toFixed(2) + ' × ' + pf.toFixed(2) + ' = ' + saved.toFixed(2); + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'prefix', 'shared prefix (% of prompt)', 0, 100, 1), + slider(state, 'hit', 'cache hit rate (%)', 0, 100, 1) + ]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['PROMPT CACHE HIT']), el('span', {}, ['drag prefix and hit rate'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, num, barWrap, meta, formula])]), + el('div', { class: 'lf-cap' }, ['A long system prompt or document that repeats across calls can be prefilled once and reused. The cached prefix (dashed) skips recomputation, so only the new suffix (solid) is processed. The saving is the hit rate times the prefix fraction: a big shared prefix that is hit often is where caching pays off most.']) + ])); + state._render(); + } + + // ── semantic-cache: similarity threshold trades hit rate against safety ────── + function semanticCache(host) { + var state = { thr: 0.85 }; + var W = 520, H = 200, PAD = 26; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // deterministic incoming queries: cosine similarity to nearest cached entry, + // and whether the cached answer is actually correct for that query + var Q = [ + { sim: 0.97, ok: true }, { sim: 0.93, ok: true }, { sim: 0.90, ok: true }, + { sim: 0.88, ok: true }, { sim: 0.84, ok: false }, { sim: 0.80, ok: true }, + { sim: 0.76, ok: false }, { sim: 0.71, ok: false }, { sim: 0.62, ok: true }, + { sim: 0.50, ok: false } + ]; + function px(s) { return PAD + (s - 0.4) / 0.6 * (W - 2 * PAD); } + state._render = function () { + var thr = state.thr; + while (svg.firstChild) svg.removeChild(svg.firstChild); + var tx = px(thr); + svg.appendChild(svgEl('rect', { x: tx.toFixed(1), y: PAD, width: (W - PAD - tx).toFixed(1), height: (H - 2 * PAD).toFixed(1), fill: 'var(--blueprint,#3553ff)', opacity: '0.06' })); + svg.appendChild(svgEl('line', { x1: tx, y1: PAD - 4, x2: tx, y2: H - PAD, stroke: 'var(--warn,#b8870f)', 'stroke-width': '1.5' })); + var hits = 0, wrong = 0; + Q.forEach(function (q, i) { + var hit = q.sim >= thr; + if (hit) { hits++; if (!q.ok) wrong++; } + var cy = PAD + (i + 0.5) / Q.length * (H - 2 * PAD); + var col = !hit ? 'var(--rule-soft,#ccc)' : (q.ok ? 'var(--blueprint,#3553ff)' : 'var(--warn,#b8870f)'); + svg.appendChild(svgEl('circle', { cx: px(q.sim), cy: cy.toFixed(1), r: hit ? '5' : '3.5', fill: col })); + }); + num.innerHTML = hits + ' of ' + Q.length + ' served from cache'; + meta.textContent = 'threshold ' + thr.toFixed(2) + ' · ' + wrong + ' of those hits return a wrong answer · ' + (thr >= 0.9 ? 'high: few hits, safe' : thr <= 0.7 ? 'low: many hits, risky' : 'balanced'); + formula.textContent = 'serve cached answer when cos(query, cached) ≥ ' + thr.toFixed(2) + ' · lower threshold = more hits, more risk'; + }; + var grid = el('div', {}, [slider(state, 'thr', 'similarity threshold', 0.5, 0.99, 0.01)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['SEMANTIC CACHE']), el('span', {}, ['drag the threshold'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, num, meta, formula])]), + el('div', { class: 'lf-cap' }, ['A semantic cache answers a new query from a stored answer when their embeddings are close enough. Each dot is an incoming query placed by its cosine similarity to the nearest cached entry; the shaded zone past the orange line is served from cache. Raise the threshold and you get fewer hits but rarely serve a wrong answer; lower it and you cache more but risk returning a stale or mismatched reply (orange dots).']) + ])); + state._render(); + } + + // ── function-call-args: model picks a tool then fills typed JSON slots ─────── + function functionCallArgs(host) { + var state = { step: 2 }; + var W = 520, H = 200, PAD = 18; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var code = el('div', { class: 'lf-formula' }); + var meta = el('div', { class: 'lf-meta' }); + // schema slots filled progressively from the user request + var SLOTS = [ + { name: 'tool', type: 'enum', val: 'book_flight' }, + { name: 'from', type: 'string', val: '"BOM"' }, + { name: 'to', type: 'string', val: '"SFO"' }, + { name: 'date', type: 'date', val: '"2026-07-01"' }, + { name: 'pax', type: 'int', val: '2' } + ]; + var REQUEST = 'book 2 seats Mumbai to San Francisco on Jul 1'; + function box(x, y, w, h, label, active) { + var g = svgEl('g', {}); + g.appendChild(svgEl('rect', { x: x, y: y, width: w, height: h, rx: '3', fill: active ? 'var(--blueprint,#3553ff)' : 'var(--bg-surface,#eee)', stroke: 'var(--ink-soft,#555)', 'stroke-width': '1.1' })); + var t = svgEl('text', { x: (x + 8).toFixed(1), y: (y + h / 2 + 4).toFixed(1), 'font-family': 'monospace', 'font-size': '10.5', fill: active ? 'var(--bg,#fafaf5)' : 'var(--ink,#1a1a1a)' }); + t.appendChild(document.createTextNode(label)); + g.appendChild(t); + return g; + } + state._render = function () { + var s = state.step; + while (svg.firstChild) svg.removeChild(svg.firstChild); + var rq = svgEl('text', { x: PAD, y: 22, 'font-family': 'monospace', 'font-size': '10.5', fill: 'var(--ink-soft,#555)' }); + rq.appendChild(document.createTextNode('request: ' + REQUEST)); + svg.appendChild(rq); + var y0 = 36, rh = 28, w = W - 2 * PAD; + SLOTS.forEach(function (slot, i) { + var filled = i <= s; + var y = y0 + i * (rh + 2); + var label = slot.name + ' : ' + slot.type + (filled ? ' = ' + slot.val : ' = ?'); + svg.appendChild(box(PAD, y, w, rh, label, filled)); + }); + code.textContent = '{ ' + SLOTS.slice(0, s + 1).map(function (sl) { return '"' + sl.name + '": ' + sl.val; }).join(', ') + (s < SLOTS.length - 1 ? ', ... ' : ' ') + '}'; + meta.textContent = (s + 1) + ' of ' + SLOTS.length + ' slots filled · the model selects the tool, then extracts each typed argument from the request'; + }; + var grid = el('div', {}, [slider(state, 'step', 'arguments filled', 0, SLOTS.length - 1, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['FUNCTION CALL ARGS']), el('span', {}, ['drag to fill the schema'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, code, meta])]), + el('div', { class: 'lf-cap' }, ['Function calling is two jobs in one. First the model selects which tool to invoke from the available set, then it populates that tool\'s typed argument schema by extracting values from the user request. The output is not prose but a structured call: a tool name plus a JSON object whose fields match the declared types, ready to execute.']) + ])); + state._render(); + } + + // ── llm-judge-rubric: weighted rubric score, LLM-as-judge aggregation ─────── + function llmJudgeRubric(host) { + var state = { wHelp: 40, wCorr: 40, wSafe: 20 }; + var rows = el('div', {}); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // judge's per-criterion scores for one response, on a 0..1 scale + var CRIT = [ + { key: 'wHelp', label: 'helpfulness', score: 0.85 }, + { key: 'wCorr', label: 'correctness', score: 0.60 }, + { key: 'wSafe', label: 'safety', score: 0.95 } + ]; + state._render = function () { + var wsum = state.wHelp + state.wCorr + state.wSafe; + if (wsum <= 0) wsum = 1; + while (rows.firstChild) rows.removeChild(rows.firstChild); + var total = 0; + CRIT.forEach(function (c) { + var w = state[c.key] / wsum; + total += w * c.score; + var bar = el('i'); bar.style.width = (c.score * 100).toFixed(0) + '%'; + rows.appendChild(el('div', { class: 'lf-ctrl' }, [ + el('label', {}, [c.label + ' (w ' + (w * 100).toFixed(0) + '%)', el('b', {}, [c.score.toFixed(2)])]), + el('div', { class: 'lf-bar' }, [bar]) + ])); + }); + num.innerHTML = total.toFixed(3) + ' weighted score'; + meta.textContent = 'weights normalized to sum 1 · shift weight onto correctness and this response drops, since it scored lowest there'; + formula.textContent = 'score = Σ (wᵢ / Σw) · sᵢ · s = [0.85, 0.60, 0.95] · the judge rates each criterion, the rubric aggregates'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'wHelp', 'helpfulness weight', 0, 100, 1), + slider(state, 'wCorr', 'correctness weight', 0, 100, 1), + slider(state, 'wSafe', 'safety weight', 0, 100, 1) + ]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['LLM-AS-JUDGE RUBRIC']), el('span', {}, ['drag the rubric weights'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [rows, num, meta, formula])]), + el('div', { class: 'lf-cap' }, ['An LLM judge rates a response on several criteria, here helpfulness, correctness, and safety. The final score is a weighted average, and the weights encode what you actually care about. This response is strong on safety but weak on correctness, so leaning the rubric toward correctness pulls its score down. The weights, not the model, decide what good means.']) + ])); + state._render(); + } + + // ── lost-in-the-middle: retrieval accuracy is U-shaped over fact position ──── + function lostInTheMiddle(host) { + var state = { pos: 50 }; + var W = 520, H = 220, PAD = 32; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // U-shape: high at the ends (recency + primacy), dips in the middle + function acc(p) { var x = p / 100; var u = 0.40 + 0.55 * Math.pow(2 * x - 1, 2); return clamp(u, 0, 1); } + function px(p) { return PAD + p / 100 * (W - 2 * PAD); } + function py(a) { return H - PAD - (a - 0.3) / 0.7 * (H - 2 * PAD); } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var d = '', i; + for (i = 0; i <= 120; i++) { var p = 100 * i / 120; d += (i ? 'L' : 'M') + px(p).toFixed(1) + ' ' + py(acc(p)).toFixed(1) + ' '; } + svg.appendChild(svgEl('path', { d: d, fill: 'none', stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2' })); + var s = svgEl('text', { x: PAD, y: H - 8, 'font-family': 'monospace', 'font-size': '10', fill: 'var(--ink-mute,#777)' }); + s.appendChild(document.createTextNode('start of context')); + svg.appendChild(s); + var e = svgEl('text', { x: (W - PAD).toFixed(1), y: H - 8, 'text-anchor': 'end', 'font-family': 'monospace', 'font-size': '10', fill: 'var(--ink-mute,#777)' }); + e.appendChild(document.createTextNode('end of context')); + svg.appendChild(e); + svg.appendChild(svgEl('circle', { cx: px(state.pos), cy: py(acc(state.pos)), r: '5', fill: 'var(--blueprint,#3553ff)' })); + var a = acc(state.pos); + num.innerHTML = (a * 100).toFixed(0) + ' % retrieval accuracy'; + meta.textContent = 'fact at ' + state.pos + '% of context · ' + (state.pos < 20 || state.pos > 80 ? 'near an edge: easy to recall' : 'buried in the middle: easy to miss'); + formula.textContent = 'accuracy(pos) ≈ 0.40 + 0.55·(2·pos − 1)² · U-shaped: primacy + recency, dip in the middle'; + }; + var grid = el('div', {}, [slider(state, 'pos', 'key fact position (% of context)', 0, 100, 1)]); + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, ['LOST IN THE MIDDLE']), el('span', {}, ['drag the fact position'])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula])]), + el('div', { class: 'lf-cap' }, ['When a single relevant fact is buried in a long context, the model recalls it well if it sits near the start or the end, and far worse if it lands in the middle. Accuracy is U-shaped over position. This is why context engineering puts the most important material at the top or bottom of the prompt and never relies on a long unstructured dump.']) + ])); + state._render(); + } + + LF.register({ + 'few-shot-curve': fewShotCurve, + 'cot-decomposition': cotDecomposition, + 'constrained-decoding': constrainedDecoding, + 'prompt-cache-hit': promptCacheHit, + 'semantic-cache': semanticCache, + 'function-call-args': functionCallArgs, + 'llm-judge-rubric': llmJudgeRubric, + 'lost-in-the-middle': lostInTheMiddle + }); +})(); diff --git a/site/figures-multimodal.js b/site/figures-multimodal.js new file mode 100644 index 0000000000..777fdcdbd0 --- /dev/null +++ b/site/figures-multimodal.js @@ -0,0 +1,397 @@ +/* figures-multimodal.js — interactive lesson figures for Phase 12 (multimodal + AI). Loads after lesson-figures.js, uses the shared LF toolkit, registers via + LF.register. No deps, ES5 only, theme via CSS vars. Authoring is the same + fenced ```figure block in docs/en.md. */ +(function () { + 'use strict'; + var LF = window.LF; + if (!LF) { return; } + var el = LF.el, svgEl = LF.svgEl, slider = LF.slider, clamp = LF.clamp, fmtInt = LF.fmtInt; + + function shell(host, label, hint, grid, outKids, cap) { + host.appendChild(el('div', { class: 'lf' }, [ + el('div', { class: 'lf-head' }, [el('span', { class: 'lf-label' }, [label]), el('span', {}, [hint])]), + el('div', { class: 'lf-body' }, [grid, el('div', { class: 'lf-out' }, outKids)]), + el('div', { class: 'lf-cap' }, [cap]) + ])); + } + + function txt(x, y, s, anchor, color, size) { + return svgEl('text', { x: String(x), y: String(y), 'text-anchor': anchor || 'middle', 'font-size': String(size || 10), 'font-family': 'monospace', fill: color || 'var(--ink-soft,#555)' }, [document.createTextNode(s)]); + } + + // ── contrastive-matrix: CLIP InfoNCE similarity matrix, drag temperature ──── + function contrastiveMatrix(host) { + var n = 5; + var labels = ['dog', 'car', 'tree', 'boat', 'bird']; + // Fixed cosine similarities in [-1,1]; diagonal high, off-diagonal lower. + var sim = [ + [0.92, 0.18, 0.24, 0.10, 0.30], + [0.15, 0.90, 0.12, 0.40, 0.08], + [0.27, 0.10, 0.88, 0.14, 0.35], + [0.12, 0.42, 0.16, 0.91, 0.06], + [0.33, 0.09, 0.38, 0.07, 0.89] + ]; + var state = { tau: 0.10 }; + var W = 520, H = 280, PAD = 70, CELL = (W - PAD - 20) / n; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var tau = Math.max(0.01, state.tau); + var r, c, x, y, diag = 0; + for (r = 0; r < n; r++) { + // Softmax over the row of image r against all texts (InfoNCE numerator). + var sc = []; + for (c = 0; c < n; c++) { sc.push(sim[r][c] / tau); } + var mx = Math.max.apply(null, sc); + var ex = sc.map(function (s) { return Math.exp(s - mx); }); + var sum = ex.reduce(function (a, b) { return a + b; }, 0); + var probs = ex.map(function (e) { return e / sum; }); + diag += probs[r]; + for (c = 0; c < n; c++) { + x = PAD + c * CELL; y = 34 + r * CELL; + svg.appendChild(svgEl('rect', { x: x.toFixed(1), y: y.toFixed(1), width: (CELL - 2).toFixed(1), height: (CELL - 2).toFixed(1), fill: 'var(--blueprint,#3553ff)', 'fill-opacity': probs[c].toFixed(3), stroke: c === r ? 'var(--warn,#b8870f)' : 'var(--rule-soft,#ddd)', 'stroke-width': c === r ? '1.5' : '0.5' })); + } + svg.appendChild(txt((PAD - 8).toFixed(1), (y + CELL / 2 + 3).toFixed(1), 'img:' + labels[r], 'end', 'var(--ink-soft,#555)')); + } + for (c = 0; c < n; c++) { + x = PAD + c * CELL; + svg.appendChild(txt((x + CELL / 2).toFixed(1), '26', 'txt:' + labels[c], 'middle', 'var(--ink-mute,#777)')); + } + var acc = diag / n; + meta.textContent = 'matched-pair mass ' + (acc * 100).toFixed(0) + '% · diagonal outlined · ' + (state.tau < 0.06 ? 'low τ: matrix sharpens to a clear diagonal' : state.tau > 0.25 ? 'high τ: rows flatten, pairs blur together' : 'balanced'); + formula.textContent = 'L = −log softmax(sim / τ)[matched], τ = ' + tau.toFixed(2) + ' · each image-row and text-column softmaxes to the matched pair'; + }; + var grid = el('div', {}, [slider(state, 'tau', 'temperature τ', 0.02, 0.5, 0.01)]); + shell(host, 'CONTRASTIVE MATRIX', 'drag τ', grid, [svg, meta, formula], + 'CLIP scores every image against every caption in the batch, forming a similarity matrix. The contrastive loss pulls the matched diagonal pairs together and pushes the off-diagonal apart. Dividing by a small temperature sharpens the softmax so the diagonal lights up; a large temperature flattens it and the model stops distinguishing pairs.'); + state._render(); + } + + // ── cross-attention-fusion: text queries attend to image patch keys ───────── + function crossAttentionFusion(host) { + var texts = ['a', 'red', 'bird', 'on', 'branch']; + var patches = 8; + var nt = texts.length; + var state = { focus: 2, sharp: 1.4 }; + // Fixed affinity of each text token to each image patch (bird at patch 4-5). + var aff = [ + [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], + [0.2, 0.3, 0.5, 0.7, 0.8, 0.6, 0.3, 0.2], + [0.1, 0.2, 0.4, 0.8, 1.0, 0.9, 0.4, 0.2], + [0.4, 0.4, 0.3, 0.3, 0.3, 0.4, 0.5, 0.5], + [0.3, 0.2, 0.2, 0.3, 0.4, 0.6, 0.9, 1.0] + ]; + var W = 520, H = 250, PAD = 64, CELL = (W - PAD - 20) / patches; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var s = Math.max(0.1, state.sharp), r, c, x, y, peak = 0, peakCol = 0; + var rowH = (H - 50) / nt; + for (r = 0; r < nt; r++) { + var logits = []; + for (c = 0; c < patches; c++) { logits.push(aff[r][c] * s); } + var mx = Math.max.apply(null, logits); + var ex = logits.map(function (z) { return Math.exp(z - mx); }); + var sum = ex.reduce(function (a, b) { return a + b; }, 0); + var probs = ex.map(function (e) { return e / sum; }); + for (c = 0; c < patches; c++) { + x = PAD + c * CELL; y = 30 + r * rowH; + svg.appendChild(svgEl('rect', { x: x.toFixed(1), y: y.toFixed(1), width: (CELL - 2).toFixed(1), height: (rowH - 3).toFixed(1), fill: 'var(--blueprint,#3553ff)', 'fill-opacity': probs[c].toFixed(3), stroke: r === state.focus && c === probs.indexOf(Math.max.apply(null, probs)) ? 'var(--warn,#b8870f)' : 'var(--rule-soft,#ddd)', 'stroke-width': '0.5' })); + } + if (r === state.focus) { peak = Math.max.apply(null, probs); peakCol = probs.indexOf(peak); } + svg.appendChild(txt((PAD - 8).toFixed(1), (y + rowH / 2 + 3).toFixed(1), texts[r], 'end', r === state.focus ? 'var(--blueprint,#3553ff)' : 'var(--ink-soft,#555)')); + } + for (c = 0; c < patches; c++) { + x = PAD + c * CELL; + svg.appendChild(txt((x + CELL / 2).toFixed(1), '24', 'p' + c, 'middle', 'var(--ink-mute,#777)', 9)); + } + meta.textContent = '"' + texts[state.focus] + '" puts ' + (peak * 100).toFixed(0) + '% of its attention on patch ' + peakCol + ' · rows = text queries, columns = image patches'; + formula.textContent = 'A = softmax(Q_text · Kᵀ_image), rows sum to 1 · higher sharpness peaks each query onto its patch'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'focus', 'highlighted text token', 0, nt - 1, 1), + slider(state, 'sharp', 'attention sharpness', 0.3, 4.0, 0.1) + ]); + shell(host, 'CROSS-ATTENTION FUSION', 'drag the query and sharpness', grid, [svg, meta, formula], + 'In a vision-language model, each text token is a query that attends across the image patch keys. The grid is one attention map: rows are text tokens, columns are image patches, and each row softmaxes to one. The content words like "bird" concentrate onto the patches that contain the object, which is how language grounds itself in pixels.'); + state._render(); + } + + // ── modality-projection: align image and text vectors in a shared space ───── + function modalityProjection(host) { + var state = { align: 0 }; + var W = 360, H = 260, CX = 70, CY = 150, R = 120; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + var imgDeg0 = 78, txtDeg0 = 14; // misaligned to start + function vec(deg, color, label) { + var rad = deg * Math.PI / 180; + var x2 = CX + R * Math.cos(rad), y2 = CY - R * Math.sin(rad); + svg.appendChild(svgEl('line', { x1: CX, y1: CY, x2: x2.toFixed(1), y2: y2.toFixed(1), stroke: color, 'stroke-width': '2.5' })); + svg.appendChild(txt((x2 + 6).toFixed(1), (y2 - 2).toFixed(1), label, 'start', color, 11)); + } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var t = clamp(state.align, 0, 1); + // Both vectors converge toward a common 45-degree direction as t -> 1. + var target = 46; + var imgDeg = imgDeg0 + t * (target - imgDeg0); + var txtDeg = txtDeg0 + t * (target - txtDeg0); + svg.appendChild(svgEl('path', { d: 'M ' + CX + ' ' + CY + ' L ' + (CX + R) + ' ' + CY, stroke: 'var(--rule-soft,#eee)', 'stroke-width': '1' })); + vec(imgDeg, 'var(--blueprint,#3553ff)', 'image'); + vec(txtDeg, 'var(--warn,#b8870f)', 'text'); + var cos = Math.cos((imgDeg - txtDeg) * Math.PI / 180); + num.innerHTML = cos.toFixed(3) + ' cosine'; + meta.textContent = 'angle between projections ' + Math.abs(imgDeg - txtDeg).toFixed(0) + '° · ' + (cos > 0.97 ? 'aligned: the matched pair points the same way' : cos > 0.6 ? 'partly aligned' : 'misaligned: separate subspaces'); + formula.textContent = 'enc_img(x) → ℝ^d ← enc_txt(y), train to maximize cos(z_img, z_txt) for matched pairs'; + }; + var grid = el('div', {}, [slider(state, 'align', 'projection training progress', 0, 1, 0.02)]); + shell(host, 'MODALITY PROJECTION', 'drag to align the pair', grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'A separate image encoder and text encoder land in their own spaces, so a matched pair starts pointing in different directions. A learned projection maps both into one shared d-dimensional space, and training rotates the matched vectors together until their cosine approaches one. Once aligned, a single distance compares across modalities.'); + state._render(); + } + + // ── cfg-guidance-scale: guided = uncond + w (cond - uncond) ───────────────── + function cfgGuidanceScale(host) { + var state = { w: 3.0 }; + var W = 520, H = 240, PAD = 40; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // 1D illustration: prediction is a point on a number line; vectors add. + var uncond = 1.4, cond = 3.6; // base predictions (e.g. denoised estimate) + var XMIN = 0, XMAX = 9; + function px(v) { return PAD + (v - XMIN) / (XMAX - XMIN) * (W - 2 * PAD); } + var axisY = 120; + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var w = state.w; + var guided = uncond + w * (cond - uncond); + var gClamp = clamp(guided, XMIN, XMAX); + svg.appendChild(svgEl('line', { x1: PAD, y1: axisY, x2: W - PAD, y2: axisY, stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1' })); + function tick(v, color, label, dy) { + svg.appendChild(svgEl('circle', { cx: px(v).toFixed(1), cy: String(axisY), r: '5', fill: color })); + svg.appendChild(txt(px(v).toFixed(1), String(axisY + dy), label, 'middle', color, 10)); + } + // arrow from uncond toward cond direction, scaled by w + svg.appendChild(svgEl('line', { x1: px(uncond).toFixed(1), y1: String(axisY - 26), x2: px(gClamp).toFixed(1), y2: String(axisY - 26), stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '2' })); + svg.appendChild(txt(px((uncond + gClamp) / 2).toFixed(1), String(axisY - 34), 'w · (cond − uncond)', 'middle', 'var(--blueprint,#3553ff)', 10)); + tick(uncond, 'var(--ink-mute,#999)', 'uncond', 22); + tick(cond, 'var(--warn,#b8870f)', 'cond', 38); + tick(gClamp, 'var(--blueprint,#3553ff)', 'guided', 54); + // diversity / sharpness bars + var diversity = clamp(1 / (1 + 0.5 * w), 0, 1); + var sharp = clamp(w / 12, 0, 1); + svg.appendChild(txt(PAD.toFixed(1), '200', 'diversity', 'start', 'var(--ink-soft,#555)', 10)); + svg.appendChild(svgEl('rect', { x: String(PAD), y: '204', width: (diversity * 180).toFixed(1), height: '8', fill: 'var(--ink-mute,#999)' })); + svg.appendChild(txt((W / 2 + 20).toFixed(1), '200', 'prompt adherence', 'start', 'var(--ink-soft,#555)', 10)); + svg.appendChild(svgEl('rect', { x: String(W / 2 + 20), y: '204', width: (sharp * 180).toFixed(1), height: '8', fill: 'var(--blueprint,#3553ff)' })); + num.innerHTML = 'w = ' + w.toFixed(1); + meta.textContent = w <= 1.05 ? 'w ≈ 1: prediction sits near unconditional, diverse but loosely on-prompt' + : w >= 9 ? 'w very high: saturated, sharp but less diverse and prone to artifacts' + : 'guided estimate pushed ' + ((guided - uncond)).toFixed(1) + ' past unconditional toward the prompt'; + formula.textContent = 'ε_guided = ε_uncond + w · (ε_cond − ε_uncond), w = ' + w.toFixed(1) + ' · w=1 is plain conditional, larger w over-extrapolates'; + }; + var grid = el('div', {}, [slider(state, 'w', 'guidance scale w', 1.0, 12.0, 0.1)]); + shell(host, 'CLASSIFIER-FREE GUIDANCE', 'drag w', grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'Classifier-free guidance runs the diffusion model twice, once with the prompt and once without, then extrapolates along the difference. A scale of one is the plain conditional prediction; higher scales push further toward the prompt, trading diversity for adherence. Push too far and the sample saturates and breaks, so practical scales sit in a middle band.'); + state._render(); + } + + // ── vq-codebook: continuous encoder outputs snap to nearest code ──────────── + function vqCodebook(host) { + var state = { logK: 4 }; + var W = 520, H = 240, PAD = 34; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + // Fixed set of continuous encoder outputs in 2D; deterministic. + var enc = [ + [0.12, 0.18], [0.22, 0.74], [0.55, 0.30], [0.78, 0.62], + [0.40, 0.88], [0.66, 0.12], [0.88, 0.40], [0.32, 0.46], + [0.50, 0.66], [0.14, 0.92], [0.92, 0.84], [0.70, 0.92] + ]; + function px(x) { return PAD + x * (W - 2 * PAD); } + function py(y) { return H - PAD - y * (H - 2 * PAD); } + function codebook(K) { + // Deterministic grid of codes covering the unit square. + var side = Math.max(1, Math.round(Math.sqrt(K))); + var pts = [], i, j; + for (i = 0; i < side; i++) { + for (j = 0; j < side; j++) { + if (pts.length >= K) { break; } + pts.push([(i + 0.5) / side, (j + 0.5) / side]); + } + } + return pts; + } + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var K = Math.round(Math.pow(2, state.logK)); + var codes = codebook(K); + var used = {}, totErr = 0; + // draw codebook vectors + codes.forEach(function (c) { + svg.appendChild(svgEl('rect', { x: (px(c[0]) - 4).toFixed(1), y: (py(c[1]) - 4).toFixed(1), width: '8', height: '8', fill: 'none', stroke: 'var(--ink-mute,#999)', 'stroke-width': '1' })); + }); + enc.forEach(function (e) { + // nearest code (quantization) + var best = 0, bd = 1e9, k; + for (k = 0; k < codes.length; k++) { + var dx = e[0] - codes[k][0], dy = e[1] - codes[k][1]; + var d = dx * dx + dy * dy; + if (d < bd) { bd = d; best = k; } + } + used[best] = 1; totErr += Math.sqrt(bd); + svg.appendChild(svgEl('line', { x1: px(e[0]).toFixed(1), y1: py(e[1]).toFixed(1), x2: px(codes[best][0]).toFixed(1), y2: py(codes[best][1]).toFixed(1), stroke: 'var(--rule-soft,#ccc)', 'stroke-width': '1' })); + svg.appendChild(svgEl('circle', { cx: px(e[0]).toFixed(1), cy: py(e[1]).toFixed(1), r: '3.5', fill: 'var(--blueprint,#3553ff)' })); + svg.appendChild(svgEl('rect', { x: (px(codes[best][0]) - 3).toFixed(1), y: (py(codes[best][1]) - 3).toFixed(1), width: '6', height: '6', fill: 'var(--warn,#b8870f)' })); + }); + var usage = Object.keys(used).length; + var avgErr = totErr / enc.length; + num.innerHTML = K + ' codes'; + meta.textContent = usage + ' of ' + K + ' codes used by ' + enc.length + ' vectors · avg quantization error ' + avgErr.toFixed(3) + ' · bits/token ' + Math.log2(K).toFixed(0); + formula.textContent = 'z_q = argmin_k ‖z_e − e_k‖, codebook size K = ' + K + ' · larger K → lower error but more codes to learn'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'logK', 'codebook size (2^x)', 2, 8, 1, function (v) { return String(Math.round(Math.pow(2, v))); }) + ]); + shell(host, 'VQ CODEBOOK', 'drag the codebook size', grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'A VQ-VAE encoder produces continuous vectors (blue dots), but the model needs discrete tokens. Each vector snaps to its nearest codebook entry (orange square), turning the image into a sequence of integer codes. A bigger codebook quantizes more finely, lowering reconstruction error, but spends more bits per token and risks codes that never get used.'); + state._render(); + } + + // ── video-temporal-patches: tokens = frames × (H/p)(W/p) ──────────────────── + function videoTemporalPatches(host) { + var state = { frames: 8, patch: 16, tubelet: 2 }; + var GRID = 224; // assume 224x224 frames + var W = 520, H = 230, PAD = 30; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var p = state.patch, F = state.frames, tub = state.tubelet; + var perSide = Math.floor(GRID / p); + var spatial = perSide * perSide; + var temporal = Math.max(1, Math.floor(F / tub)); + var tokens = spatial * temporal; + // draw a single representative frame grid + stack indicator + var face = 120, ox = 40, oy = 36, depth = 5; + var stack = Math.min(temporal, 6); + var s; + for (s = stack - 1; s >= 0; s--) { + var sx = ox + s * depth * 4, sy = oy + s * depth * 2; + svg.appendChild(svgEl('rect', { x: sx.toFixed(1), y: sy.toFixed(1), width: String(face), height: String(face), fill: s === 0 ? 'var(--bg-surface,#eee)' : 'var(--bg,#fafaf5)', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '1', 'fill-opacity': (1 - s * 0.12).toFixed(2) })); + } + // patch grid on the front frame + var i, j; + for (i = 0; i <= perSide; i++) { + svg.appendChild(svgEl('line', { x1: (ox + i * face / perSide).toFixed(1), y1: String(oy), x2: (ox + i * face / perSide).toFixed(1), y2: String(oy + face), stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '0.6', 'stroke-opacity': '0.55' })); + svg.appendChild(svgEl('line', { x1: String(ox), y1: (oy + i * face / perSide).toFixed(1), x2: String(ox + face), y2: (oy + i * face / perSide).toFixed(1), stroke: 'var(--blueprint,#3553ff)', 'stroke-width': '0.6', 'stroke-opacity': '0.55' })); + } + svg.appendChild(txt((ox + face / 2).toFixed(1), String(oy + face + 18), perSide + ' × ' + perSide + ' patches/frame', 'middle', 'var(--ink-soft,#555)', 10)); + // readout on the right + svg.appendChild(txt('300', '60', F + ' frames', 'start', 'var(--ink-soft,#555)', 11)); + svg.appendChild(txt('300', '82', '÷ ' + tub + ' (tubelet) = ' + temporal + ' temporal', 'start', 'var(--ink-mute,#777)', 10)); + svg.appendChild(txt('300', '108', spatial + ' spatial patches', 'start', 'var(--ink-soft,#555)', 11)); + svg.appendChild(txt('300', '134', temporal + ' × ' + spatial + ' =', 'start', 'var(--ink-mute,#777)', 11)); + svg.appendChild(txt('300', '160', fmtInt(tokens) + ' tokens', 'start', 'var(--blueprint,#3553ff)', 15)); + num.innerHTML = fmtInt(tokens) + ' tokens'; + meta.textContent = F + ' frames · ' + GRID + '² px · patch ' + p + ' · tubelet ' + tub + ' · ' + spatial + ' spatial × ' + temporal + ' temporal'; + formula.textContent = 'tokens = ⌊frames / tubelet⌋ · (H/p)·(W/p) = ' + temporal + ' · ' + spatial + ' = ' + fmtInt(tokens) + ' · token count drives the attention cost'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'frames', 'frames', 1, 64, 1), + slider(state, 'patch', 'patch size (px)', 8, 56, 4), + slider(state, 'tubelet', 'tubelet (frames/token)', 1, 8, 1) + ]); + shell(host, 'VIDEO TEMPORAL PATCHES', 'drag frames and patch size', grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'Video tokenizes in space and time at once. Each 224-pixel frame splits into a grid of spatial patches, and frames group into tubelets along time. The total token count is the temporal count times the spatial count, and since attention cost grows with the square of tokens, long clips at fine patches explode the budget. Larger patches and tubelets are the main levers to keep it tractable.'); + state._render(); + } + + // ── audio-text-ctc: monotonic alignment, blanks collapse to shorter text ──── + function audioTextCtc(host) { + var state = { frames: 12, dup: 1 }; + var target = ['C', 'A', 'T']; + var W = 520, H = 240, PAD = 30; + var svg = svgEl('svg', { viewBox: '0 0 ' + W + ' ' + H }); + var num = el('span', { class: 'lf-num' }); + var meta = el('div', { class: 'lf-meta' }); + var formula = el('div', { class: 'lf-formula' }); + state._render = function () { + while (svg.firstChild) svg.removeChild(svg.firstChild); + var T = state.frames, dup = state.dup; + // Build a deterministic monotonic emission: distribute target letters + // across frames with repeats (dup) and blanks filling the rest. + var emit = []; + var spoken = target.length * dup; + // place letters in the centre region, blanks at the ends and between. + var lead = Math.max(0, Math.floor((T - spoken) / 2)); + var f, idx = 0; + for (f = 0; f < T; f++) { + if (f >= lead && idx < spoken) { + emit.push(target[Math.floor(idx / dup)]); + idx++; + } else { + emit.push('_'); // blank + } + } + // CTC collapse: remove repeats then remove blanks. + var collapsed = [], prev = null, k; + for (k = 0; k < emit.length; k++) { + if (emit[k] !== prev) { if (emit[k] !== '_') { collapsed.push(emit[k]); } } + prev = emit[k]; + } + var cellW = (W - 2 * PAD) / T; + // top row: audio frames with emitted symbol + for (f = 0; f < T; f++) { + var x = PAD + f * cellW; + var isBlank = emit[f] === '_'; + svg.appendChild(svgEl('rect', { x: (x + 1).toFixed(1), y: '40', width: (cellW - 2).toFixed(1), height: '34', fill: isBlank ? 'var(--bg-surface,#eee)' : 'var(--blueprint,#3553ff)', 'fill-opacity': isBlank ? '0.5' : '0.8', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '0.5' })); + svg.appendChild(txt((x + cellW / 2).toFixed(1), '62', emit[f] === '_' ? '∅' : emit[f], 'middle', isBlank ? 'var(--ink-mute,#999)' : 'var(--bg,#fafaf5)', 12)); + } + svg.appendChild(txt(PAD.toFixed(1), '32', T + ' audio frames (∅ = blank)', 'start', 'var(--ink-mute,#777)', 10)); + // alignment path arrows down to collapsed text + var ty = 150, tStep = (W - 2 * PAD) / Math.max(1, target.length); + for (k = 0; k < target.length; k++) { + var tx = PAD + (k + 0.5) * tStep; + svg.appendChild(svgEl('rect', { x: (tx - 16).toFixed(1), y: ty.toFixed(1), width: '32', height: '30', fill: 'var(--warn,#b8870f)', 'fill-opacity': '0.75', stroke: 'var(--rule-soft,#ddd)', 'stroke-width': '0.5' })); + svg.appendChild(txt(tx.toFixed(1), (ty + 20).toFixed(1), target[k], 'middle', 'var(--bg,#fafaf5)', 13)); + } + svg.appendChild(txt(PAD.toFixed(1), (ty - 8).toFixed(1), 'collapse repeats, drop blanks →', 'start', 'var(--ink-mute,#777)', 10)); + var ok = collapsed.join('') === target.join(''); + num.innerHTML = T + ' → ' + collapsed.length + ' frames → chars'; + meta.textContent = 'emitted "' + emit.join('') + '" collapses to "' + collapsed.join('') + '" · ' + (ok ? 'matches target CAT' : 'does not yet spell CAT'); + formula.textContent = 'CTC: many alignments map to one label · collapse(a a _ b) = a b · blanks separate repeats so "AA" survives'; + }; + var grid = el('div', { class: 'lf-grid' }, [ + slider(state, 'frames', 'audio frames', 4, 24, 1), + slider(state, 'dup', 'frames per letter', 1, 4, 1) + ]); + shell(host, 'AUDIO-TEXT CTC', 'drag frames and duration', grid, [svg, el('div', { style: 'margin-top:10px' }, [num]), meta, formula], + 'Audio is a long frame sequence; the transcript is short. CTC lets the model emit a label or a blank at every frame, then collapses the output by merging repeats and dropping blanks. The alignment stays monotonic, time only moves forward, and the blank token is what keeps a genuine double letter from collapsing into one. Many frame-level alignments map onto the same final text.'); + state._render(); + } + + LF.register({ + 'contrastive-matrix': contrastiveMatrix, + 'cross-attention-fusion': crossAttentionFusion, + 'modality-projection': modalityProjection, + 'cfg-guidance-scale': cfgGuidanceScale, + 'vq-codebook': vqCodebook, + 'video-temporal-patches': videoTemporalPatches, + 'audio-text-ctc': audioTextCtc + }); +})(); diff --git a/site/lesson.html b/site/lesson.html index 5782c53b9b..080ef02b9a 100644 --- a/site/lesson.html +++ b/site/lesson.html @@ -1630,21 +1630,26 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + +