Add receipt image evaluation cookbook app

.agents/skills/check-models/scripts/models.json

@@ -1,18 +1,20 @@
 {
   "_comment": "Single source of truth for the check-models skill. Update the numbers in `openai` and `anthropic` whenever a new model ships, then re-run the scanner. The scanner derives the full replacement table from these policy values — you should not need to enumerate every old ID.",
-  "updated": "2026-06-18",
-  "verifiedBy": "WebFetch developers.openai.com/api/docs/models + /deprecations (2026-06-18); Google via ai.google.dev/gemini-api/docs/models (2026-06-18); Anthropic via claude-api skill shared/models.md",
+  "updated": "2026-08-28",
+  "verifiedBy": "OpenAI Models documentation (2026-08-28); Google via ai.google.dev/gemini-api/docs/models (2026-06-18); Anthropic via claude-api skill shared/models.md",
 
   "openai": {
-    "flagship": "gpt-5.5",
+    "flagship": "gpt-5.6-sol",
+    "terra": "gpt-5.6-terra",
+    "luna": "gpt-5.6-luna",
     "pro": "gpt-5.5-pro",
     "mini": "gpt-5.4-mini",
     "nano": "gpt-5.4-nano",
     "flagshipMinVersion": 5.4,
     "proMinVersion": 5.5,
     "miniMinVersion": 5.4,
     "nanoMinVersion": 5.4,
-    "notes": "GPT-5.5 is the latest flagship (released 2026-04-23, API id gpt-5.5-2026-04-23) and has no mini/nano variant — mini/nano stay on the 5.4 generation. GPT-5.4 also remains available as a cheaper flagship (still current, NOT deprecated per OpenAI's models page), so flagshipMinVersion is 5.4 — only gpt-5.3 and older flag for migration; gpt-5.4 and gpt-5.5 both pass. Migration target for outdated flagships is still gpt-5.5 (the `flagship` value). Match the SIZE tier: a *-mini model migrates to the latest mini, never the flagship. The GPT-5 family are REASONING models — they reject temperature/top_p (see codeChanges). For temperature-dependent code use the nonReasoning tier below.",
+    "notes": "GPT-5.6 Sol is the latest flagship. GPT-5.6 Terra balances intelligence and cost, and GPT-5.6 Luna is the cost-sensitive, high-volume option; the scanner recognizes all GPT-5.6 variants through the flagship policy ceiling. Mini/nano stay on the 5.4 generation. GPT-5.4 and GPT-5.5 remain accepted by the 5.4 flagship floor; only gpt-5.3 and older flag for migration. Migration target for outdated flagships is gpt-5.6-sol (the `flagship` value). Match the SIZE tier: a *-mini model migrates to the latest mini, never the flagship. The GPT-5 family are REASONING models — they reject temperature/top_p (see codeChanges). For temperature-dependent code use the nonReasoning tier below.",
     "nonReasoning": {
       "flagship": "gpt-4.1",
       "mini": "gpt-4.1-mini",

python/cookbooks/receipt_image_evals/.env.example

@@ -0,0 +1,20 @@
+# Required for live receipt extraction.
+OPENAI_API_KEY=
+
+# Required to send traces to Arize AX.
+ARIZE_API_KEY=
+ARIZE_SPACE_ID=
+ARIZE_PROJECT_NAME=receipt-image-evals
+
+# Public URLs work for the fictional tutorial images. Replace the branch with
+# main after the companion pull request is merged. For real receipts, use
+# approved private object storage and suitably long-lived signed URLs.
+RECEIPT_IMAGE_BASE_URL=https://raw.githubusercontent.com/Arize-ai/tutorials/main/python/cookbooks/receipt_image_evals/images
+
+# Optional extraction-model override. AX runs the evaluator using the AI
+# integration configured in the AX UI.
+RECEIPT_EXTRACTION_MODEL=gpt-5.4-mini
+
+# Optional Gradio server settings.
+GRADIO_SERVER_NAME=127.0.0.1
+PORT=7860

python/cookbooks/receipt_image_evals/README.md

@@ -0,0 +1,51 @@
+# Receipt Image Evals
+
+A demo application to show how to use LLM-as-a-judge evals with images. This example app is for the [Evaluate Receipt Agents with an Image Judge guide](https://arize.com/docs/ax/cookbooks/evaluate/evaluate-receipt-agents-with-image-judge).
+
+This is an example receipt processing application that uses a cheaper model to extract information from the receipt, then a better model for evals to ensure the extraction is working well.
+
+## Prerequisites
+
+- Python 3.10 or later
+- An OpenAI API key with access to `gpt-5.4-mini`
+- An Arize AX API key and Space ID
+- The [AX CLI](https://arize.com/docs/ax/api-and-sdks/ax-cli) and `jq` to run the evaluator setup script
+
+## Run it
+
+```bash
+cd python/cookbooks/receipt_image_evals
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements.txt
+
+cp .env.example .env
+# Edit .env to add your API keys.
+
+python receipt_app.py
+```
+
+The app loads its local `.env` file on startup. Restart it after changing a setting.
+
+Open the local URL printed by Gradio, select a submitted receipt from the expense inbox, and choose **Process expense**. The page shows the receipt document, an expense summary, and the structured expense record. Processed receipts move from **Pending** to **Processed** in the in-memory queue; refresh the page to reset it.
+
+To trace every receipt without launching the UI:
+
+```bash
+python receipt_app.py --batch
+```
+
+The app prints one JSON result per example image. Browse the project named by `ARIZE_PROJECT_NAME` in AX to open its individual traces.
+
+## Evals
+
+Follow the [guide](https://arize.com/docs/ax/cookbooks/evaluate/evaluate-receipt-agents-with-image-judge) to understand the visual-groundedness evaluator. AX runs the stronger judge model through the AI integration you configure in the AX UI; the app traces only the extraction model.
+
+To create the evaluator and its continuous task with the AX CLI, configure an AI integration in AX, then set its ID and run:
+
+```bash
+export ARIZE_AI_INTEGRATION_ID="your-ax-ai-integration-id"
+./setup_evaluator.sh
+```
+
+The script uses `gpt-5.6-luna` by default. Override `RECEIPT_JUDGE_MODEL` when your AX AI integration exposes a different judge model. It is safe to rerun: existing evaluator and task names are reused; an AI integration ID is required only when it needs to create the evaluator.

python/cookbooks/receipt_image_evals/app.css

@@ -0,0 +1,47 @@
+:root {
+  --body-background-fill: #f7f9fc !important;
+  --body-text-color: #4a6881 !important;
+  --body-text-color-subdued: #718096 !important;
+  --block-background-fill: #ffffff !important;
+  --block-label-background-fill: #ffffff !important;
+  --block-label-text-color: #4a6881 !important;
+  --input-background-fill: #ffffff !important;
+  --input-background-fill-focus: #ffffff !important;
+  --input-background-fill-hover: #f8fbfe !important;
+  --code-background-fill: #fbfdff !important;
+  --button-primary-background-fill: #5f87ae !important;
+  --button-primary-background-fill-hover: #4f769b !important;
+  --button-primary-text-color: #ffffff !important;
+}
+
+html, body { background: #f7f9fc !important; color-scheme: light !important; }
+.gradio-container { width: min(1120px, calc(100% - 32px)) !important; max-width: 1120px !important; margin: 0 auto !important; background: #f7f9fc !important; color: #4a6881 !important; padding: 24px 0 !important; }
+.gradio-container > .main, .gradio-container .main { width: 100% !important; max-width: none !important; margin: 0 auto !important; }
+.gradio-container .block, .gradio-container .form, .gradio-container .gr-box, .gradio-container .gr-panel { background: #ffffff !important; border-color: #e4ebf2 !important; color: #4a6881 !important; }
+.gradio-container .wrap, .gradio-container .wrap-inner, .gradio-container .container { background: #ffffff !important; color: #4a6881 !important; }
+.gradio-container input, .gradio-container textarea, .gradio-container button { color-scheme: light !important; }
+.gradio-container input, .gradio-container textarea { background: #ffffff !important; color: #4a6881 !important; border-color: #d7e2ec !important; }
+.gradio-container .cm-editor, .gradio-container .cm-scroller, .gradio-container .cm-gutters, .gradio-container .cm-content { background: #fbfdff !important; color: #4a6881 !important; }
+.gradio-container .cm-activeLine, .gradio-container .cm-activeLineGutter { background: transparent !important; }
+#process-expense { min-height: 44px !important; align-self: end; }
+#process-expense button { min-height: 44px !important; }
+#expense-record, #expense-record textarea { background: #fbfdff !important; color: #4a6881 !important; }
+#expense-record label, #receipt-selector label { color: #1f2937 !important; font-weight: 700 !important; opacity: 1 !important; }
+.gradio-container label, .gradio-container .label-wrap, .gradio-container .label-wrap span { color: #1f2937 !important; font-weight: 700 !important; opacity: 1 !important; }
+#receipt-selector input { color: #35536f !important; }
+#receipt-selector svg { color: #426b8f !important; fill: #426b8f !important; stroke: #426b8f !important; opacity: 1 !important; }
+.section-title { font-size: 22px; font-weight: 700; color: #35536f; margin: 8px 0 6px; text-align: center; }
+.section-copy { color: #718096; font-size: 14px; margin-bottom: 18px; text-align: center; }
+.queue-status { color: #5f7b94; font-size: 13px; font-weight: 600; margin: -8px 0 16px; text-align: center; }
+.field-label { color: #1f2937; font-size: 14px; font-weight: 700; margin: 0 0 7px; text-align: left; }
+.expense-summary { border: 1px solid #e4ebf2; border-radius: 14px; padding: 22px; background: #fbfdff; text-align: center; }
+.expense-summary h3 { margin: 0 0 8px; color: #35536f; font-size: 20px; }
+.expense-total { font-size: 30px; font-weight: 700; color: #426b8f; margin: 4px 0 18px; }
+.expense-meta { display: flex; justify-content: center; gap: 30px; color: #1f2937; font-size: 13px; }
+.expense-meta span { color: #1f2937 !important; font-weight: 600; }
+.expense-meta strong { display: block; color: #4a6881; font-size: 14px; margin-top: 3px; }
+.review-badge { display: inline-block; margin-top: 18px; padding: 6px 10px; border-radius: 999px; font-size: 12px; font-weight: 700; }
+.review-ok { background: #e7f4ed; color: #4b7d60; }
+.review-needed { background: #fff3df; color: #a36f32; }
+.trace-status { color: #6a849a; font-size: 13px; padding-top: 16px; text-align: center; }
+.gr-button-primary { background: #5f87ae !important; border-color: #5f87ae !important; }

python/cookbooks/receipt_image_evals/receipt_app.py

@@ -0,0 +1,213 @@
+"""Gradio receipt-intake app with AX-ready image-grounded trace data."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+import gradio as gr
+from openai import OpenAI
+
+from tracing import initialize_tracing
+from ui import (
+    EXPENSE_INBOX_HEADER,
+    INITIAL_EXPENSE_SUMMARY,
+    STRUCTURED_RECORD_LABEL,
+    SUBMITTED_RECEIPT_LABEL,
+    render_expense_summary,
+    render_processing_error,
+    render_processing_status,
+    render_queue_status,
+)
+
+ROOT = Path(__file__).parent
+# Load this tutorial's local configuration before initializing tracing or OpenAI.
+load_dotenv(ROOT / ".env")
+IMAGES = ROOT / "images"
+DEFAULT_RECEIPT_IMAGE_BASE_URL = (
+    "https://raw.githubusercontent.com/Arize-ai/tutorials/"
+    "main/python/cookbooks/receipt_image_evals/images"
+)
+RECEIPT_IMAGE_BASE_URL = os.environ.get(
+    "RECEIPT_IMAGE_BASE_URL", DEFAULT_RECEIPT_IMAGE_BASE_URL
+).rstrip("/")
+IMAGE_PATHS = sorted(IMAGES.glob("*.png"), key=lambda path: int(path.stem))
+if not IMAGE_PATHS:
+    raise RuntimeError(f"No numbered PNG images found in {IMAGES}")
+RECEIPTS_BY_ID = {
+    path.stem: {"id": path.stem, "image": path.name}
+    for path in IMAGE_PATHS
+}
+RECEIPT_EXTRACTION_MODEL = os.environ.get("RECEIPT_EXTRACTION_MODEL", "gpt-5.4-mini")
+APP_CSS = (ROOT / "app.css").read_text(encoding="utf-8")
+EXTRACTION_SYSTEM_PROMPT = """Extract the receipt into the requested JSON schema. Use only what is visually supported by the image.
+If a field is unclear, return null or an empty list and set needs_review to true."""
+
+openai_api_key = os.environ.get("OPENAI_API_KEY")
+if not openai_api_key:
+    raise RuntimeError("Set OPENAI_API_KEY before starting the receipt app.")
+# Register and instrument before creating the client so every OpenAI call is traced.
+TRACER = initialize_tracing(__name__)
+OPENAI_CLIENT = OpenAI(api_key=openai_api_key)
+
+RECEIPT_SCHEMA = {
+    "type": "object",
+    "additionalProperties": False,
+    "properties": {
+        "merchant": {"type": ["string", "null"]},
+        "currency": {"type": ["string", "null"]},
+        "items": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "additionalProperties": False,
+                "properties": {"name": {"type": "string"}, "amount": {"type": "number"}},
+                "required": ["name", "amount"],
+            },
+        },
+        "subtotal": {"type": ["number", "null"]},
+        "tax": {"type": ["number", "null"]},
+        "tip": {"type": ["number", "null"]},
+        "total": {"type": ["number", "null"]},
+        "needs_review": {"type": "boolean"},
+    },
+    "required": ["merchant", "currency", "items", "subtotal", "tax", "tip", "total", "needs_review"],
+}
+
+
+def image_url(receipt: dict[str, Any]) -> str:
+    """Return a fetchable image URL; traces never contain base64 image data."""
+    return f"{RECEIPT_IMAGE_BASE_URL}/{receipt['image']}"
+
+
+def extraction_input(receipt: dict[str, Any]) -> dict[str, str]:
+    """Build the sole argument captured as the chain span's input.value."""
+    return {
+        "system_prompt": EXTRACTION_SYSTEM_PROMPT,
+        "image_url": image_url(receipt),
+    }
+
+
+@TRACER.chain(name="receipt.extract")
+def model_extract(receipt_input: dict[str, str]) -> dict[str, Any]:
+    # The decorator captures receipt_input and the returned JSON as chain I/O.
+    response = OPENAI_CLIENT.responses.create(
+        model=RECEIPT_EXTRACTION_MODEL,
+        input=[
+            {
+                "role": "system",
+                "content": [
+                    {"type": "input_text", "text": receipt_input["system_prompt"]},
+                ],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_image", "image_url": receipt_input["image_url"], "detail": "high"},
+                ],
+            }
+        ],
+        text={
+            "format": {
+                "type": "json_schema",
+                "name": "receipt_extraction",
+                "strict": True,
+                "schema": RECEIPT_SCHEMA,
+            }
+        },
+    )
+    return json.loads(response.output_text)
+
+
+def run_receipt(receipt_id: str) -> dict[str, Any]:
+    receipt = RECEIPTS_BY_ID[receipt_id]
+    return model_extract(extraction_input(receipt))
+
+
+def receipt_choices(processed_ids: list[str]) -> list[tuple[str, str]]:
+    processed = set(processed_ids)
+    return [
+        (f"Receipt #{int(receipt_id):03d} · {'Processed' if receipt_id in processed else 'Pending'}", receipt_id)
+        for receipt_id in RECEIPTS_BY_ID
+    ]
+
+
+def queue_status(processed_ids: list[str]) -> str:
+    return render_queue_status(total_receipts=len(RECEIPTS_BY_ID), processed_count=len(processed_ids))
+
+
+def ui_run(receipt_id: str, processed_ids: list[str]):
+    processed_ids = processed_ids or []
+    try:
+        result = run_receipt(receipt_id)
+        receipt = RECEIPTS_BY_ID[receipt_id]
+        updated_processed = list(dict.fromkeys([*processed_ids, receipt_id]))
+        return (
+            image_url(receipt),
+            render_expense_summary(result),
+            json.dumps(result, indent=2),
+            render_processing_status(),
+            queue_status(updated_processed),
+            gr.Dropdown(choices=receipt_choices(updated_processed), value=receipt_id),
+            updated_processed,
+        )
+    except Exception as error:
+        receipt = RECEIPTS_BY_ID[receipt_id]
+        return (
+            image_url(receipt),
+            "",
+            "",
+            render_processing_error(error),
+            queue_status(processed_ids),
+            gr.Dropdown(choices=receipt_choices(processed_ids), value=receipt_id),
+            processed_ids,
+        )
+
+
+def run_batch() -> None:
+    for receipt in RECEIPTS_BY_ID.values():
+        result = run_receipt(receipt["id"])
+        print(json.dumps({"receipt_id": receipt["id"], "result": result}))
+
+
+def build_app():
+    initial_processed: list[str] = []
+    choices = receipt_choices(initial_processed)
+    with gr.Blocks(title="Expense Inbox") as app:
+        gr.HTML(EXPENSE_INBOX_HEADER)
+        processed = gr.State(initial_processed)
+        queue = gr.HTML(queue_status(initial_processed))
+        with gr.Row():
+            with gr.Column(scale=3):
+                gr.HTML(SUBMITTED_RECEIPT_LABEL)
+                selected_receipt = gr.Dropdown(choices=choices, value=choices[0][1], show_label=False, elem_id="receipt-selector")
+            with gr.Column(scale=1):
+                run = gr.Button("Process expense", variant="primary", elem_id="process-expense")
+        with gr.Row():
+            image = gr.Image(value=image_url(RECEIPTS_BY_ID[choices[0][1]]), show_label=False, buttons=[], type="filepath", height=560, scale=1)
+            with gr.Column(scale=1):
+                summary = gr.HTML(INITIAL_EXPENSE_SUMMARY)
+                gr.HTML(STRUCTURED_RECORD_LABEL)
+                output = gr.Textbox(lines=18, interactive=False, show_label=False, elem_id="expense-record")
+        status = gr.HTML()
+        run.click(ui_run, inputs=[selected_receipt, processed], outputs=[image, summary, output, status, queue, selected_receipt, processed])
+        selected_receipt.change(lambda receipt_id: image_url(RECEIPTS_BY_ID[receipt_id]), selected_receipt, image)
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--batch", action="store_true", help="Trace every numbered image, then exit.")
+    args = parser.parse_args()
+    if args.batch:
+        run_batch()
+    else:
+        build_app().launch(
+            server_name=os.environ.get("GRADIO_SERVER_NAME", "127.0.0.1"),
+            server_port=int(os.environ.get("PORT", "7860")),
+            css=APP_CSS,
+        )

python/cookbooks/receipt_image_evals/requirements.txt

@@ -0,0 +1,7 @@
+arize-otel
+gradio
+openai
+openinference-instrumentation
+openinference-instrumentation-openai
+opentelemetry-api
+python-dotenv