diff --git a/.agents/skills/check-models/scripts/models.json b/.agents/skills/check-models/scripts/models.json index 988d0a0..b78acfe 100644 --- a/.agents/skills/check-models/scripts/models.json +++ b/.agents/skills/check-models/scripts/models.json @@ -1,10 +1,12 @@ { "_comment": "Single source of truth for the check-models skill. Update the numbers in `openai` and `anthropic` whenever a new model ships, then re-run the scanner. The scanner derives the full replacement table from these policy values — you should not need to enumerate every old ID.", - "updated": "2026-06-18", - "verifiedBy": "WebFetch developers.openai.com/api/docs/models + /deprecations (2026-06-18); Google via ai.google.dev/gemini-api/docs/models (2026-06-18); Anthropic via claude-api skill shared/models.md", + "updated": "2026-08-28", + "verifiedBy": "OpenAI Models documentation (2026-08-28); Google via ai.google.dev/gemini-api/docs/models (2026-06-18); Anthropic via claude-api skill shared/models.md", "openai": { - "flagship": "gpt-5.5", + "flagship": "gpt-5.6-sol", + "terra": "gpt-5.6-terra", + "luna": "gpt-5.6-luna", "pro": "gpt-5.5-pro", "mini": "gpt-5.4-mini", "nano": "gpt-5.4-nano", @@ -12,7 +14,7 @@ "proMinVersion": 5.5, "miniMinVersion": 5.4, "nanoMinVersion": 5.4, - "notes": "GPT-5.5 is the latest flagship (released 2026-04-23, API id gpt-5.5-2026-04-23) and has no mini/nano variant — mini/nano stay on the 5.4 generation. GPT-5.4 also remains available as a cheaper flagship (still current, NOT deprecated per OpenAI's models page), so flagshipMinVersion is 5.4 — only gpt-5.3 and older flag for migration; gpt-5.4 and gpt-5.5 both pass. Migration target for outdated flagships is still gpt-5.5 (the `flagship` value). Match the SIZE tier: a *-mini model migrates to the latest mini, never the flagship. The GPT-5 family are REASONING models — they reject temperature/top_p (see codeChanges). For temperature-dependent code use the nonReasoning tier below.", + "notes": "GPT-5.6 Sol is the latest flagship. GPT-5.6 Terra balances intelligence and cost, and GPT-5.6 Luna is the cost-sensitive, high-volume option; the scanner recognizes all GPT-5.6 variants through the flagship policy ceiling. Mini/nano stay on the 5.4 generation. GPT-5.4 and GPT-5.5 remain accepted by the 5.4 flagship floor; only gpt-5.3 and older flag for migration. Migration target for outdated flagships is gpt-5.6-sol (the `flagship` value). Match the SIZE tier: a *-mini model migrates to the latest mini, never the flagship. The GPT-5 family are REASONING models — they reject temperature/top_p (see codeChanges). For temperature-dependent code use the nonReasoning tier below.", "nonReasoning": { "flagship": "gpt-4.1", "mini": "gpt-4.1-mini", diff --git a/python/cookbooks/receipt_image_evals/.env.example b/python/cookbooks/receipt_image_evals/.env.example new file mode 100644 index 0000000..f5b8a4e --- /dev/null +++ b/python/cookbooks/receipt_image_evals/.env.example @@ -0,0 +1,20 @@ +# Required for live receipt extraction. +OPENAI_API_KEY= + +# Required to send traces to Arize AX. +ARIZE_API_KEY= +ARIZE_SPACE_ID= +ARIZE_PROJECT_NAME=receipt-image-evals + +# Public URLs work for the fictional tutorial images. Replace the branch with +# main after the companion pull request is merged. For real receipts, use +# approved private object storage and suitably long-lived signed URLs. +RECEIPT_IMAGE_BASE_URL=https://raw.githubusercontent.com/Arize-ai/tutorials/main/python/cookbooks/receipt_image_evals/images + +# Optional extraction-model override. AX runs the evaluator using the AI +# integration configured in the AX UI. +RECEIPT_EXTRACTION_MODEL=gpt-5.4-mini + +# Optional Gradio server settings. +GRADIO_SERVER_NAME=127.0.0.1 +PORT=7860 diff --git a/python/cookbooks/receipt_image_evals/README.md b/python/cookbooks/receipt_image_evals/README.md new file mode 100644 index 0000000..51b0a49 --- /dev/null +++ b/python/cookbooks/receipt_image_evals/README.md @@ -0,0 +1,51 @@ +# Receipt Image Evals + +A demo application to show how to use LLM-as-a-judge evals with images. This example app is for the [Evaluate Receipt Agents with an Image Judge guide](https://arize.com/docs/ax/cookbooks/evaluate/evaluate-receipt-agents-with-image-judge). + +This is an example receipt processing application that uses a cheaper model to extract information from the receipt, then a better model for evals to ensure the extraction is working well. + +## Prerequisites + +- Python 3.10 or later +- An OpenAI API key with access to `gpt-5.4-mini` +- An Arize AX API key and Space ID +- The [AX CLI](https://arize.com/docs/ax/api-and-sdks/ax-cli) and `jq` to run the evaluator setup script + +## Run it + +```bash +cd python/cookbooks/receipt_image_evals +python -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt + +cp .env.example .env +# Edit .env to add your API keys. + +python receipt_app.py +``` + +The app loads its local `.env` file on startup. Restart it after changing a setting. + +Open the local URL printed by Gradio, select a submitted receipt from the expense inbox, and choose **Process expense**. The page shows the receipt document, an expense summary, and the structured expense record. Processed receipts move from **Pending** to **Processed** in the in-memory queue; refresh the page to reset it. + +To trace every receipt without launching the UI: + +```bash +python receipt_app.py --batch +``` + +The app prints one JSON result per example image. Browse the project named by `ARIZE_PROJECT_NAME` in AX to open its individual traces. + +## Evals + +Follow the [guide](https://arize.com/docs/ax/cookbooks/evaluate/evaluate-receipt-agents-with-image-judge) to understand the visual-groundedness evaluator. AX runs the stronger judge model through the AI integration you configure in the AX UI; the app traces only the extraction model. + +To create the evaluator and its continuous task with the AX CLI, configure an AI integration in AX, then set its ID and run: + +```bash +export ARIZE_AI_INTEGRATION_ID="your-ax-ai-integration-id" +./setup_evaluator.sh +``` + +The script uses `gpt-5.6-luna` by default. Override `RECEIPT_JUDGE_MODEL` when your AX AI integration exposes a different judge model. It is safe to rerun: existing evaluator and task names are reused; an AI integration ID is required only when it needs to create the evaluator. diff --git a/python/cookbooks/receipt_image_evals/app.css b/python/cookbooks/receipt_image_evals/app.css new file mode 100644 index 0000000..01976cb --- /dev/null +++ b/python/cookbooks/receipt_image_evals/app.css @@ -0,0 +1,47 @@ +:root { + --body-background-fill: #f7f9fc !important; + --body-text-color: #4a6881 !important; + --body-text-color-subdued: #718096 !important; + --block-background-fill: #ffffff !important; + --block-label-background-fill: #ffffff !important; + --block-label-text-color: #4a6881 !important; + --input-background-fill: #ffffff !important; + --input-background-fill-focus: #ffffff !important; + --input-background-fill-hover: #f8fbfe !important; + --code-background-fill: #fbfdff !important; + --button-primary-background-fill: #5f87ae !important; + --button-primary-background-fill-hover: #4f769b !important; + --button-primary-text-color: #ffffff !important; +} + +html, body { background: #f7f9fc !important; color-scheme: light !important; } +.gradio-container { width: min(1120px, calc(100% - 32px)) !important; max-width: 1120px !important; margin: 0 auto !important; background: #f7f9fc !important; color: #4a6881 !important; padding: 24px 0 !important; } +.gradio-container > .main, .gradio-container .main { width: 100% !important; max-width: none !important; margin: 0 auto !important; } +.gradio-container .block, .gradio-container .form, .gradio-container .gr-box, .gradio-container .gr-panel { background: #ffffff !important; border-color: #e4ebf2 !important; color: #4a6881 !important; } +.gradio-container .wrap, .gradio-container .wrap-inner, .gradio-container .container { background: #ffffff !important; color: #4a6881 !important; } +.gradio-container input, .gradio-container textarea, .gradio-container button { color-scheme: light !important; } +.gradio-container input, .gradio-container textarea { background: #ffffff !important; color: #4a6881 !important; border-color: #d7e2ec !important; } +.gradio-container .cm-editor, .gradio-container .cm-scroller, .gradio-container .cm-gutters, .gradio-container .cm-content { background: #fbfdff !important; color: #4a6881 !important; } +.gradio-container .cm-activeLine, .gradio-container .cm-activeLineGutter { background: transparent !important; } +#process-expense { min-height: 44px !important; align-self: end; } +#process-expense button { min-height: 44px !important; } +#expense-record, #expense-record textarea { background: #fbfdff !important; color: #4a6881 !important; } +#expense-record label, #receipt-selector label { color: #1f2937 !important; font-weight: 700 !important; opacity: 1 !important; } +.gradio-container label, .gradio-container .label-wrap, .gradio-container .label-wrap span { color: #1f2937 !important; font-weight: 700 !important; opacity: 1 !important; } +#receipt-selector input { color: #35536f !important; } +#receipt-selector svg { color: #426b8f !important; fill: #426b8f !important; stroke: #426b8f !important; opacity: 1 !important; } +.section-title { font-size: 22px; font-weight: 700; color: #35536f; margin: 8px 0 6px; text-align: center; } +.section-copy { color: #718096; font-size: 14px; margin-bottom: 18px; text-align: center; } +.queue-status { color: #5f7b94; font-size: 13px; font-weight: 600; margin: -8px 0 16px; text-align: center; } +.field-label { color: #1f2937; font-size: 14px; font-weight: 700; margin: 0 0 7px; text-align: left; } +.expense-summary { border: 1px solid #e4ebf2; border-radius: 14px; padding: 22px; background: #fbfdff; text-align: center; } +.expense-summary h3 { margin: 0 0 8px; color: #35536f; font-size: 20px; } +.expense-total { font-size: 30px; font-weight: 700; color: #426b8f; margin: 4px 0 18px; } +.expense-meta { display: flex; justify-content: center; gap: 30px; color: #1f2937; font-size: 13px; } +.expense-meta span { color: #1f2937 !important; font-weight: 600; } +.expense-meta strong { display: block; color: #4a6881; font-size: 14px; margin-top: 3px; } +.review-badge { display: inline-block; margin-top: 18px; padding: 6px 10px; border-radius: 999px; font-size: 12px; font-weight: 700; } +.review-ok { background: #e7f4ed; color: #4b7d60; } +.review-needed { background: #fff3df; color: #a36f32; } +.trace-status { color: #6a849a; font-size: 13px; padding-top: 16px; text-align: center; } +.gr-button-primary { background: #5f87ae !important; border-color: #5f87ae !important; } diff --git a/python/cookbooks/receipt_image_evals/images/1.png b/python/cookbooks/receipt_image_evals/images/1.png new file mode 100644 index 0000000..9e93223 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/1.png differ diff --git a/python/cookbooks/receipt_image_evals/images/10.png b/python/cookbooks/receipt_image_evals/images/10.png new file mode 100644 index 0000000..2a138f6 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/10.png differ diff --git a/python/cookbooks/receipt_image_evals/images/11.png b/python/cookbooks/receipt_image_evals/images/11.png new file mode 100644 index 0000000..473a630 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/11.png differ diff --git a/python/cookbooks/receipt_image_evals/images/12.png b/python/cookbooks/receipt_image_evals/images/12.png new file mode 100644 index 0000000..a2dc6dd Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/12.png differ diff --git a/python/cookbooks/receipt_image_evals/images/13.png b/python/cookbooks/receipt_image_evals/images/13.png new file mode 100644 index 0000000..e12248f Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/13.png differ diff --git a/python/cookbooks/receipt_image_evals/images/14.png b/python/cookbooks/receipt_image_evals/images/14.png new file mode 100644 index 0000000..8e71682 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/14.png differ diff --git a/python/cookbooks/receipt_image_evals/images/2.png b/python/cookbooks/receipt_image_evals/images/2.png new file mode 100644 index 0000000..b03a8b1 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/2.png differ diff --git a/python/cookbooks/receipt_image_evals/images/3.png b/python/cookbooks/receipt_image_evals/images/3.png new file mode 100644 index 0000000..8304c34 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/3.png differ diff --git a/python/cookbooks/receipt_image_evals/images/4.png b/python/cookbooks/receipt_image_evals/images/4.png new file mode 100644 index 0000000..fd280a7 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/4.png differ diff --git a/python/cookbooks/receipt_image_evals/images/5.png b/python/cookbooks/receipt_image_evals/images/5.png new file mode 100644 index 0000000..f26931d Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/5.png differ diff --git a/python/cookbooks/receipt_image_evals/images/6.png b/python/cookbooks/receipt_image_evals/images/6.png new file mode 100644 index 0000000..88d72ea Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/6.png differ diff --git a/python/cookbooks/receipt_image_evals/images/7.png b/python/cookbooks/receipt_image_evals/images/7.png new file mode 100644 index 0000000..a723ac3 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/7.png differ diff --git a/python/cookbooks/receipt_image_evals/images/8.png b/python/cookbooks/receipt_image_evals/images/8.png new file mode 100644 index 0000000..419ab88 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/8.png differ diff --git a/python/cookbooks/receipt_image_evals/images/9.png b/python/cookbooks/receipt_image_evals/images/9.png new file mode 100644 index 0000000..54be323 Binary files /dev/null and b/python/cookbooks/receipt_image_evals/images/9.png differ diff --git a/python/cookbooks/receipt_image_evals/receipt_app.py b/python/cookbooks/receipt_image_evals/receipt_app.py new file mode 100644 index 0000000..d7810e1 --- /dev/null +++ b/python/cookbooks/receipt_image_evals/receipt_app.py @@ -0,0 +1,213 @@ +"""Gradio receipt-intake app with AX-ready image-grounded trace data.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv +import gradio as gr +from openai import OpenAI + +from tracing import initialize_tracing +from ui import ( + EXPENSE_INBOX_HEADER, + INITIAL_EXPENSE_SUMMARY, + STRUCTURED_RECORD_LABEL, + SUBMITTED_RECEIPT_LABEL, + render_expense_summary, + render_processing_error, + render_processing_status, + render_queue_status, +) + +ROOT = Path(__file__).parent +# Load this tutorial's local configuration before initializing tracing or OpenAI. +load_dotenv(ROOT / ".env") +IMAGES = ROOT / "images" +DEFAULT_RECEIPT_IMAGE_BASE_URL = ( + "https://raw.githubusercontent.com/Arize-ai/tutorials/" + "main/python/cookbooks/receipt_image_evals/images" +) +RECEIPT_IMAGE_BASE_URL = os.environ.get( + "RECEIPT_IMAGE_BASE_URL", DEFAULT_RECEIPT_IMAGE_BASE_URL +).rstrip("/") +IMAGE_PATHS = sorted(IMAGES.glob("*.png"), key=lambda path: int(path.stem)) +if not IMAGE_PATHS: + raise RuntimeError(f"No numbered PNG images found in {IMAGES}") +RECEIPTS_BY_ID = { + path.stem: {"id": path.stem, "image": path.name} + for path in IMAGE_PATHS +} +RECEIPT_EXTRACTION_MODEL = os.environ.get("RECEIPT_EXTRACTION_MODEL", "gpt-5.4-mini") +APP_CSS = (ROOT / "app.css").read_text(encoding="utf-8") +EXTRACTION_SYSTEM_PROMPT = """Extract the receipt into the requested JSON schema. Use only what is visually supported by the image. +If a field is unclear, return null or an empty list and set needs_review to true.""" + +openai_api_key = os.environ.get("OPENAI_API_KEY") +if not openai_api_key: + raise RuntimeError("Set OPENAI_API_KEY before starting the receipt app.") +# Register and instrument before creating the client so every OpenAI call is traced. +TRACER = initialize_tracing(__name__) +OPENAI_CLIENT = OpenAI(api_key=openai_api_key) + +RECEIPT_SCHEMA = { + "type": "object", + "additionalProperties": False, + "properties": { + "merchant": {"type": ["string", "null"]}, + "currency": {"type": ["string", "null"]}, + "items": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "properties": {"name": {"type": "string"}, "amount": {"type": "number"}}, + "required": ["name", "amount"], + }, + }, + "subtotal": {"type": ["number", "null"]}, + "tax": {"type": ["number", "null"]}, + "tip": {"type": ["number", "null"]}, + "total": {"type": ["number", "null"]}, + "needs_review": {"type": "boolean"}, + }, + "required": ["merchant", "currency", "items", "subtotal", "tax", "tip", "total", "needs_review"], +} + + +def image_url(receipt: dict[str, Any]) -> str: + """Return a fetchable image URL; traces never contain base64 image data.""" + return f"{RECEIPT_IMAGE_BASE_URL}/{receipt['image']}" + + +def extraction_input(receipt: dict[str, Any]) -> dict[str, str]: + """Build the sole argument captured as the chain span's input.value.""" + return { + "system_prompt": EXTRACTION_SYSTEM_PROMPT, + "image_url": image_url(receipt), + } + + +@TRACER.chain(name="receipt.extract") +def model_extract(receipt_input: dict[str, str]) -> dict[str, Any]: + # The decorator captures receipt_input and the returned JSON as chain I/O. + response = OPENAI_CLIENT.responses.create( + model=RECEIPT_EXTRACTION_MODEL, + input=[ + { + "role": "system", + "content": [ + {"type": "input_text", "text": receipt_input["system_prompt"]}, + ], + }, + { + "role": "user", + "content": [ + {"type": "input_image", "image_url": receipt_input["image_url"], "detail": "high"}, + ], + } + ], + text={ + "format": { + "type": "json_schema", + "name": "receipt_extraction", + "strict": True, + "schema": RECEIPT_SCHEMA, + } + }, + ) + return json.loads(response.output_text) + + +def run_receipt(receipt_id: str) -> dict[str, Any]: + receipt = RECEIPTS_BY_ID[receipt_id] + return model_extract(extraction_input(receipt)) + + +def receipt_choices(processed_ids: list[str]) -> list[tuple[str, str]]: + processed = set(processed_ids) + return [ + (f"Receipt #{int(receipt_id):03d} · {'Processed' if receipt_id in processed else 'Pending'}", receipt_id) + for receipt_id in RECEIPTS_BY_ID + ] + + +def queue_status(processed_ids: list[str]) -> str: + return render_queue_status(total_receipts=len(RECEIPTS_BY_ID), processed_count=len(processed_ids)) + + +def ui_run(receipt_id: str, processed_ids: list[str]): + processed_ids = processed_ids or [] + try: + result = run_receipt(receipt_id) + receipt = RECEIPTS_BY_ID[receipt_id] + updated_processed = list(dict.fromkeys([*processed_ids, receipt_id])) + return ( + image_url(receipt), + render_expense_summary(result), + json.dumps(result, indent=2), + render_processing_status(), + queue_status(updated_processed), + gr.Dropdown(choices=receipt_choices(updated_processed), value=receipt_id), + updated_processed, + ) + except Exception as error: + receipt = RECEIPTS_BY_ID[receipt_id] + return ( + image_url(receipt), + "", + "", + render_processing_error(error), + queue_status(processed_ids), + gr.Dropdown(choices=receipt_choices(processed_ids), value=receipt_id), + processed_ids, + ) + + +def run_batch() -> None: + for receipt in RECEIPTS_BY_ID.values(): + result = run_receipt(receipt["id"]) + print(json.dumps({"receipt_id": receipt["id"], "result": result})) + + +def build_app(): + initial_processed: list[str] = [] + choices = receipt_choices(initial_processed) + with gr.Blocks(title="Expense Inbox") as app: + gr.HTML(EXPENSE_INBOX_HEADER) + processed = gr.State(initial_processed) + queue = gr.HTML(queue_status(initial_processed)) + with gr.Row(): + with gr.Column(scale=3): + gr.HTML(SUBMITTED_RECEIPT_LABEL) + selected_receipt = gr.Dropdown(choices=choices, value=choices[0][1], show_label=False, elem_id="receipt-selector") + with gr.Column(scale=1): + run = gr.Button("Process expense", variant="primary", elem_id="process-expense") + with gr.Row(): + image = gr.Image(value=image_url(RECEIPTS_BY_ID[choices[0][1]]), show_label=False, buttons=[], type="filepath", height=560, scale=1) + with gr.Column(scale=1): + summary = gr.HTML(INITIAL_EXPENSE_SUMMARY) + gr.HTML(STRUCTURED_RECORD_LABEL) + output = gr.Textbox(lines=18, interactive=False, show_label=False, elem_id="expense-record") + status = gr.HTML() + run.click(ui_run, inputs=[selected_receipt, processed], outputs=[image, summary, output, status, queue, selected_receipt, processed]) + selected_receipt.change(lambda receipt_id: image_url(RECEIPTS_BY_ID[receipt_id]), selected_receipt, image) + return app + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--batch", action="store_true", help="Trace every numbered image, then exit.") + args = parser.parse_args() + if args.batch: + run_batch() + else: + build_app().launch( + server_name=os.environ.get("GRADIO_SERVER_NAME", "127.0.0.1"), + server_port=int(os.environ.get("PORT", "7860")), + css=APP_CSS, + ) diff --git a/python/cookbooks/receipt_image_evals/requirements.txt b/python/cookbooks/receipt_image_evals/requirements.txt new file mode 100644 index 0000000..b32ed2e --- /dev/null +++ b/python/cookbooks/receipt_image_evals/requirements.txt @@ -0,0 +1,7 @@ +arize-otel +gradio +openai +openinference-instrumentation +openinference-instrumentation-openai +opentelemetry-api +python-dotenv diff --git a/python/cookbooks/receipt_image_evals/setup_evaluator.sh b/python/cookbooks/receipt_image_evals/setup_evaluator.sh new file mode 100755 index 0000000..fa4f34a --- /dev/null +++ b/python/cookbooks/receipt_image_evals/setup_evaluator.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Create the AX evaluator and continuous task used by this receipt tutorial. +set -euo pipefail + +: "${ARIZE_SPACE_ID:?Set ARIZE_SPACE_ID to the target AX space.}" + +command -v ax >/dev/null || { echo "The AX CLI must be installed." >&2; exit 1; } +command -v jq >/dev/null || { echo "jq must be installed." >&2; exit 1; } + +PROJECT_NAME="${ARIZE_PROJECT_NAME:-receipt-image-evals}" +JUDGE_MODEL="${RECEIPT_JUDGE_MODEL:-gpt-5.6-luna}" +EVALUATOR_NAME="Receipt Visual Groundedness" +EVALUATOR_COLUMN="receipt_visual_groundedness" +TASK_NAME="Receipt Visual Groundedness Monitor" + +read_json_id() { + jq -er '.id' +} + +find_task_id() { + jq -er --arg name "$TASK_NAME" ' + (.tasks // .items // [])[] | select(.name == $name) | .id + ' || true +} + +if evaluator_json="$(ax evaluators get "$EVALUATOR_NAME" --space "$ARIZE_SPACE_ID" --output json 2>/dev/null)"; then + EVALUATOR_ID="$(printf '%s' "$evaluator_json" | read_json_id)" + echo "Reusing evaluator: $EVALUATOR_ID" +else + : "${ARIZE_AI_INTEGRATION_ID:?Set ARIZE_AI_INTEGRATION_ID to the AX AI integration ID for the judge model.}" + ax evaluators create-template-evaluator \ + --name "$EVALUATOR_NAME" \ + --space "$ARIZE_SPACE_ID" \ + --template-name "$EVALUATOR_COLUMN" \ + --commit-message "Create receipt visual-groundedness evaluator" \ + --ai-integration-id "$ARIZE_AI_INTEGRATION_ID" \ + --model-name "$JUDGE_MODEL" \ + --description "Checks whether receipt extraction JSON is visually grounded in its receipt image." \ + --include-explanations \ + --invocation-params '{"temperature": 0}' \ + --classification-choices '{"grounded": 1, "not_grounded": 0, "needs_review": 0}' \ + --direction MAXIMIZE \ + --data-granularity span \ + --template 'Assess whether the structured extraction is visually grounded in the receipt image. + + +{{receipt_image}} + + + +{{extraction}} + + +Choose grounded only when every asserted merchant, item, amount, currency, and total is visibly supported. Choose not_grounded for invented or contradicted values. Choose needs_review when the image is degraded or ambiguous enough that a reliable decision cannot be made. + +Return a classification label and an explanation. The explanation must be valid JSON, without Markdown fences, with exactly these fields: +{ + "evaluated_extraction": , + "judge_result": { + "label": , + "reason": + } +}' + + EVALUATOR_ID="$(ax evaluators get "$EVALUATOR_NAME" --space "$ARIZE_SPACE_ID" --output json | read_json_id)" + echo "Created evaluator: $EVALUATOR_ID" +fi + +task_json="$(ax tasks list --name "$TASK_NAME" --project "$PROJECT_NAME" --space "$ARIZE_SPACE_ID" --limit 100 --output json)" +TASK_ID="$(printf '%s' "$task_json" | find_task_id)" +if [[ -n "$TASK_ID" ]]; then + echo "Reusing continuous evaluation task: $TASK_ID" + exit 0 +fi + +EVALUATORS="$(jq -nc --arg evaluator_id "$EVALUATOR_ID" '[{ + evaluator_id: $evaluator_id, + query_filter: "span_kind = '\''CHAIN'\''", + column_mappings: { + receipt_image: "attributes.input.value", + extraction: "attributes.output.value" + } +}]')" + +ax tasks create-evaluation \ + --name "$TASK_NAME" \ + --task-type TEMPLATE_EVALUATION \ + --project "$PROJECT_NAME" \ + --space "$ARIZE_SPACE_ID" \ + --evaluators "$EVALUATORS" \ + --is-continuous \ + --sampling-rate 1 + +echo "Created continuous evaluation task for project: $PROJECT_NAME" diff --git a/python/cookbooks/receipt_image_evals/tracing.py b/python/cookbooks/receipt_image_evals/tracing.py new file mode 100644 index 0000000..12ce931 --- /dev/null +++ b/python/cookbooks/receipt_image_evals/tracing.py @@ -0,0 +1,27 @@ +"""Arize AX and OpenTelemetry setup for the receipt demo.""" + +from __future__ import annotations + +import os + +from arize.otel import register +from openinference.instrumentation import OITracer, TraceConfig +from openinference.instrumentation.openai import OpenAIInstrumentor + + +def initialize_tracing(instrumentation_scope: str) -> OITracer: + """Configure mandatory AX tracing before any OpenAI client is created.""" + required = ("ARIZE_API_KEY", "ARIZE_SPACE_ID") + if not all(os.environ.get(name) for name in required): + raise RuntimeError("Tracing requires ARIZE_API_KEY and ARIZE_SPACE_ID before starting the receipt app.") + provider = register( + project_name=os.environ.get("ARIZE_PROJECT_NAME", "receipt-image-evals"), + space_id=os.environ["ARIZE_SPACE_ID"], + api_key=os.environ["ARIZE_API_KEY"], + # Export each span before the process moves on; batch mode is disabled + # so the CLI batch command does not need an explicit flush. + batch=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + # OITracer adds @chain, which captures the receipt function's input/output. + return OITracer(provider.get_tracer(instrumentation_scope), config=TraceConfig()) diff --git a/python/cookbooks/receipt_image_evals/ui.py b/python/cookbooks/receipt_image_evals/ui.py new file mode 100644 index 0000000..b506304 --- /dev/null +++ b/python/cookbooks/receipt_image_evals/ui.py @@ -0,0 +1,64 @@ +"""HTML renderers for the receipt expense inbox.""" + +from __future__ import annotations + +from html import escape +from typing import Any + +EXPENSE_INBOX_HEADER = ( + '
Expense inbox
' + '
Select a submitted receipt and create a structured expense record.
' +) +SUBMITTED_RECEIPT_LABEL = '
Submitted receipt
' +STRUCTURED_RECORD_LABEL = '
Structured expense record
' +INITIAL_EXPENSE_SUMMARY = ( + '

Expense details

' + '
Process a receipt to create an expense record.
' +) + + +def render_stat(label: str, value: str) -> str: + """Render one labeled value in the expense summary.""" + return f"
{escape(label)}{escape(value)}
" + + +def render_expense_summary(record: dict[str, Any]) -> str: + """Render an extracted receipt as an expense summary card.""" + currency = str(record.get("currency") or "—") + merchant = str(record.get("merchant") or "Merchant pending review") + total = record.get("total") + total_value = f"{currency} {total:,.2f}" if isinstance(total, (int, float)) else "Amount pending review" + review_needed = bool(record.get("needs_review", False)) + review_class = "review-needed" if review_needed else "review-ok" + review_text = "Review needed" if review_needed else "Ready for review" + stats = "".join( + ( + render_stat("Expense type", "Receipt"), + render_stat("Line items", str(len(record.get("items") or []))), + render_stat("Currency", currency), + ) + ) + return ( + '
' + f"

{escape(merchant)}

" + f'
{escape(total_value)}
' + f'
{stats}
' + f'{review_text}' + "
" + ) + + +def render_queue_status(*, total_receipts: int, processed_count: int) -> str: + """Render the pending and processed receipt counts.""" + pending_count = total_receipts - processed_count + return f'
{pending_count} pending · {processed_count} processed
' + + +def render_processing_status() -> str: + """Render successful processing and trace status.""" + return '
Trace created in AX. · Receipt moved to Processed.
' + + +def render_processing_error(error: Exception) -> str: + """Render a safely escaped extraction error.""" + return f'
Processing failed: {escape(str(error))}
'