Skip to content

docs: migrate cookbook notebooks to current phoenix-evals API #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jimbobbennett wants to merge 5 commits into from
Closed

docs: migrate cookbook notebooks to current phoenix-evals API #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
66fbcf4
docs: migrate cookbook notebooks to current phoenix-evals API
jimbobbennett Jul 23, 2026
31f797e
fix: bump generation model gpt-4o -> gpt-5.4-mini (check-models)
jimbobbennett Jul 23, 2026
e0f2806
fix: migrate notebook experiments to v8 API + fix runtime bugs
jimbobbennett Jul 23, 2026
834f423
fix: make agent cookbooks run end-to-end (verified live)
jimbobbennett Jul 23, 2026
9ef6fdb
fix: make couchbase agentic-RAG notebook run end-to-end (drop agentc)
jimbobbennett Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 95 additions & 75 deletions python/llm/agents/agents-cookbook.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@
"metadata": {},
"outputs": [],
"source": [
"!pip install -qq arize-otel openinference-instrumentation-openai arize-phoenix-evals \"arize[Datasets]\"\n",
"!pip install -qq arize-otel openinference-instrumentation-openai arize-phoenix-evals \"arize>=8.0.0\"\n",
"\n",
"!pip install -qq openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs nest_asyncio"
"!pip install -qq openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs"
]
},
{
Expand All @@ -82,9 +82,6 @@
"source": [
"import os\n",
"from getpass import getpass\n",
"import nest_asyncio\n",
"\n",
"nest_asyncio.apply()\n",
"\n",
"SPACE_ID = globals().get(\"SPACE_ID\") or getpass(\n",
" \"🔑 Enter your Arize Space ID: \"\n",
Expand Down Expand Up @@ -442,15 +439,12 @@
"metadata": {},
"outputs": [],
"source": [
"import nest_asyncio\n",
"import pandas as pd\n",
"\n",
"nest_asyncio.apply()\n",
"from phoenix.evals import OpenAIModel\n",
"from phoenix.evals import LLM\n",
"\n",
"pd.set_option(\"display.max_colwidth\", 500)\n",
"\n",
"model = OpenAIModel(model=\"gpt-4o\", max_tokens=1300)"
"model = LLM(provider=\"openai\", model=\"gpt-5.4-mini\")"
]
},
{
Expand All @@ -459,7 +453,7 @@
"metadata": {},
"outputs": [],
"source": [
"resp = model(GEN_TEMPLATE)"
"resp = model.generate_text(GEN_TEMPLATE)"
]
},
{
Expand All @@ -471,6 +465,8 @@
"split_response = resp.strip().split(\"\\n\")\n",
"\n",
"questions_df = pd.DataFrame(split_response, columns=[\"question\"])\n",
"# Unique key per row: questions can repeat, so we join example IDs on this, not the text\n",
"questions_df[\"row_key\"] = questions_df.index.astype(str)\n",
"print(questions_df)"
]
},
Expand Down Expand Up @@ -615,7 +611,7 @@
"id": "1aivaxTCRQFl"
},
"source": [
"Let's run evaluations using Phoenix's llm_classify function for our responses dataframe we generated above!"
"Let's run evaluations using Phoenix's `create_classifier` and `evaluate_dataframe` functions for our responses dataframe we generated above!"
]
},
{
Expand All @@ -624,38 +620,47 @@
"metadata": {},
"outputs": [],
"source": [
"from phoenix.evals import OpenAIModel, llm_classify\n",
"from phoenix.evals import LLM, create_classifier, evaluate_dataframe\n",
"\n",
"# create_classifier needs a tool-calling / structured-output model\n",
"judge = LLM(provider=\"openai\", model=\"gpt-4.1\")\n",
"\n",
"choices = {\"correct\": 1.0, \"incorrect\": 0.0}\n",
"\n",
"router_classifier = create_classifier(\n",
" name=\"router_eval\",\n",
" prompt_template=ROUTER_EVAL_TEMPLATE,\n",
" llm=judge,\n",
" choices=choices,\n",
" direction=\"maximize\",\n",
")\n",
"\n",
"rails = [\"incorrect\", \"correct\"]\n",
"function_selection_classifier = create_classifier(\n",
" name=\"function_selection_eval\",\n",
" prompt_template=FUNCTION_SELECTION_EVAL_TEMPLATE,\n",
" llm=judge,\n",
" choices=choices,\n",
" direction=\"maximize\",\n",
")\n",
"\n",
"router_eval_df = llm_classify(\n",
" dataframe=response_df,\n",
" template=ROUTER_EVAL_TEMPLATE,\n",
" model=OpenAIModel(model=\"gpt-4o\"),\n",
" rails=rails,\n",
" provide_explanation=True,\n",
" include_prompt=True,\n",
" concurrency=4,\n",
"parameter_extraction_classifier = create_classifier(\n",
" name=\"parameter_extraction_eval\",\n",
" prompt_template=PARAMETER_EXTRACTION_EVAL_TEMPLATE,\n",
" llm=judge,\n",
" choices=choices,\n",
" direction=\"maximize\",\n",
")\n",
"\n",
"function_selection_eval_df = llm_classify(\n",
" dataframe=response_df,\n",
" template=FUNCTION_SELECTION_EVAL_TEMPLATE,\n",
" model=OpenAIModel(model=\"gpt-4o\"),\n",
" rails=rails,\n",
" provide_explanation=True,\n",
" include_prompt=True,\n",
" concurrency=4,\n",
"router_eval_df = evaluate_dataframe(\n",
" dataframe=response_df, evaluators=[router_classifier]\n",
")\n",
"\n",
"parameter_extraction_eval_df = llm_classify(\n",
" dataframe=response_df,\n",
" template=PARAMETER_EXTRACTION_EVAL_TEMPLATE,\n",
" model=OpenAIModel(model=\"gpt-4o\"),\n",
" rails=rails,\n",
" provide_explanation=True,\n",
" include_prompt=True,\n",
" concurrency=4,\n",
"function_selection_eval_df = evaluate_dataframe(\n",
" dataframe=response_df, evaluators=[function_selection_classifier]\n",
")\n",
"\n",
"parameter_extraction_eval_df = evaluate_dataframe(\n",
" dataframe=response_df, evaluators=[parameter_extraction_classifier]\n",
")"
]
},
Expand Down Expand Up @@ -721,26 +726,23 @@
"metadata": {},
"outputs": [],
"source": [
"from arize.experimental.datasets import ArizeDatasetsClient\n",
"from uuid import uuid1\n",
"from arize.experimental.datasets.experiments.types import (\n",
" ExperimentTaskResultColumnNames,\n",
" EvaluationResultColumnNames,\n",
"from arize import ArizeClient\n",
"from arize.experiments import (\n",
" ExperimentTaskFieldNames,\n",
" EvaluationResultFieldNames,\n",
")\n",
"from arize.experimental.datasets.utils.constants import GENERATIVE\n",
"from uuid import uuid1\n",
"\n",
"# Set up the arize client\n",
"arize_client = ArizeDatasetsClient(api_key=API_KEY)\n",
"arize_client = ArizeClient(api_key=API_KEY)\n",
"\n",
"dataset_name = \"agents-cookbook-\" + str(uuid1())[:5]\n",
"DATASET_NAME = \"agents-cookbook-\" + str(uuid1())[:5]\n",
"\n",
"dataset_id = arize_client.create_dataset(\n",
" space_id=SPACE_ID,\n",
" dataset_name=dataset_name,\n",
" dataset_type=GENERATIVE,\n",
" data=questions_df,\n",
"dataset = arize_client.datasets.create(\n",
" name=DATASET_NAME,\n",
" space=SPACE_ID,\n",
" examples=questions_df,\n",
")\n",
"dataset = arize_client.get_dataset(space_id=SPACE_ID, dataset_id=dataset_id)\n",
"print(dataset)"
]
},
Expand All @@ -750,34 +752,52 @@
"metadata": {},
"outputs": [],
"source": [
"# Map the evaluation results to the dataset\n",
"response_df[\"example_id\"] = dataset[\"id\"]\n",
"\n",
"response_df[\"router_eval_label\"] = router_eval_df[\"label\"]\n",
"response_df[\"router_eval_explanation\"] = router_eval_df[\"explanation\"]\n",
"response_df[\"parameter_eval_label\"] = parameter_extraction_eval_df[\"label\"]\n",
"# Assign each evaluator's label/explanation (index-aligned with response_df)\n",
"response_df[\"router_eval_label\"] = router_eval_df[\"router_eval_score\"].apply(\n",
" lambda r: r[\"label\"]\n",
")\n",
"response_df[\"router_eval_explanation\"] = router_eval_df[\n",
" \"router_eval_score\"\n",
"].apply(lambda r: r[\"explanation\"])\n",
"response_df[\"parameter_eval_label\"] = parameter_extraction_eval_df[\n",
" \"parameter_extraction_eval_score\"\n",
"].apply(lambda r: r[\"label\"])\n",
"response_df[\"parameter_eval_explanation\"] = parameter_extraction_eval_df[\n",
" \"explanation\"\n",
"]\n",
"response_df[\"function_eval_label\"] = function_selection_eval_df[\"label\"]\n",
" \"parameter_extraction_eval_score\"\n",
"].apply(lambda r: r[\"explanation\"])\n",
"response_df[\"function_eval_label\"] = function_selection_eval_df[\n",
" \"function_selection_eval_score\"\n",
"].apply(lambda r: r[\"label\"])\n",
"response_df[\"function_eval_explanation\"] = function_selection_eval_df[\n",
" \"explanation\"\n",
"]\n",
" \"function_selection_eval_score\"\n",
"].apply(lambda r: r[\"explanation\"])\n",
"\n",
"# Fetch the dataset's server-assigned example IDs and attach them by the unique\n",
"# row_key (question text repeats, so joining on it would duplicate rows).\n",
"examples = arize_client.datasets.list_examples(\n",
" dataset=DATASET_NAME, space=SPACE_ID, all=True\n",
")\n",
"examples_df = pd.DataFrame(\n",
" [{**ex.to_dict(), \"example_id\": ex.id} for ex in examples.examples]\n",
")\n",
"response_df = response_df.merge(\n",
" examples_df[[\"row_key\", \"example_id\"]], on=\"row_key\", how=\"left\"\n",
")\n",
"\n",
"# Define column mappings for task\n",
"task_cols = ExperimentTaskResultColumnNames(\n",
" example_id=\"example_id\", result=\"response\"\n",
"task_cols = ExperimentTaskFieldNames(\n",
" example_id=\"example_id\", output=\"response\"\n",
")\n",
"# Define column mappings for evaluator\n",
"router_evaluator_cols = EvaluationResultColumnNames(\n",
"router_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"router_eval_label\",\n",
" explanation=\"router_eval_explanation\",\n",
")\n",
"parameter_evaluator_cols = EvaluationResultColumnNames(\n",
"parameter_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"parameter_eval_label\",\n",
" explanation=\"parameter_eval_explanation\",\n",
")\n",
"function_evaluator_cols = EvaluationResultColumnNames(\n",
"function_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"function_eval_label\",\n",
" explanation=\"function_eval_explanation\",\n",
")"
Expand All @@ -789,18 +809,18 @@
"metadata": {},
"outputs": [],
"source": [
"# Use with ArizeDatasetsClient.log_experiment()\n",
"arize_client.log_experiment(\n",
" space_id=SPACE_ID,\n",
" experiment_name=\"my_experiment\" + str(uuid1())[:5],\n",
" experiment_df=response_df,\n",
" task_columns=task_cols,\n",
"# Log the precomputed experiment runs to Arize\n",
"arize_client.experiments.create(\n",
" space=SPACE_ID,\n",
" name=\"my_experiment\" + str(uuid1())[:5],\n",
" dataset=DATASET_NAME,\n",
" experiment_runs=response_df,\n",
" task_fields=task_cols,\n",
" evaluator_columns={\n",
" \"router\": router_evaluator_cols,\n",
" \"parameter_extraction\": parameter_evaluator_cols,\n",
" \"function_selection\": function_evaluator_cols,\n",
" },\n",
" dataset_name=dataset_name,\n",
")"
]
}
Expand Down
Loading