diff --git a/python/llm/agents/agents-cookbook.ipynb b/python/llm/agents/agents-cookbook.ipynb
index eb8d573..ef5fb4a 100644
--- a/python/llm/agents/agents-cookbook.ipynb
+++ b/python/llm/agents/agents-cookbook.ipynb
@@ -60,9 +60,9 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install -qq arize-otel openinference-instrumentation-openai arize-phoenix-evals \"arize[Datasets]\"\n",
+ "!pip install -qq arize-otel openinference-instrumentation-openai arize-phoenix-evals \"arize>=8.0.0\"\n",
"\n",
- "!pip install -qq openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs nest_asyncio"
+ "!pip install -qq openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs"
]
},
{
@@ -82,9 +82,6 @@
"source": [
"import os\n",
"from getpass import getpass\n",
- "import nest_asyncio\n",
- "\n",
- "nest_asyncio.apply()\n",
"\n",
"SPACE_ID = globals().get(\"SPACE_ID\") or getpass(\n",
" \"🔑 Enter your Arize Space ID: \"\n",
@@ -442,15 +439,12 @@
"metadata": {},
"outputs": [],
"source": [
- "import nest_asyncio\n",
"import pandas as pd\n",
- "\n",
- "nest_asyncio.apply()\n",
- "from phoenix.evals import OpenAIModel\n",
+ "from phoenix.evals import LLM\n",
"\n",
"pd.set_option(\"display.max_colwidth\", 500)\n",
"\n",
- "model = OpenAIModel(model=\"gpt-4o\", max_tokens=1300)"
+ "model = LLM(provider=\"openai\", model=\"gpt-5.4-mini\")"
]
},
{
@@ -459,7 +453,7 @@
"metadata": {},
"outputs": [],
"source": [
- "resp = model(GEN_TEMPLATE)"
+ "resp = model.generate_text(GEN_TEMPLATE)"
]
},
{
@@ -471,6 +465,8 @@
"split_response = resp.strip().split(\"\\n\")\n",
"\n",
"questions_df = pd.DataFrame(split_response, columns=[\"question\"])\n",
+ "# Unique key per row: questions can repeat, so we join example IDs on this, not the text\n",
+ "questions_df[\"row_key\"] = questions_df.index.astype(str)\n",
"print(questions_df)"
]
},
@@ -615,7 +611,7 @@
"id": "1aivaxTCRQFl"
},
"source": [
- "Let's run evaluations using Phoenix's llm_classify function for our responses dataframe we generated above!"
+ "Let's run evaluations using Phoenix's `create_classifier` and `evaluate_dataframe` functions for our responses dataframe we generated above!"
]
},
{
@@ -624,38 +620,47 @@
"metadata": {},
"outputs": [],
"source": [
- "from phoenix.evals import OpenAIModel, llm_classify\n",
+ "from phoenix.evals import LLM, create_classifier, evaluate_dataframe\n",
+ "\n",
+ "# create_classifier needs a tool-calling / structured-output model\n",
+ "judge = LLM(provider=\"openai\", model=\"gpt-4.1\")\n",
+ "\n",
+ "choices = {\"correct\": 1.0, \"incorrect\": 0.0}\n",
+ "\n",
+ "router_classifier = create_classifier(\n",
+ " name=\"router_eval\",\n",
+ " prompt_template=ROUTER_EVAL_TEMPLATE,\n",
+ " llm=judge,\n",
+ " choices=choices,\n",
+ " direction=\"maximize\",\n",
+ ")\n",
"\n",
- "rails = [\"incorrect\", \"correct\"]\n",
+ "function_selection_classifier = create_classifier(\n",
+ " name=\"function_selection_eval\",\n",
+ " prompt_template=FUNCTION_SELECTION_EVAL_TEMPLATE,\n",
+ " llm=judge,\n",
+ " choices=choices,\n",
+ " direction=\"maximize\",\n",
+ ")\n",
"\n",
- "router_eval_df = llm_classify(\n",
- " dataframe=response_df,\n",
- " template=ROUTER_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=rails,\n",
- " provide_explanation=True,\n",
- " include_prompt=True,\n",
- " concurrency=4,\n",
+ "parameter_extraction_classifier = create_classifier(\n",
+ " name=\"parameter_extraction_eval\",\n",
+ " prompt_template=PARAMETER_EXTRACTION_EVAL_TEMPLATE,\n",
+ " llm=judge,\n",
+ " choices=choices,\n",
+ " direction=\"maximize\",\n",
")\n",
"\n",
- "function_selection_eval_df = llm_classify(\n",
- " dataframe=response_df,\n",
- " template=FUNCTION_SELECTION_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=rails,\n",
- " provide_explanation=True,\n",
- " include_prompt=True,\n",
- " concurrency=4,\n",
+ "router_eval_df = evaluate_dataframe(\n",
+ " dataframe=response_df, evaluators=[router_classifier]\n",
")\n",
"\n",
- "parameter_extraction_eval_df = llm_classify(\n",
- " dataframe=response_df,\n",
- " template=PARAMETER_EXTRACTION_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=rails,\n",
- " provide_explanation=True,\n",
- " include_prompt=True,\n",
- " concurrency=4,\n",
+ "function_selection_eval_df = evaluate_dataframe(\n",
+ " dataframe=response_df, evaluators=[function_selection_classifier]\n",
+ ")\n",
+ "\n",
+ "parameter_extraction_eval_df = evaluate_dataframe(\n",
+ " dataframe=response_df, evaluators=[parameter_extraction_classifier]\n",
")"
]
},
@@ -721,26 +726,23 @@
"metadata": {},
"outputs": [],
"source": [
- "from arize.experimental.datasets import ArizeDatasetsClient\n",
- "from uuid import uuid1\n",
- "from arize.experimental.datasets.experiments.types import (\n",
- " ExperimentTaskResultColumnNames,\n",
- " EvaluationResultColumnNames,\n",
+ "from arize import ArizeClient\n",
+ "from arize.experiments import (\n",
+ " ExperimentTaskFieldNames,\n",
+ " EvaluationResultFieldNames,\n",
")\n",
- "from arize.experimental.datasets.utils.constants import GENERATIVE\n",
+ "from uuid import uuid1\n",
"\n",
"# Set up the arize client\n",
- "arize_client = ArizeDatasetsClient(api_key=API_KEY)\n",
+ "arize_client = ArizeClient(api_key=API_KEY)\n",
"\n",
- "dataset_name = \"agents-cookbook-\" + str(uuid1())[:5]\n",
+ "DATASET_NAME = \"agents-cookbook-\" + str(uuid1())[:5]\n",
"\n",
- "dataset_id = arize_client.create_dataset(\n",
- " space_id=SPACE_ID,\n",
- " dataset_name=dataset_name,\n",
- " dataset_type=GENERATIVE,\n",
- " data=questions_df,\n",
+ "dataset = arize_client.datasets.create(\n",
+ " name=DATASET_NAME,\n",
+ " space=SPACE_ID,\n",
+ " examples=questions_df,\n",
")\n",
- "dataset = arize_client.get_dataset(space_id=SPACE_ID, dataset_id=dataset_id)\n",
"print(dataset)"
]
},
@@ -750,34 +752,52 @@
"metadata": {},
"outputs": [],
"source": [
- "# Map the evaluation results to the dataset\n",
- "response_df[\"example_id\"] = dataset[\"id\"]\n",
- "\n",
- "response_df[\"router_eval_label\"] = router_eval_df[\"label\"]\n",
- "response_df[\"router_eval_explanation\"] = router_eval_df[\"explanation\"]\n",
- "response_df[\"parameter_eval_label\"] = parameter_extraction_eval_df[\"label\"]\n",
+ "# Assign each evaluator's label/explanation (index-aligned with response_df)\n",
+ "response_df[\"router_eval_label\"] = router_eval_df[\"router_eval_score\"].apply(\n",
+ " lambda r: r[\"label\"]\n",
+ ")\n",
+ "response_df[\"router_eval_explanation\"] = router_eval_df[\n",
+ " \"router_eval_score\"\n",
+ "].apply(lambda r: r[\"explanation\"])\n",
+ "response_df[\"parameter_eval_label\"] = parameter_extraction_eval_df[\n",
+ " \"parameter_extraction_eval_score\"\n",
+ "].apply(lambda r: r[\"label\"])\n",
"response_df[\"parameter_eval_explanation\"] = parameter_extraction_eval_df[\n",
- " \"explanation\"\n",
- "]\n",
- "response_df[\"function_eval_label\"] = function_selection_eval_df[\"label\"]\n",
+ " \"parameter_extraction_eval_score\"\n",
+ "].apply(lambda r: r[\"explanation\"])\n",
+ "response_df[\"function_eval_label\"] = function_selection_eval_df[\n",
+ " \"function_selection_eval_score\"\n",
+ "].apply(lambda r: r[\"label\"])\n",
"response_df[\"function_eval_explanation\"] = function_selection_eval_df[\n",
- " \"explanation\"\n",
- "]\n",
+ " \"function_selection_eval_score\"\n",
+ "].apply(lambda r: r[\"explanation\"])\n",
+ "\n",
+ "# Fetch the dataset's server-assigned example IDs and attach them by the unique\n",
+ "# row_key (question text repeats, so joining on it would duplicate rows).\n",
+ "examples = arize_client.datasets.list_examples(\n",
+ " dataset=DATASET_NAME, space=SPACE_ID, all=True\n",
+ ")\n",
+ "examples_df = pd.DataFrame(\n",
+ " [{**ex.to_dict(), \"example_id\": ex.id} for ex in examples.examples]\n",
+ ")\n",
+ "response_df = response_df.merge(\n",
+ " examples_df[[\"row_key\", \"example_id\"]], on=\"row_key\", how=\"left\"\n",
+ ")\n",
"\n",
"# Define column mappings for task\n",
- "task_cols = ExperimentTaskResultColumnNames(\n",
- " example_id=\"example_id\", result=\"response\"\n",
+ "task_cols = ExperimentTaskFieldNames(\n",
+ " example_id=\"example_id\", output=\"response\"\n",
")\n",
"# Define column mappings for evaluator\n",
- "router_evaluator_cols = EvaluationResultColumnNames(\n",
+ "router_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"router_eval_label\",\n",
" explanation=\"router_eval_explanation\",\n",
")\n",
- "parameter_evaluator_cols = EvaluationResultColumnNames(\n",
+ "parameter_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"parameter_eval_label\",\n",
" explanation=\"parameter_eval_explanation\",\n",
")\n",
- "function_evaluator_cols = EvaluationResultColumnNames(\n",
+ "function_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"function_eval_label\",\n",
" explanation=\"function_eval_explanation\",\n",
")"
@@ -789,18 +809,18 @@
"metadata": {},
"outputs": [],
"source": [
- "# Use with ArizeDatasetsClient.log_experiment()\n",
- "arize_client.log_experiment(\n",
- " space_id=SPACE_ID,\n",
- " experiment_name=\"my_experiment\" + str(uuid1())[:5],\n",
- " experiment_df=response_df,\n",
- " task_columns=task_cols,\n",
+ "# Log the precomputed experiment runs to Arize\n",
+ "arize_client.experiments.create(\n",
+ " space=SPACE_ID,\n",
+ " name=\"my_experiment\" + str(uuid1())[:5],\n",
+ " dataset=DATASET_NAME,\n",
+ " experiment_runs=response_df,\n",
+ " task_fields=task_cols,\n",
" evaluator_columns={\n",
" \"router\": router_evaluator_cols,\n",
" \"parameter_extraction\": parameter_evaluator_cols,\n",
" \"function_selection\": function_evaluator_cols,\n",
" },\n",
- " dataset_name=dataset_name,\n",
")"
]
}
diff --git a/python/llm/agents/couchbase_langgraph_agentic_rag.ipynb b/python/llm/agents/couchbase_langgraph_agentic_rag.ipynb
index 32b5954..2b8660e 100644
--- a/python/llm/agents/couchbase_langgraph_agentic_rag.ipynb
+++ b/python/llm/agents/couchbase_langgraph_agentic_rag.ipynb
@@ -5,46 +5,7 @@
"id": "425fb020-e864-40ce-a31f-8da40c73d14b",
"metadata": {},
"source": [
- "\n",
- " \n",
- " ![\"arize](\"https://storage.googleapis.com/arize-assets/arize-logo-white.jpg\")
\n",
- "
\n",
- " Docs\n",
- " |\n",
- " GitHub\n",
- " |\n",
- " Community\n",
- "
\n",
- "\n",
- "\n",
- "Evaluating Agentic RAG using Arize + Couchbase
\n",
- "\n",
- "\n",
- "This tutorial is adapted from the [Langgraph Agentic RAG notebook](https://github.com/langchain-ai/langgraph/blob/main/examples/rag/langgraph_agentic_rag.ipynb).\n",
- "\n",
- "\n",
- "This guide shows you how to create a Retrieval Augmented Generation (RAG) Agent using Couchbase Vectorstore and evaluate performance with Arize. Agentic RAG combines RAG with the power of agents. [Retrieval Agents](https://python.langchain.com/v0.2/docs/tutorials/qa_chat_history/#agents) are useful when we want to make decisions about whether to retrieve from an index. To implement a Retrieval Agent, we simply need to give an LLM access to a retriever tool.\n",
- "\n",
- "We'll go through the following steps:\n",
- "\n",
- "* Create a Agentic RAG QA chatbot with OpenAI, Langgraph, Couchbase and Agent Catalog\n",
- "\n",
- "* Trace the agent's function calls including retrieval and LLM calls using Arize\n",
- "\n",
- "* Create a dataset to benchmark performance\n",
- "\n",
- "* Evaluate performance using LLM as a judge\n",
- "\n",
- "* Experiment with different chunk sizes, overlaps, and k number of documents retrieved to see how these affect the performance of the Agentic RAG\n",
- "\n",
- "* Compare these experiments in Arize\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "# Notebook Setup\n",
- "\n",
- "First, let's download the required packages and set our API keys:"
+ "Define the retriever tool directly from the Couchbase vector store, and the relevance-grading and RAG prompts inline. (An earlier version of this cookbook fetched these from Couchbase [Agent Catalog](https://github.com/couchbaselabs/agent-catalog); inlining them keeps the notebook self-contained.)"
]
},
{
@@ -54,9 +15,8 @@
"metadata": {},
"outputs": [],
"source": [
- "%pip install -qU langchain-openai langchain-community langchain langgraph langgraph.prebuilt openai langchain-couchbase agentc langchain-huggingface langchain_core\n",
- "\n",
- "%pip install -qq \"arize-phoenix[evals]\" arize-otel openinference-instrumentation-openai openinference-instrumentation-langchain\n"
+ "%pip install -qU langchain-openai langchain-community langchain langgraph openai langchain-couchbase langchain-huggingface langchain_core sentence-transformers beautifulsoup4 pydantic\n",
+ "%pip install -qq \"arize-phoenix[evals]\" \"arize>=8.0.0\" arize-otel openinference-instrumentation-openai openinference-instrumentation-langchain"
]
},
{
@@ -180,7 +140,7 @@
"from couchbase.auth import PasswordAuthenticator\n",
"from couchbase.cluster import Cluster\n",
"from couchbase.options import ClusterOptions\n",
- "from langchain_couchbase.vectorstores import CouchbaseVectorStore\n",
+ "from langchain_couchbase.vectorstores import CouchbaseSearchVectorStore\n",
"from langchain_huggingface import HuggingFaceEmbeddings\n",
"\n",
"#Cluster settings\n",
@@ -202,7 +162,7 @@
"\n",
"#Initialize vector store\n",
"embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L12-v2\")\n",
- "vector_store = CouchbaseVectorStore(\n",
+ "vector_store = CouchbaseSearchVectorStore(\n",
" cluster=cluster,\n",
" bucket_name=BUCKET_NAME,\n",
" scope_name=SCOPE_NAME,\n",
@@ -229,52 +189,53 @@
"source": [
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+ "import time\n",
+ "\n",
"\n",
- "# Define the reset_vector_store function so we can run experiments with different chunk sizes\n",
"def reset_vector_store(vector_store, chunk_size=1024, chunk_overlap=20):\n",
- " try: \n",
+ " # Clear any documents from a previous run so we can re-ingest at a new chunk size\n",
+ " try:\n",
" results = vector_store.similarity_search(\n",
" k=1000,\n",
- " query=\"\", # Use an empty query or a specific one if needed\n",
+ " query=\"\",\n",
" search_options={\n",
" \"query\": {\"field\": \"metadata.source\", \"match\": \"lilian_weng_blog\"}\n",
" },\n",
" )\n",
" if results:\n",
- " deleted_ids = []\n",
- " for result in results:\n",
- " deleted_ids.append(result.id)\n",
- " vector_store.delete(ids=deleted_ids)\n",
- " # Load documents from a URL or file\n",
- " urls = [\n",
- " \"https://lilianweng.github.io/posts/2024-07-07-hallucination/\",\n",
- " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
- " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
- " ]\n",
- " docs = [WebBaseLoader(url).load() for url in urls]\n",
- " docs_list = [item for sublist in docs for item in sublist]\n",
- "\n",
- " # Use RecursiveCharacterTextSplitter\n",
- " text_splitter = RecursiveCharacterTextSplitter(\n",
- " chunk_size=chunk_size,\n",
- " chunk_overlap=chunk_overlap,\n",
- " separators=[\"\\n\\n\", \"\\n\", \" \", \"\"], # Hierarchical separators\n",
- " )\n",
- " doc_splits = text_splitter.split_documents(docs_list)\n",
- "\n",
- " # Adding metadata to documents\n",
- " for i, doc in enumerate(doc_splits):\n",
- " doc.metadata[\"source\"] = \"lilian_weng_blog\"\n",
- " try:\n",
- " vector_store.add_documents(doc_splits)\n",
- " except ValueError as e:\n",
- " print(f\"Failed to insert documents: {e}\")\n",
- " return vector_store\n",
+ " vector_store.delete(ids=[r.id for r in results])\n",
" except ValueError as e:\n",
" print(f\"Search failed with error: {e}\")\n",
"\n",
+ " # Load, split, and ingest the source documents\n",
+ " urls = [\n",
+ " \"https://lilianweng.github.io/posts/2024-07-07-hallucination/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ " ]\n",
+ " docs = [WebBaseLoader(url).load() for url in urls]\n",
+ " docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ " text_splitter = RecursiveCharacterTextSplitter(\n",
+ " chunk_size=chunk_size,\n",
+ " chunk_overlap=chunk_overlap,\n",
+ " separators=[\"\\n\\n\", \"\\n\", \" \", \"\"],\n",
+ " )\n",
+ " doc_splits = text_splitter.split_documents(docs_list)\n",
+ " for doc in doc_splits:\n",
+ " doc.metadata[\"source\"] = \"lilian_weng_blog\"\n",
+ "\n",
+ " try:\n",
+ " vector_store.add_documents(doc_splits)\n",
+ " # Couchbase FTS is eventually consistent; give the index a moment to catch up\n",
+ " time.sleep(5)\n",
+ " except ValueError as e:\n",
+ " print(f\"Failed to insert documents: {e}\")\n",
+ " return vector_store\n",
+ "\n",
+ "\n",
"# Reset the vector store\n",
- "reset_vector_store(vector_store)\n"
+ "reset_vector_store(vector_store)"
]
},
{
@@ -290,7 +251,7 @@
"id": "c2b05193",
"metadata": {},
"source": [
- "### Create tools and prompts with Agent Catalog"
+ "### Create the retriever tool and grading prompt"
]
},
{
@@ -298,9 +259,7 @@
"id": "225d2277-45b2-4ae8-a7d6-62b07fb4a002",
"metadata": {},
"source": [
- "Fetch our retriever tool from the Agent Catalog using the agentc provider. In the future, when more tools (and/or prompts) are required and the application grows more complex, Agent Catalog SDK and CLI can be used to automatically fetch the tools based on the use case (semantic search) or by name.\n",
- "\n",
- "For instructions on how this tool was created and more capabilities of Agent catalog, please refer to the documentation [here](https://couchbaselabs.github.io/agent-catalog/index.html)."
+ "Define the retriever tool directly from the Couchbase vector store, and the relevance-grading and RAG prompts inline. (An earlier version of this cookbook fetched these from Couchbase [Agent Catalog](https://github.com/couchbaselabs/agent-catalog); inlining them keeps the notebook self-contained.)"
]
},
{
@@ -310,29 +269,20 @@
"metadata": {},
"outputs": [],
"source": [
- "import agentc.langchain\n",
- "import agentc\n",
- "from langchain_core.tools import tool\n",
- "import os\n",
- "\n",
- "# For retrieval from the local catalog\n",
- "provider = agentc.Provider(\n",
- " decorator=lambda t: tool(t.func)\n",
+ "from langchain_core.tools.retriever import create_retriever_tool\n",
+ "\n",
+ "# Build the retriever tool directly from the Couchbase vector store.\n",
+ "# (Previously fetched from Couchbase Agent Catalog via the agentc provider.)\n",
+ "retriever = vector_store.as_retriever(search_kwargs={\"k\": 2})\n",
+ "retriever_tool = create_retriever_tool(\n",
+ " retriever,\n",
+ " \"retrieve_blog_posts\",\n",
+ " \"Search and return information from Lilian Weng's blog posts on LLM \"\n",
+ " \"hallucination, prompt engineering, and adversarial attacks on LLMs.\",\n",
")\n",
+ "tools = [retriever_tool]\n",
"\n",
- "# In case the tools were published to the Couchbase cluster beforehand\n",
- "# provider = agentc.Provider(\n",
- "# decorator=lambda t: tool(t.func),\n",
- "# secrets={\"CB_USERNAME\": CB_USERNAME,\n",
- "# \"CB_PASSWORD\": CB_PASSWORD,\n",
- "# \"CB_CONN_STRING\": CB_CONN_STRING})\n",
- "\n",
- "# This is the tool that will be used to retrieve documents from the vector store\n",
- "retriever_tool = provider.get_item(name=\"retriever_tool\", item_type=\"tool\")\n",
- "\n",
- "tools = retriever_tool\n",
- "\n",
- "print (retriever_tool)\n"
+ "print(retriever_tool)"
]
},
{
@@ -391,18 +341,18 @@
"metadata": {},
"outputs": [],
"source": [
- "from typing import Annotated, Literal, Sequence, TypedDict\n",
+ "from typing import Literal\n",
"\n",
- "from langchain import hub\n",
- "from langchain_core.messages import BaseMessage, HumanMessage\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "from langchain_core.messages import HumanMessage\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"from langchain_core.prompts import PromptTemplate\n",
- "from langchain_core.pydantic_v1 import BaseModel, Field\n",
- "from langchain_openai import ChatOpenAI\n",
+ "from pydantic import BaseModel, Field\n",
"from langgraph.prebuilt import tools_condition\n",
"\n",
- "### Edges\n",
"\n",
+ "### Edges\n",
"def grade_documents(state) -> Literal[\"generate\", \"rewrite\"]:\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
@@ -413,7 +363,6 @@
" Returns:\n",
" str: A decision for whether the documents are relevant or not\n",
" \"\"\"\n",
- "\n",
" print(\"---CHECK RELEVANCE---\")\n",
"\n",
" # Data model\n",
@@ -423,19 +372,24 @@
" binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n",
"\n",
" # LLM\n",
- " model = ChatOpenAI(temperature=0, model=\"gpt-4o\", streaming=True)\n",
+ " model = ChatOpenAI(temperature=0, model=\"gpt-4.1\", streaming=True)\n",
"\n",
" # LLM with tool and validation\n",
" llm_with_tool = model.with_structured_output(grade)\n",
"\n",
- "\n",
- " #fetch a prompt called \"grade_documents\" from the Agent Catalog\n",
+ " # Relevance-grading prompt (defined inline; previously fetched from Agent Catalog)\n",
" grade_documents_prompt = PromptTemplate(\n",
- " template=provider.get_item(name=\"grade_documents\", item_type=\"prompt\").prompt.render(),\n",
- " input_variables=[\"context\", \"question\"],\n",
- " )\n",
- "\n",
- " print (grade_documents_prompt)\n",
+ " template=(\n",
+ " \"You are a grader assessing relevance of a retrieved document to a user \"\n",
+ " \"question.\\n\\n\"\n",
+ " \"Retrieved document:\\n{context}\\n\\n\"\n",
+ " \"User question: {question}\\n\\n\"\n",
+ " \"If the document contains keyword(s) or semantic meaning related to the \"\n",
+ " \"user question, grade it as relevant. Give a binary score 'yes' or 'no' \"\n",
+ " \"to indicate whether the document is relevant to the question.\"\n",
+ " ),\n",
+ " input_variables=[\"context\", \"question\"],\n",
+ " )\n",
"\n",
" # Chain\n",
" chain = grade_documents_prompt | llm_with_tool\n",
@@ -447,13 +401,11 @@
" docs = last_message.content\n",
"\n",
" scored_result = chain.invoke({\"question\": question, \"context\": docs})\n",
- "\n",
" score = scored_result.binary_score\n",
"\n",
" if score == \"yes\":\n",
" print(\"---DECISION: DOCS RELEVANT---\")\n",
" return \"generate\"\n",
- "\n",
" else:\n",
" print(\"---DECISION: DOCS NOT RELEVANT---\")\n",
" print(score)\n",
@@ -461,97 +413,64 @@
"\n",
"\n",
"### Nodes\n",
- "\n",
"def agent(state):\n",
" \"\"\"\n",
- " Invokes the agent model to generate a response based on the current state. Given\n",
- " the question, it will decide to retrieve using the retriever tool, or simply end.\n",
- "\n",
- " Args:\n",
- " state (messages): The current state\n",
- "\n",
- " Returns:\n",
- " dict: The updated state with the agent response appended to messages\n",
+ " Invokes the agent model to decide whether to retrieve using the retriever\n",
+ " tool, or simply end.\n",
" \"\"\"\n",
" print(\"---CALL AGENT---\")\n",
" messages = state[\"messages\"]\n",
- " model = ChatOpenAI(temperature=0, streaming=True, model=\"gpt-4-turbo\")\n",
+ " model = ChatOpenAI(temperature=0, streaming=True, model=\"gpt-4.1\")\n",
" model = model.bind_tools(tools)\n",
" response = model.invoke(messages)\n",
- " # We return a list, because this will get added to the existing list\n",
" return {\"messages\": [response]}\n",
"\n",
"\n",
"def rewrite(state):\n",
- " \"\"\"\n",
- " Transform the query to produce a better question.\n",
- "\n",
- " Args:\n",
- " state (messages): The current state\n",
- "\n",
- " Returns:\n",
- " dict: The updated state with re-phrased question\n",
- " \"\"\"\n",
- "\n",
+ " \"\"\"Transform the query to produce a better question.\"\"\"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" messages = state[\"messages\"]\n",
" question = messages[0].content\n",
"\n",
" msg = [\n",
" HumanMessage(\n",
- " content=f\"\"\" \\n \n",
- " Look at the input and try to reason about the underlying semantic intent / meaning. \\n \n",
- " Here is the initial question:\n",
- " \\n ------- \\n\n",
- " {question} \n",
- " \\n ------- \\n\n",
- " Formulate an improved question: \"\"\",\n",
+ " content=f\"\"\"Look at the input and try to reason about the underlying semantic intent / meaning.\n",
+ "Here is the initial question:\n",
+ "------\n",
+ "{question}\n",
+ "------\n",
+ "Formulate an improved question:\"\"\",\n",
" )\n",
" ]\n",
"\n",
- " # Grader\n",
- " model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
+ " model = ChatOpenAI(temperature=0, model=\"gpt-4.1\", streaming=True)\n",
" response = model.invoke(msg)\n",
" return {\"messages\": [response]}\n",
"\n",
"\n",
"def generate(state):\n",
- " \"\"\"\n",
- " Generate answer\n",
- "\n",
- " Args:\n",
- " state (messages): The current state\n",
- "\n",
- " Returns:\n",
- " dict: The updated state with re-phrased question\n",
- " \"\"\"\n",
+ " \"\"\"Generate an answer from the retrieved documents.\"\"\"\n",
" print(\"---GENERATE---\")\n",
" messages = state[\"messages\"]\n",
" question = messages[0].content\n",
" last_message = messages[-1]\n",
- "\n",
" docs = last_message.content\n",
"\n",
- " # Prompt\n",
- " prompt = hub.pull(\"rlm/rag-prompt\")\n",
- "\n",
- " # LLM\n",
- " llm = ChatOpenAI(model_name=\"gpt-4o-mini\", temperature=0, streaming=True)\n",
- " # Post-processing\n",
- " def format_docs(docs):\n",
- " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
- "\n",
- " # Chain\n",
+ " # RAG prompt (defined inline; previously hub.pull(\"rlm/rag-prompt\"))\n",
+ " prompt = PromptTemplate(\n",
+ " template=(\n",
+ " \"You are an assistant for question-answering tasks. Use the following \"\n",
+ " \"pieces of retrieved context to answer the question. If you don't know \"\n",
+ " \"the answer, just say that you don't know. Use three sentences maximum \"\n",
+ " \"and keep the answer concise.\\n\"\n",
+ " \"Question: {question}\\nContext: {context}\\nAnswer:\"\n",
+ " ),\n",
+ " input_variables=[\"context\", \"question\"],\n",
+ " )\n",
+ " llm = ChatOpenAI(model=\"gpt-4.1\", temperature=0, streaming=True)\n",
" rag_chain = prompt | llm | StrOutputParser()\n",
- "\n",
- " # Run\n",
" response = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
- " return {\"messages\": [response]}\n",
- "\n",
- "\n",
- "print(\"*\" * 20 + \"Prompt[rlm/rag-prompt]\" + \"*\" * 20)\n",
- "prompt = hub.pull(\"rlm/rag-prompt\") # Show what the prompt looks like\n",
- "prompt.pretty_print()"
+ " return {\"messages\": [response]}"
]
},
{
@@ -582,7 +501,7 @@
"\n",
"# Define the nodes we will cycle between\n",
"workflow.add_node(\"agent\", agent) # agent\n",
- "retrieve = ToolNode(retriever_tool)\n",
+ "retrieve = ToolNode(tools)\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieval\n",
"workflow.add_node(\"rewrite\", rewrite) # Re-writing the question\n",
"workflow.add_node(\n",
@@ -709,7 +628,6 @@
"outputs": [],
"source": [
"import pandas as pd\n",
- "from langchain import hub\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"# Define a template for generating questions\n",
@@ -737,7 +655,7 @@
"formatted_template = GEN_TEMPLATE.format(content=content)\n",
"\n",
"# Initialize the language model\n",
- "model = ChatOpenAI(model=\"gpt-4o\", max_tokens=1300)\n",
+ "model = ChatOpenAI(model=\"gpt-4.1\", max_tokens=1300)\n",
"\n",
"# Generate questions using the language model\n",
"response = model.invoke(formatted_template)\n",
@@ -752,7 +670,7 @@
"questions_df = pd.DataFrame(questions, columns=[\"input\"])\n",
"\n",
"# Display the first few questions\n",
- "questions_df.head()\n"
+ "questions_df.head()"
]
},
{
@@ -828,7 +746,7 @@
"id": "3062d687",
"metadata": {},
"source": [
- "We will be creating an LLM as a judge using prebuilt prompt templates, taking the spans recorded by Phoenix, and then giving them labels using the `llm_classify` function. This function uses LLMs to evaluate your LLM calls and gives them labels and explanations. You can read more detail [here](https://docs.arize.com/phoenix/api/evals#phoenix.evals.llm_classify)."
+ "We will be creating an LLM as a judge using Phoenix's prebuilt evaluators, taking the spans recorded by Phoenix, and then giving them labels using the `evaluate_dataframe` function. This function uses LLMs to evaluate your LLM calls and gives them scores, labels, and explanations. You can read more detail [here](https://arize.com/docs/phoenix/evaluation/how-to-evals/running-pre-tested-evals)."
]
},
{
@@ -838,37 +756,32 @@
"metadata": {},
"outputs": [],
"source": [
- "from phoenix.evals import (\n",
- " RAG_RELEVANCY_PROMPT_RAILS_MAP,\n",
- " RAG_RELEVANCY_PROMPT_TEMPLATE,\n",
- " QA_PROMPT_RAILS_MAP,\n",
- " QA_PROMPT_TEMPLATE,\n",
- " OpenAIModel,\n",
- " llm_classify\n",
+ "from phoenix.evals import LLM, evaluate_dataframe\n",
+ "from phoenix.evals.metrics import (\n",
+ " CorrectnessEvaluator,\n",
+ " DocumentRelevanceEvaluator,\n",
")\n",
"\n",
- "# The rails is used to hold the output to specific values based on the template\n",
- "RELEVANCE_RAILS = list(RAG_RELEVANCY_PROMPT_RAILS_MAP.values())\n",
- "QA_RAILS = list(QA_PROMPT_RAILS_MAP.values())\n",
+ "# Prebuilt evaluators need a tool-calling / structured-output model\n",
+ "judge = LLM(provider=\"openai\", model=\"gpt-4.1\")\n",
"\n",
- "relevance_eval_df = llm_classify(\n",
+ "# Relevance: is the retrieved reference text relevant to the question?\n",
+ "# The evaluator expects `input` and `document_text`, so map `document_text`\n",
+ "# to our `reference` column.\n",
+ "relevance_evaluator = DocumentRelevanceEvaluator(llm=judge)\n",
+ "relevance_evaluator.bind({\"input\": \"input\", \"document_text\": \"reference\"})\n",
+ "\n",
+ "# Correctness: does the output correctly answer the question?\n",
+ "correctness_evaluator = CorrectnessEvaluator(llm=judge)\n",
+ "\n",
+ "relevance_eval_df = evaluate_dataframe(\n",
" dataframe=response_df,\n",
- " template=RAG_RELEVANCY_PROMPT_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=RELEVANCE_RAILS,\n",
- " provide_explanation=True,\n",
- " include_prompt=True,\n",
- " concurrency=4,\n",
+ " evaluators=[relevance_evaluator],\n",
")\n",
"\n",
- "correctness_eval_df = llm_classify(\n",
+ "correctness_eval_df = evaluate_dataframe(\n",
" dataframe=response_df,\n",
- " template=QA_PROMPT_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=QA_RAILS,\n",
- " provide_explanation=True,\n",
- " include_prompt=True,\n",
- " concurrency=4,\n",
+ " evaluators=[correctness_evaluator],\n",
")"
]
},
@@ -926,27 +839,27 @@
"outputs": [],
"source": [
"def run_evaluators(rag_df):\n",
- " relevance_eval_df = llm_classify(\n",
+ " relevance_eval_df = evaluate_dataframe(\n",
" dataframe=rag_df,\n",
- " template=RELEVANCE_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=RELEVANCE_RAILS,\n",
- " provide_explanation=True,\n",
- " concurrency=4,\n",
+ " evaluators=[relevance_evaluator],\n",
+ " )\n",
+ " rag_df[\"relevance\"] = relevance_eval_df[\"document_relevance_score\"].apply(\n",
+ " lambda r: r[\"label\"]\n",
" )\n",
- " rag_df[\"relevance\"] = relevance_eval_df[\"label\"]\n",
- " rag_df[\"relevance_explanation\"] = relevance_eval_df[\"explanation\"]\n",
+ " rag_df[\"relevance_explanation\"] = relevance_eval_df[\n",
+ " \"document_relevance_score\"\n",
+ " ].apply(lambda r: r[\"explanation\"])\n",
"\n",
- " correctness_eval_df = llm_classify(\n",
+ " correctness_eval_df = evaluate_dataframe(\n",
" dataframe=rag_df,\n",
- " template=CORRECTNESS_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=CORRECTNESS_RAILS,\n",
- " provide_explanation=True,\n",
- " concurrency=4,\n",
+ " evaluators=[correctness_evaluator],\n",
" )\n",
- " rag_df[\"correctness\"] = correctness_eval_df[\"label\"]\n",
- " rag_df[\"correctness_explanation\"] = correctness_eval_df[\"explanation\"]\n",
+ " rag_df[\"correctness\"] = correctness_eval_df[\"correctness_score\"].apply(\n",
+ " lambda r: r[\"label\"]\n",
+ " )\n",
+ " rag_df[\"correctness_explanation\"] = correctness_eval_df[\n",
+ " \"correctness_score\"\n",
+ " ].apply(lambda r: r[\"explanation\"])\n",
" return rag_df"
]
},
@@ -967,28 +880,33 @@
"metadata": {},
"outputs": [],
"source": [
- "from arize.experimental.datasets import ArizeDatasetsClient\n",
- "from uuid import uuid1\n",
- "from arize.experimental.datasets.experiments.types import (\n",
- " ExperimentTaskResultColumnNames,\n",
- " EvaluationResultColumnNames,\n",
+ "from arize import ArizeClient\n",
+ "from arize.experiments import (\n",
+ " ExperimentTaskFieldNames,\n",
+ " EvaluationResultFieldNames,\n",
")\n",
- "from arize.experimental.datasets.utils.constants import GENERATIVE\n",
+ "from uuid import uuid1\n",
"import pandas as pd\n",
"\n",
"# Set up the arize client\n",
- "arize_client = ArizeDatasetsClient(api_key=API_KEY)\n",
- "dataset = None\n",
- "dataset_name = \"rag-experiments-\" + str(uuid1())[:3]\n",
- "\n",
- "dataset_id = arize_client.create_dataset(\n",
- " space_id=SPACE_ID,\n",
- " dataset_name=dataset_name,\n",
- " dataset_type=GENERATIVE,\n",
- " data=questions_df,\n",
+ "client = ArizeClient(api_key=API_KEY)\n",
+ "DATASET_NAME = \"rag-experiments-\" + str(uuid1())[:3]\n",
+ "\n",
+ "dataset = client.datasets.create(\n",
+ " name=DATASET_NAME,\n",
+ " space=SPACE_ID,\n",
+ " examples=questions_df,\n",
")\n",
- "dataset = arize_client.get_dataset(space_id=SPACE_ID, dataset_id=dataset_id)\n",
- "print(dataset)"
+ "print(dataset)\n",
+ "\n",
+ "# Fetch the dataset's server-assigned example IDs so we can attach them to the\n",
+ "# precomputed experiment runs (matched on the question text).\n",
+ "examples = client.datasets.list_examples(\n",
+ " dataset=DATASET_NAME, space=SPACE_ID, all=True\n",
+ ")\n",
+ "example_id_df = pd.DataFrame(\n",
+ " [{**ex.to_dict(), \"example_id\": ex.id} for ex in examples.examples]\n",
+ ")[[\"input\", \"example_id\"]]"
]
},
{
@@ -1007,32 +925,34 @@
"outputs": [],
"source": [
"# Define column mappings for task\n",
- "task_cols = ExperimentTaskResultColumnNames(\n",
- " example_id=\"example_id\", result=\"output\"\n",
+ "task_cols = ExperimentTaskFieldNames(\n",
+ " example_id=\"example_id\", output=\"output\"\n",
")\n",
"# Define column mappings for evaluator\n",
- "relevance_evaluator_cols = EvaluationResultColumnNames(\n",
+ "relevance_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"relevance\",\n",
" explanation=\"relevance_explanation\",\n",
")\n",
- "correctness_evaluator_cols = EvaluationResultColumnNames(\n",
+ "correctness_evaluator_cols = EvaluationResultFieldNames(\n",
" label=\"correctness\",\n",
" explanation=\"correctness_explanation\",\n",
")\n",
"\n",
"\n",
"def log_experiment_to_arize(experiment_df, experiment_name):\n",
- " experiment_df[\"example_id\"] = dataset[\"id\"]\n",
- " return arize_client.log_experiment(\n",
- " space_id=SPACE_ID,\n",
- " experiment_name=experiment_name + \"-\" + str(uuid1())[:2],\n",
- " experiment_df=experiment_df,\n",
- " task_columns=task_cols,\n",
+ " experiment_df = experiment_df.merge(\n",
+ " example_id_df, on=\"input\", how=\"left\"\n",
+ " )\n",
+ " return client.experiments.create(\n",
+ " space=SPACE_ID,\n",
+ " name=experiment_name + \"-\" + str(uuid1())[:2],\n",
+ " dataset=DATASET_NAME,\n",
+ " experiment_runs=experiment_df,\n",
+ " task_fields=task_cols,\n",
" evaluator_columns={\n",
" \"correctness\": correctness_evaluator_cols,\n",
" \"relevance\": relevance_evaluator_cols,\n",
" },\n",
- " dataset_name=dataset_name,\n",
" )"
]
},
diff --git a/python/llm/agents/openai-agents-cookbook.ipynb b/python/llm/agents/openai-agents-cookbook.ipynb
index 8566b07..8eb173d 100644
--- a/python/llm/agents/openai-agents-cookbook.ipynb
+++ b/python/llm/agents/openai-agents-cookbook.ipynb
@@ -56,9 +56,9 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install -q arize-otel openinference-instrumentation-openai-agents openinference-instrumentation-openai arize-phoenix-evals \"arize[Datasets]\"\n",
+ "!pip install -q arize-otel openinference-instrumentation-openai-agents nest_asyncio openinference-instrumentation-openai arize-phoenix-evals \"arize>=8.0.0\"\n",
"\n",
- "!pip install -q openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs nest_asyncio openai-agents"
+ "!pip install -q openai opentelemetry-sdk opentelemetry-exporter-otlp gcsfs openai-agents nest_asyncio"
]
},
{
@@ -81,11 +81,8 @@
"outputs": [],
"source": [
"import os\n",
- "import nest_asyncio\n",
"from getpass import getpass\n",
"\n",
- "nest_asyncio.apply()\n",
- "\n",
"SPACE_ID = globals().get(\"SPACE_ID\") or getpass(\n",
" \"🔑 Enter your Arize Space ID: \"\n",
")\n",
@@ -111,6 +108,9 @@
"metadata": {},
"outputs": [],
"source": [
+ "import nest_asyncio\n",
+ "nest_asyncio.apply()\n",
+ "\n",
"from arize.otel import register\n",
"from openinference.instrumentation.openai import OpenAIInstrumentor\n",
"from openinference.instrumentation.openai_agents import OpenAIAgentsInstrumentor\n",
@@ -269,51 +269,53 @@
"outputs": [],
"source": [
"import pandas as pd\n",
- "from phoenix.evals import OpenAIModel, llm_classify\n",
- "from arize.experimental.datasets.experiments.types import EvaluationResult\n",
- "\n",
- "\n",
- "def correctness_eval(dataset_row: dict, output: dict) -> EvaluationResult:\n",
- " # Create a dataframe with the question and answer\n",
- " df_in = pd.DataFrame(\n",
- " {\"question\": [dataset_row.get(\"question\")], \"response\": [output]}\n",
- " )\n",
+ "from phoenix.evals import LLM, create_classifier\n",
+ "from arize.experiments import EvaluationResult\n",
+ "\n",
+ "# Template for evaluating math problem solutions\n",
+ "MATH_EVAL_TEMPLATE = \"\"\"\n",
+ "You are evaluating whether a math problem was solved correctly.\n",
+ "\n",
+ "[BEGIN DATA]\n",
+ "************\n",
+ "[Question]: {question}\n",
+ "************\n",
+ "[Response]: {response}\n",
+ "[END DATA]\n",
+ "\n",
+ "Assess if the answer to the math problem is correct. First work out the correct answer yourself,\n",
+ "then compare with the provided response. Consider that there may be different ways to express the same answer \n",
+ "(e.g., \"43\" vs \"The answer is 43\" or \"5.0\" vs \"5\").\n",
+ "\n",
+ "Your answer must be a single word, either \"correct\" or \"incorrect\"\n",
+ "\"\"\"\n",
+ "\n",
+ "# create_classifier needs a tool-calling / structured-output model\n",
+ "judge = LLM(provider=\"openai\", model=\"gpt-4.1\")\n",
+ "\n",
+ "math_correctness = create_classifier(\n",
+ " name=\"correctness\",\n",
+ " prompt_template=MATH_EVAL_TEMPLATE,\n",
+ " llm=judge,\n",
+ " choices={\"correct\": 1.0, \"incorrect\": 0.0},\n",
+ " direction=\"maximize\",\n",
+ ")\n",
"\n",
- " # Template for evaluating math problem solutions\n",
- " MATH_EVAL_TEMPLATE = \"\"\"\n",
- " You are evaluating whether a math problem was solved correctly.\n",
- " \n",
- " [BEGIN DATA]\n",
- " ************\n",
- " [Question]: {question}\n",
- " ************\n",
- " [Response]: {response}\n",
- " [END DATA]\n",
- " \n",
- " Assess if the answer to the math problem is correct. First work out the correct answer yourself,\n",
- " then compare with the provided response. Consider that there may be different ways to express the same answer \n",
- " (e.g., \"43\" vs \"The answer is 43\" or \"5.0\" vs \"5\").\n",
- " \n",
- " Your answer must be a single word, either \"correct\" or \"incorrect\"\n",
- " \"\"\"\n",
"\n",
- " # Run the evaluation\n",
- " rails = [\"correct\", \"incorrect\"]\n",
- " eval_df = llm_classify(\n",
- " data=df_in,\n",
- " template=MATH_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=rails,\n",
- " provide_explanation=True,\n",
+ "# Experiment evaluators run inside asyncio, so we use the async classifier API.\n",
+ "async def correctness_eval(dataset_row: dict, output: dict) -> EvaluationResult:\n",
+ " scores = await math_correctness.async_evaluate(\n",
+ " {\n",
+ " \"question\": dataset_row.get(\"question\"),\n",
+ " \"response\": str(output),\n",
+ " }\n",
" )\n",
- "\n",
- " # Extract results\n",
- " label = eval_df[\"label\"][0]\n",
- " score = 1 if label == \"correct\" else 0\n",
- " explanation = eval_df[\"explanation\"][0]\n",
- "\n",
- " # Return the evaluation result\n",
- " return EvaluationResult(score=score, label=label, explanation=explanation)"
+ " score = scores[0]\n",
+ " return EvaluationResult(\n",
+ " score=score.score,\n",
+ " label=score.label,\n",
+ " explanation=score.explanation or \"no explanation\",\n",
+ " )"
]
},
{
@@ -359,16 +361,15 @@
"metadata": {},
"outputs": [],
"source": [
- "import nest_asyncio\n",
+ "from phoenix.evals import LLM\n",
"\n",
- "nest_asyncio.apply()\n",
"pd.set_option(\"display.max_colwidth\", 500)\n",
"\n",
"# Initialize the model\n",
- "model = OpenAIModel(model=\"gpt-4o\", max_tokens=1300)\n",
+ "model = LLM(provider=\"openai\", model=\"gpt-5.4-mini\")\n",
"\n",
"# Generate math problems\n",
- "resp = model(MATH_GEN_TEMPLATE)\n",
+ "resp = model.generate_text(MATH_GEN_TEMPLATE)\n",
"\n",
"# Create DataFrame\n",
"split_response = resp.strip().split(\"\\n\")\n",
@@ -411,22 +412,19 @@
"metadata": {},
"outputs": [],
"source": [
- "from arize.experimental.datasets import ArizeDatasetsClient\n",
+ "from arize import ArizeClient\n",
"from uuid import uuid1\n",
- "from arize.experimental.datasets.utils.constants import GENERATIVE\n",
"\n",
"# Set up the arize client\n",
- "arize_client = ArizeDatasetsClient(api_key=API_KEY)\n",
+ "client = ArizeClient(api_key=API_KEY)\n",
"\n",
- "dataset_name = \"math-questions-\" + str(uuid1())[:5]\n",
+ "DATASET_NAME = \"math-questions-\" + str(uuid1())[:5]\n",
"\n",
- "dataset_id = arize_client.create_dataset(\n",
- " space_id=SPACE_ID,\n",
- " dataset_name=dataset_name,\n",
- " dataset_type=GENERATIVE,\n",
- " data=math_problems_df,\n",
+ "dataset = client.datasets.create(\n",
+ " name=DATASET_NAME,\n",
+ " space=SPACE_ID,\n",
+ " examples=math_problems_df,\n",
")\n",
- "dataset = arize_client.get_dataset(space_id=SPACE_ID, dataset_id=dataset_id)\n",
"print(dataset)"
]
},
@@ -436,12 +434,13 @@
"metadata": {},
"outputs": [],
"source": [
- "experiment_id, experiment_dataframe = arize_client.run_experiment(\n",
- " space_id=SPACE_ID,\n",
- " dataset_id=dataset_id,\n",
+ "experiment, experiment_dataframe = client.experiments.run(\n",
+ " space=SPACE_ID,\n",
+ " name=f\"solve-math-questions-{str(uuid1())[:5]}\",\n",
+ " dataset=DATASET_NAME,\n",
" task=solve_math_problem,\n",
- " evaluators=[correctness_eval],\n",
- " experiment_name=f\"solve-math-questions-{str(uuid1())[:5]}\",\n",
+ " evaluators={\"correctness\": correctness_eval},\n",
+ " # dry_run runs task+evaluators locally without uploading; remove to persist in Arize AX\n",
" dry_run=True,\n",
")"
]
diff --git a/python/llm/experiments/text2sql-experiment.ipynb b/python/llm/experiments/text2sql-experiment.ipynb
index 72c65f6..3e8f236 100644
--- a/python/llm/experiments/text2sql-experiment.ipynb
+++ b/python/llm/experiments/text2sql-experiment.ipynb
@@ -35,7 +35,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install -q \"arize[Datasets]\" openai datasets pyarrow pydantic nest_asyncio arize-phoenix-evals"
+ "!pip install -q \"arize>=8.0.0\" openai datasets pyarrow pydantic nest_asyncio arize-phoenix-evals"
]
},
{
@@ -70,14 +70,11 @@
"source": [
"import os\n",
"\n",
- "from arize.experimental.datasets import ArizeDatasetsClient\n",
- "from arize.experimental.datasets.utils.constants import GENERATIVE\n",
+ "from arize import ArizeClient\n",
"\n",
"import pandas as pd\n",
"\n",
- "import asyncio\n",
- "\n",
- "import json"
+ "import asyncio"
]
},
{
@@ -283,37 +280,16 @@
"metadata": {},
"outputs": [],
"source": [
- "arize_client = ArizeDatasetsClient(\n",
- " api_key=os.environ.get(\"ARIZE_API_KEY\"),\n",
- ")\n",
- "# Create a dataset from a DataFrame add your own data here\n",
+ "arize_client = ArizeClient(api_key=os.environ.get(\"ARIZE_API_KEY\"))\n",
+ "\n",
+ "# Create a dataset from a DataFrame — add your own data here\n",
"test_df = pd.DataFrame([{\"question\": question} for question in questions])\n",
- "dataset_id = arize_client.create_dataset(\n",
- " space_id=space_id,\n",
- " dataset_name=dataset_name,\n",
- " dataset_type=GENERATIVE,\n",
- " data=test_df,\n",
+ "dataset = arize_client.datasets.create(\n",
+ " name=dataset_name,\n",
+ " space=space_id,\n",
+ " examples=test_df,\n",
")\n",
- "dataset_name"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "KrYDvmS0z439"
- },
- "source": [
- "Let's now pull down the dataset from Arize in this environment."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "dataset = arize_client.get_dataset(space_id=space_id, dataset_id=dataset_id)\n",
- "dataset.head()"
+ "print(dataset)"
]
},
{
@@ -340,12 +316,11 @@
" except duckdb.Error as e:\n",
" error = str(e)\n",
"\n",
- " r = {\n",
+ " return {\n",
" \"query\": query,\n",
" \"results\": results,\n",
" \"error\": error,\n",
- " }\n",
- " return json.dumps(r)"
+ " }"
]
},
{
@@ -363,18 +338,28 @@
"metadata": {},
"outputs": [],
"source": [
+ "from arize.experiments import EvaluationResult\n",
+ "\n",
+ "\n",
"# Test if there are no sql execution errors\n",
- "def no_error(output):\n",
- " output = json.loads(output)\n",
- " return 1.0 if output.get(\"error\") is None else 0.0\n",
+ "def no_error(output) -> EvaluationResult:\n",
+ " error = output.get(\"error\")\n",
+ " return EvaluationResult(\n",
+ " score=1.0 if error is None else 0.0,\n",
+ " label=\"no_error\" if error is None else \"error\",\n",
+ " explanation=error or \"query executed without errors\",\n",
+ " )\n",
"\n",
"\n",
"# Test if the query has results\n",
- "def has_results(output):\n",
- " output = json.loads(output)\n",
+ "def has_results(output) -> EvaluationResult:\n",
" results = output.get(\"results\")\n",
- " has_results = results is not None and len(results) > 0\n",
- " return 1.0 if has_results else 0.0"
+ " ok = results is not None and len(results) > 0\n",
+ " return EvaluationResult(\n",
+ " score=1.0 if ok else 0.0,\n",
+ " label=\"has_results\" if ok else \"no_results\",\n",
+ " explanation=f\"returned {len(results) if results else 0} rows\",\n",
+ " )"
]
},
{
@@ -408,12 +393,12 @@
" return asyncio.run(text2sql(input[\"question\"]))\n",
"\n",
"\n",
- "experiment = arize_client.run_experiment(\n",
- " space_id=space_id,\n",
- " dataset_id=dataset_id,\n",
+ "experiment, experiment_df = arize_client.experiments.run(\n",
+ " space=space_id,\n",
+ " name=\"text2sql_first_test\",\n",
+ " dataset=dataset_name,\n",
" task=task,\n",
- " evaluators=[no_error, has_results],\n",
- " experiment_name=\"text2sql_test-2\",\n",
+ " evaluators={\"no_error\": no_error, \"has_results\": has_results},\n",
")"
]
},
@@ -510,44 +495,51 @@
"metadata": {},
"outputs": [],
"source": [
- "from phoenix.evals.models import OpenAIModel\n",
- "from phoenix.evals.classify import llm_classify\n",
- "from arize.experimental.datasets.experiments.types import EvaluationResult\n",
+ "from phoenix.evals import LLM, create_classifier\n",
+ "from arize.experiments.evaluators.types import EvaluationResult\n",
"\n",
"\n",
- "IS_SQL_EVAL_TEMPLATE = \"\"\"You are a SQL expert, is the following a valid SQL query that executes without errors? Return the single workd \"valid\" if is valid, and \"invalid\" if it is not.\n",
+ "IS_SQL_EVAL_TEMPLATE = \"\"\"You are a SQL expert, is the following a valid SQL query that executes without errors? Return the single word \"valid\" if it is valid, and \"invalid\" if it is not.\n",
"\n",
"[BEGIN SQL QUERY]\n",
"{query}\n",
"[END SQL QUERY]\n",
"\"\"\"\n",
"\n",
+ "# create_classifier needs a tool-calling / structured-output model\n",
+ "judge = LLM(provider=\"openai\", model=\"gpt-4.1\")\n",
+ "\n",
+ "is_sql_classifier = create_classifier(\n",
+ " name=\"is_sql\",\n",
+ " prompt_template=IS_SQL_EVAL_TEMPLATE,\n",
+ " llm=judge,\n",
+ " choices={\"valid\": 1.0, \"invalid\": 0.0},\n",
+ " direction=\"maximize\",\n",
+ ")\n",
+ "\n",
"\n",
- "def check_is_sql(output):\n",
- " output = json.loads(output)\n",
+ "# Experiment evaluators run inside asyncio, so use the async classifier API\n",
+ "async def check_is_sql(output) -> EvaluationResult:\n",
" query = output.get(\"query\")\n",
- " df_in = pd.DataFrame({\"query\": query}, index=[0]) if query else None\n",
- " eval_df = llm_classify(\n",
- " dataframe=df_in,\n",
- " template=IS_SQL_EVAL_TEMPLATE,\n",
- " model=OpenAIModel(model=\"gpt-4o\"),\n",
- " rails=[\"valid\", \"invalid\"],\n",
- " provide_explanation=True,\n",
- " )\n",
- " # return score, label, explanation\n",
+ " scores = await is_sql_classifier.async_evaluate({\"query\": query})\n",
+ " score = scores[0]\n",
" return EvaluationResult(\n",
- " score=1,\n",
- " label=eval_df[\"label\"][0],\n",
- " explanation=eval_df[\"explanation\"][0],\n",
+ " score=score.score,\n",
+ " label=score.label,\n",
+ " explanation=score.explanation or \"no explanation\",\n",
" )\n",
"\n",
"\n",
- "experiment = arize_client.run_experiment(\n",
- " space_id=space_id,\n",
- " dataset_id=dataset_id,\n",
+ "experiment, experiment_df = arize_client.experiments.run(\n",
+ " space=space_id,\n",
+ " name=\"text2sql_test_new_prompt_and_eval\",\n",
+ " dataset=dataset_name,\n",
" task=task,\n",
- " evaluators=[no_error, has_results, check_is_sql],\n",
- " experiment_name=\"text2sql_test_new_prompt_and_eval-6\",\n",
+ " evaluators={\n",
+ " \"no_error\": no_error,\n",
+ " \"has_results\": has_results,\n",
+ " \"check_is_sql\": check_is_sql,\n",
+ " },\n",
")"
]
},