diff --git a/packages/opentelemetry-instrumentation-crewai/opentelemetry/instrumentation/crewai/instrumentation.py b/packages/opentelemetry-instrumentation-crewai/opentelemetry/instrumentation/crewai/instrumentation.py index 6d57888fd4..c360afea4e 100644 --- a/packages/opentelemetry-instrumentation-crewai/opentelemetry/instrumentation/crewai/instrumentation.py +++ b/packages/opentelemetry-instrumentation-crewai/opentelemetry/instrumentation/crewai/instrumentation.py @@ -45,6 +45,23 @@ ("command", GenAISystem.COHERE.value), ] +# CrewAI >= 1.15 routes LLM calls to native provider SDKs: `LLM.__new__` is a +# factory that returns instances of these provider classes (e.g. OpenAICompletion) +# rather than of `LLM`, so wrapping `LLM.call` alone never fires on the native +# path. Wrap each provider class's own `call` as well. `OpenAICompatibleCompletion` +# and `SnowflakeCompletion` subclass `OpenAICompletion` without overriding `call`, +# so they inherit the wrap and must not be listed here (double-wrapping them would +# emit duplicate spans). Entries whose module is not importable (crewai versions +# before the native providers existed, or a provider SDK that isn't installed) +# are skipped at instrument time. +CREWAI_NATIVE_LLM_PROVIDERS = [ + ("crewai.llms.providers.anthropic.completion", "AnthropicCompletion"), + ("crewai.llms.providers.azure.completion", "AzureCompletion"), + ("crewai.llms.providers.bedrock.completion", "BedrockCompletion"), + ("crewai.llms.providers.gemini.completion", "GeminiCompletion"), + ("crewai.llms.providers.openai.completion", "OpenAICompletion"), +] + def _infer_llm_provider_from_model(model: object | None) -> str | None: """Resolve gen_ai.provider.name for the underlying LLM on a chat span. @@ -92,12 +109,25 @@ def _instrument(self, **kwargs): wrap_task_execute(tracer, duration_histogram, token_histogram)) wrap_function_wrapper("crewai.llm", "LLM.call", wrap_llm_call(tracer, duration_histogram, token_histogram)) + for module, class_name in CREWAI_NATIVE_LLM_PROVIDERS: + try: + wrap_function_wrapper(module, f"{class_name}.call", + wrap_llm_call(tracer, duration_histogram, token_histogram)) + except (ImportError, AttributeError): + # crewai < native-provider release, or the provider SDK is not installed. + pass def _uninstrument(self, **kwargs): unwrap("crewai.crew.Crew", "kickoff") unwrap("crewai.agent.Agent", "execute_task") unwrap("crewai.task.Task", "execute_sync") unwrap("crewai.llm.LLM", "call") + for module, class_name in CREWAI_NATIVE_LLM_PROVIDERS: + try: + unwrap(f"{module}.{class_name}", "call") + except (ImportError, AttributeError): + # Mirrors _instrument: skip entries that were never wrapped. + pass def with_tracer_wrapper(func): diff --git a/packages/opentelemetry-instrumentation-crewai/tests/test_crewai_instrumentation.py b/packages/opentelemetry-instrumentation-crewai/tests/test_crewai_instrumentation.py index dccae7a27a..92f1c46e05 100644 --- a/packages/opentelemetry-instrumentation-crewai/tests/test_crewai_instrumentation.py +++ b/packages/opentelemetry-instrumentation-crewai/tests/test_crewai_instrumentation.py @@ -1,12 +1,16 @@ import pytest from unittest.mock import MagicMock -from pydantic import BaseModel +import crewai.llms.base_llm as crewai_llms_base_llm from crewai import Agent, Crew, Task from crewai.llms.base_llm import BaseLLM - from opentelemetry.instrumentation.crewai import CrewAIInstrumentor +from opentelemetry.instrumentation.crewai import instrumentation as crewai_instrumentation +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace.status import StatusCode +from pydantic import BaseModel class StubLLM(BaseLLM): @@ -27,10 +31,15 @@ def call( @pytest.fixture def mock_instrumentor(): + # BaseInstrumentor is a singleton: the instance-level MagicMock below would + # leak into later tests that call the real instrument(), so delete the + # instance attributes on teardown to restore the class methods. instrumentor = CrewAIInstrumentor() instrumentor.instrument = MagicMock() instrumentor.uninstrument = MagicMock() - return instrumentor + yield instrumentor + del instrumentor.instrument + del instrumentor.uninstrument @pytest.fixture @@ -89,3 +98,102 @@ def test_trace_status(mock_crew, mock_instrumentor): mock_instrumentor.uninstrument() mock_instrumentor.uninstrument.assert_called_once() + + +# --- Real instrumentation tests for the native-provider LLM call path (#4453) --- +# +# On crewai >= 1.15, `crewai.llm.LLM.__new__` is a factory that returns native +# provider instances (e.g. OpenAICompletion) which are NOT subclasses of `LLM`, +# so the historical `LLM.call` wrap never fires on them. `_instrument` therefore +# also wraps each native provider class's `call`. These tests drive that wrap +# path with a BaseLLM subclass standing in for a provider class, since provider +# SDKs (openai/anthropic/...) are not test dependencies. + + +class NativeStyleStubLLM(BaseLLM): + """BaseLLM subclass shaped like a crewai native provider (not an `LLM`).""" + + def call( + self, + messages, + tools=None, + callbacks=None, + available_functions=None, + from_task=None, + from_agent=None, + response_model=None, + ): + return "Mocked native response" + + +@pytest.fixture +def span_env(): + # crewai registers its own global TracerProvider on import, so never call + # trace.set_tracer_provider here — pass the provider into instrument(). + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +@pytest.fixture +def native_style_wrap(monkeypatch): + """Route the native-provider wrap list at a BaseLLM subclass in this module.""" + monkeypatch.setattr(crewai_llms_base_llm, "NativeStyleStubLLM", NativeStyleStubLLM, raising=False) + monkeypatch.setattr( + crewai_instrumentation, + "CREWAI_NATIVE_LLM_PROVIDERS", + [("crewai.llms.base_llm", "NativeStyleStubLLM")], + ) + + +def test_native_style_llm_call_emits_span(native_style_wrap, span_env): + provider, exporter = span_env + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + llm = NativeStyleStubLLM(model="gpt-4o-mini") + result = llm.call([{"role": "user", "content": "hello"}]) + assert result == "Mocked native response" + + llm_spans = [s for s in exporter.get_finished_spans() if s.name == "gpt-4o-mini.llm"] + assert len(llm_spans) == 1, "native-provider-style LLM.call must produce a {model}.llm span" + attrs = llm_spans[0].attributes + assert attrs["gen_ai.request.model"] == "gpt-4o-mini" + assert attrs["gen_ai.provider.name"] == "openai" + assert llm_spans[0].status.status_code == StatusCode.OK + finally: + # BaseInstrumentor is a singleton — a failed assertion must not leave + # the wraps in place for later tests. + instrumentor.uninstrument() + + +def test_uninstrument_restores_native_style_call(native_style_wrap, span_env): + provider, _ = span_env + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + assert hasattr(NativeStyleStubLLM.call, "__wrapped__") + finally: + instrumentor.uninstrument() + assert not hasattr(NativeStyleStubLLM.call, "__wrapped__") + + from crewai.llm import LLM + + assert not hasattr(LLM.call, "__wrapped__") + + +def test_real_native_providers_are_wrapped(span_env): + try: + from crewai.llms.providers.openai.completion import OpenAICompletion + except ImportError: + pytest.skip("crewai without native provider modules (< 1.15)") + + provider, _ = span_env + instrumentor = CrewAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + assert hasattr(OpenAICompletion.call, "__wrapped__") + finally: + instrumentor.uninstrument() + assert not hasattr(OpenAICompletion.call, "__wrapped__")