diff --git a/AGENTS.md b/AGENTS.md index 8672269b6..4a1aa67c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,7 @@ An adapter must **never** call `self.llm_judge.judge()` or parse a judge reply i - `repeats`: duplicates items for k-metrics. `generation_config.n` is deprecated and mapped. - Use `generation_config` for runtime params. `TaskConfig.timeout` / `stream` are deprecated — forwarded with a warning. - `dataset_args` merges into `BenchmarkMeta._update()` (supports `local_path`, `filters` OrderedDict prepended). -- Models are memoized by `(name, config, base_url, api_key, args)`. +- Models are memoized by `(name, eval_type, config, base_url, api_key, args)`. - Use `@thread_safe` for model creation, `run_in_threads_with_progress` for concurrent eval. - Outputs land in `outputs//{logs,predictions,reviews,reports,configs}/` (see `OutputsStructure`). `use_cache` resumes runs; `rerun_review` recomputes scores only. - `evalscope app` CLI command is **deprecated** (see `evalscope/cli/start_app.py`) — use `evalscope service` for the Web dashboard. diff --git a/evalscope/api/model/model.py b/evalscope/api/model/model.py index 970ab3d3d..aed6fdf0f 100644 --- a/evalscope/api/model/model.py +++ b/evalscope/api/model/model.py @@ -368,6 +368,7 @@ def get_model( if memoize: model_cache_key = ( model + + str(eval_type) + str(role) + config.model_dump_json(exclude_none=True) + str(base_url) diff --git a/evalscope/models/image_edit_model.py b/evalscope/models/image_edit_model.py index c1759da87..a7e6b73f6 100644 --- a/evalscope/models/image_edit_model.py +++ b/evalscope/models/image_edit_model.py @@ -98,6 +98,8 @@ def generate( config: GenerateConfig, ) -> ModelOutput: + start_time = time.monotonic() + # prepare generator kwargs: Dict[str, Any] = {} if config.num_inference_steps is not None: @@ -117,10 +119,12 @@ def generate( output = self.model(image=input_image, prompt=prompt, **kwargs) image = output.images[0] - image_base64 = PIL_to_base64(image) + # Emit a data URI so downstream consumers (e.g. an LLM-judge request) + # treat the image as inline base64 instead of a local file path. + image_base64 = PIL_to_base64(image, add_header=True) return ModelOutput( model=self.model_name, choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])], - time=time.time(), + time=time.monotonic() - start_time, ) diff --git a/evalscope/models/litellm_compatible.py b/evalscope/models/litellm_compatible.py index 784458172..53efade3c 100644 --- a/evalscope/models/litellm_compatible.py +++ b/evalscope/models/litellm_compatible.py @@ -67,7 +67,9 @@ def generate( request: Dict[str, Any] = {} - messages = openai_chat_messages(input) + messages = openai_chat_messages( + input, reasoning_format=(config.reasoning_history or 'reasoning_field'), base_url=self.base_url + ) completion_params = openai_completion_params( model=self.model_name, config=config, @@ -139,7 +141,9 @@ async def generate_async( """ import litellm - messages = openai_chat_messages(input) + messages = openai_chat_messages( + input, reasoning_format=(config.reasoning_history or 'reasoning_field'), base_url=self.base_url + ) completion_params = openai_completion_params( model=self.model_name, config=config, diff --git a/evalscope/models/modelscope.py b/evalscope/models/modelscope.py index 5fc301669..56b7b31ff 100644 --- a/evalscope/models/modelscope.py +++ b/evalscope/models/modelscope.py @@ -223,24 +223,30 @@ def generate( ) choices.append(choice) + # Aggregate usage over all returned choices: with n > 1 the loop above + # leaves `response` bound to the last choice only, undercounting tokens. + input_tokens = sum(r.input_tokens for r in responses) + output_tokens = sum(r.output_tokens for r in responses) + total_time = max(r.time for r in responses) + # return output output = ModelOutput( model=self.model_name, choices=choices, usage=ModelUsage( - input_tokens=response.input_tokens, - output_tokens=response.output_tokens, - total_tokens=response.total_tokens, + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, ), - time=response.time, + time=total_time, ) # Populate PerformanceMetrics from the already-available fields. # Local models do not produce TTFT (no streaming chunks), so ttft stays None. output.message.perf_metrics = PerformanceMetrics( - latency=response.time, + latency=total_time, ttft=None, - input_tokens=response.input_tokens, - output_tokens=response.output_tokens, + input_tokens=input_tokens, + output_tokens=output_tokens, ) return output diff --git a/evalscope/models/text2image_model.py b/evalscope/models/text2image_model.py index c151fa5f1..d5c49613c 100644 --- a/evalscope/models/text2image_model.py +++ b/evalscope/models/text2image_model.py @@ -97,6 +97,8 @@ def generate( config: GenerateConfig, ) -> ModelOutput: + start_time = time.monotonic() + # prepare generator kwargs: Dict[str, Any] = {} if config.height is not None: @@ -115,10 +117,12 @@ def generate( # get the first image as output image = self.model(prompt=prompt, **kwargs).images[0] - image_base64 = PIL_to_base64(image) + # Emit a data URI so downstream consumers (e.g. an LLM-judge request) + # treat the image as inline base64 instead of a local file path. + image_base64 = PIL_to_base64(image, add_header=True) return ModelOutput( model=self.model_name, choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])], - time=time.time(), + time=time.monotonic() - start_time, ) diff --git a/evalscope/models/utils/openai.py b/evalscope/models/utils/openai.py index 14a2486b4..53cee56da 100644 --- a/evalscope/models/utils/openai.py +++ b/evalscope/models/utils/openai.py @@ -904,8 +904,10 @@ def collect_stream_response( # use the finish_reason from the last chunk that generated this choice finish_reason = None for chunk in reversed(collected_chunks): - if chunk.choices and chunk.choices[0].index == index: - finish_reason = chunk.choices[0].finish_reason + # a single chunk may pack several choices (n > 1); match by index + matched = next((c for c in chunk.choices if c.index == index), None) + if matched is not None: + finish_reason = matched.finish_reason break message_kwargs = {'role': 'assistant', 'content': full_reply_content} @@ -1035,8 +1037,10 @@ async def async_collect_stream_response( # use the finish_reason from the last chunk that generated this choice finish_reason = None for chunk in reversed(collected_chunks): - if chunk.choices and chunk.choices[0].index == index: - finish_reason = chunk.choices[0].finish_reason + # a single chunk may pack several choices (n > 1); match by index + matched = next((c for c in chunk.choices if c.index == index), None) + if matched is not None: + finish_reason = matched.finish_reason break message_kwargs = {'role': 'assistant', 'content': full_reply_content} diff --git a/tests/models/test_image_model_output.py b/tests/models/test_image_model_output.py new file mode 100644 index 000000000..fcc381d2b --- /dev/null +++ b/tests/models/test_image_model_output.py @@ -0,0 +1,138 @@ +from typing import Any, List + +import pytest +from PIL import Image + +from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText +from evalscope.api.model import GenerateConfig +from evalscope.models.image_edit_model import ImageEditAPI +from evalscope.models.text2image_model import Text2ImageAPI +from evalscope.models.utils.openai import openai_chat_completion_part +from evalscope.utils.io_utils import PIL_to_base64 + + +class _FakePipelineResult: + + def __init__(self, image: Image.Image) -> None: + self.images = [image] + + +def _install_fake_pipeline( + monkeypatch: Any, + attr_name: str, + image: Image.Image, + on_call: Any = None, +) -> None: + class FakePipeline: + + @classmethod + def from_pretrained(cls, *args: Any, **kwargs: Any) -> 'FakePipeline': + return cls() + + def to(self, device: Any) -> 'FakePipeline': + return self + + def __call__(self, *args: Any, **kwargs: Any) -> _FakePipelineResult: + if on_call is not None: + on_call() + return _FakePipelineResult(image) + + import modelscope + + monkeypatch.setattr(modelscope, attr_name, FakePipeline, raising=False) + + +def _generated_image_content(output: Any) -> ContentImage: + content = output.choices[0].message.content + assert isinstance(content, list) and isinstance(content[0], ContentImage) + return content[0] + + +def test_text2image_output_is_usable_as_openai_chat_input(monkeypatch: Any) -> None: + image = Image.new('RGB', (8, 8), color='red') + _install_fake_pipeline(monkeypatch, 'DiffusionPipeline', image) + + api = Text2ImageAPI(model_name='test-diffusion') + output = api.generate( + input=[ChatMessageUser(content='a red square')], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + part = openai_chat_completion_part(_generated_image_content(output)) + url = part['image_url']['url'] + assert url.startswith('data:image/') + + +def test_image_edit_output_is_usable_as_openai_chat_input(monkeypatch: Any) -> None: + generated = Image.new('RGB', (8, 8), color='blue') + _install_fake_pipeline(monkeypatch, 'QwenImageEditPipeline', generated) + + source_image = PIL_to_base64(Image.new('RGB', (8, 8), color='green'), format='PNG', add_header=True) + api = ImageEditAPI(model_name='Qwen-Image-Edit-test') + output = api.generate( + input=[ + ChatMessageUser(content=[ + ContentText(text='make it blue'), + ContentImage(image=source_image), + ]) + ], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + part = openai_chat_completion_part(_generated_image_content(output)) + url = part['image_url']['url'] + assert url.startswith('data:image/') + + +def test_text2image_time_is_elapsed_seconds(monkeypatch: Any) -> None: + clock = [100.0] + monkeypatch.setattr('evalscope.models.text2image_model.time.monotonic', lambda: clock[0]) + image = Image.new('RGB', (8, 8), color='red') + + def _advance_clock() -> None: + clock[0] = 100.25 + + _install_fake_pipeline(monkeypatch, 'DiffusionPipeline', image, on_call=_advance_clock) + + api = Text2ImageAPI(model_name='test-diffusion') + output = api.generate( + input=[ChatMessageUser(content='a red square')], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + # elapsed seconds since generate() started, not a wall-clock epoch timestamp + assert output.time == pytest.approx(0.25) + + +def test_image_edit_time_is_elapsed_seconds(monkeypatch: Any) -> None: + clock = [200.0] + monkeypatch.setattr('evalscope.models.image_edit_model.time.monotonic', lambda: clock[0]) + generated = Image.new('RGB', (8, 8), color='blue') + + def _advance_clock() -> None: + clock[0] = 200.5 + + _install_fake_pipeline(monkeypatch, 'QwenImageEditPipeline', generated, on_call=_advance_clock) + + source_image = PIL_to_base64(Image.new('RGB', (8, 8), color='green'), format='PNG', add_header=True) + api = ImageEditAPI(model_name='Qwen-Image-Edit-test') + output = api.generate( + input=[ + ChatMessageUser(content=[ + ContentText(text='make it blue'), + ContentImage(image=source_image), + ]) + ], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + # elapsed seconds since generate() started, not a wall-clock epoch timestamp + assert output.time == pytest.approx(0.5) diff --git a/tests/models/test_litellm_reasoning_history.py b/tests/models/test_litellm_reasoning_history.py new file mode 100644 index 000000000..a4ff9ed7a --- /dev/null +++ b/tests/models/test_litellm_reasoning_history.py @@ -0,0 +1,102 @@ +from typing import Any, Dict, List + +import litellm +from openai.types.chat import ChatCompletion + +from evalscope.api.messages import ( + ChatMessage, + ChatMessageAssistant, + ChatMessageUser, + ContentAudio, + ContentReasoning, + ContentText, +) +from evalscope.api.model import GenerateConfig +from evalscope.api.tool import ToolChoice, ToolInfo +from evalscope.models.litellm_compatible import LiteLLMAPI + + +def _completion_response() -> ChatCompletion: + return ChatCompletion.model_validate({ + 'id': 'completion-id', + 'created': 1, + 'model': 'test-model', + 'object': 'chat.completion', + 'choices': [{ + 'index': 0, + 'finish_reason': 'stop', + 'message': {'role': 'assistant', 'content': 'answer'}, + }], + 'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2}, + }) + + +def _conversation() -> List[ChatMessage]: + return [ + ChatMessageUser(content='question'), + ChatMessageAssistant(content=[ContentReasoning(reasoning='prior thoughts'), ContentText(text='prior answer')]), + ] + + +def _capturing_completion(calls: List[Dict[str, Any]]) -> Any: + + def _completion(**request: Any) -> ChatCompletion: + calls.append(request) + return _completion_response() + + return _completion + + +def test_reasoning_history_defaults_to_reasoning_field(monkeypatch: Any) -> None: + calls: List[Dict[str, Any]] = [] + monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls)) + + api = LiteLLMAPI(model_name='openai/test-model') + api.generate(input=_conversation(), tools=[], tool_choice='none', config=GenerateConfig()) + + assistant_payload = calls[0]['messages'][1] + # parity with OpenAICompatibleAPI: reasoning lives in the top-level field, + # not smuggled into the content as a tag + assert assistant_payload['reasoning_content'] == 'prior thoughts' + assert '' not in assistant_payload['content'] + assert assistant_payload['content'].strip() == 'prior answer' + + +def test_reasoning_history_none_is_honored(monkeypatch: Any) -> None: + calls: List[Dict[str, Any]] = [] + monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls)) + + api = LiteLLMAPI(model_name='openai/test-model') + api.generate( + input=_conversation(), + tools=[], + tool_choice='none', + config=GenerateConfig(reasoning_history='none'), + ) + + assistant_payload = calls[0]['messages'][1] + assert 'reasoning_content' not in assistant_payload + assert '' not in assistant_payload['content'] + assert assistant_payload['content'].strip() == 'prior answer' + + +def test_base_url_forwarded_for_dashscope_audio_encoding(monkeypatch: Any) -> None: + calls: List[Dict[str, Any]] = [] + monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls)) + + api = LiteLLMAPI(model_name='openai/test-model', base_url='https://dashscope.aliyuncs.com/compatible-mode/v1') + api.generate( + input=[ + ChatMessageUser(content=[ + ContentAudio(audio='data:audio/wav;base64,YXVkaW8=', format='wav'), + ContentText(text='transcribe'), + ]) + ], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + audio_part = calls[0]['messages'][0]['content'][0] + # DashScope endpoints require the data-URI prefix on input audio + assert audio_part['input_audio']['data'] == 'data:audio/wav;base64,YXVkaW8=' diff --git a/tests/models/test_model_cache.py b/tests/models/test_model_cache.py new file mode 100644 index 000000000..c145c4c75 --- /dev/null +++ b/tests/models/test_model_cache.py @@ -0,0 +1,60 @@ +from typing import List + +import pytest + +from evalscope.api.messages import ChatMessage +from evalscope.api.model import GenerateConfig, ModelAPI, ModelOutput, get_model +from evalscope.api.model.model import ModelCache +from evalscope.api.registry import MODEL_APIS +from evalscope.api.tool import ToolChoice, ToolInfo + + +class FakeOpenAIBackend(ModelAPI): + + def generate( + self, + input: List[ChatMessage], + tools: List[ToolInfo], + tool_choice: ToolChoice, + config: GenerateConfig, + ) -> ModelOutput: + return ModelOutput(model=self.model_name, choices=[]) + + +class FakeLiteLLMBackend(ModelAPI): + + def generate( + self, + input: List[ChatMessage], + tools: List[ToolInfo], + tool_choice: ToolChoice, + config: GenerateConfig, + ) -> ModelOutput: + return ModelOutput(model=self.model_name, choices=[]) + + +@pytest.fixture +def fake_backends(): + MODEL_APIS['fake_openai_backend'] = FakeOpenAIBackend + MODEL_APIS['fake_litellm_backend'] = FakeLiteLLMBackend + ModelCache._models.clear() + yield + MODEL_APIS.pop('fake_openai_backend', None) + MODEL_APIS.pop('fake_litellm_backend', None) + ModelCache._models.clear() + + +def test_model_cache_key_includes_eval_type(fake_backends) -> None: + first = get_model(model='shared-model', eval_type='fake_openai_backend', api_key='key') + second = get_model(model='shared-model', eval_type='fake_litellm_backend', api_key='key') + + assert first is not second + assert isinstance(first.api, FakeOpenAIBackend) + assert isinstance(second.api, FakeLiteLLMBackend) + + +def test_same_eval_type_returns_memoized_model(fake_backends) -> None: + first = get_model(model='shared-model', eval_type='fake_openai_backend', api_key='key') + again = get_model(model='shared-model', eval_type='fake_openai_backend', api_key='key') + + assert again is first diff --git a/tests/models/test_modelscope_usage.py b/tests/models/test_modelscope_usage.py new file mode 100644 index 000000000..f7c114b41 --- /dev/null +++ b/tests/models/test_modelscope_usage.py @@ -0,0 +1,79 @@ +from typing import Any, List + +import pytest + +from evalscope.api.messages import ChatMessageUser +from evalscope.api.model import GenerateConfig +from evalscope.models import modelscope as modelscope_module +from evalscope.models.modelscope import GenerateOutput, ModelScopeAPI + + +class _FakeTokenizer: + + chat_template = None + + def __call__(self, *args: Any, **kwargs: Any) -> dict: + raise AssertionError('tokenizer must not be called when batched_generate is mocked') + + def batch_decode(self, *args: Any, **kwargs: Any) -> list: + raise AssertionError('batch_decode must not be called when batched_generate is mocked') + + +class _FakeModel: + + device = 'cpu' + + def generate(self, *args: Any, **kwargs: Any) -> Any: + raise AssertionError('model.generate must not be called when batched_generate is mocked') + + +def _make_api() -> ModelScopeAPI: + """Build a ModelScopeAPI without loading any weights.""" + api = object.__new__(ModelScopeAPI) + api.model_name = 'test-model' + api.tokenizer = _FakeTokenizer() + api.model = _FakeModel() + api.chat_template = None + api.tokenizer_call_args = {} + api.enable_thinking = None + return api + + +def test_generate_aggregates_usage_across_choices(monkeypatch: Any) -> None: + responses: List[GenerateOutput] = [ + GenerateOutput( + output='first', + input_tokens=10, + output_tokens=4, + total_tokens=14, + logprobs=None, + time=1.2, + stop_reason='stop', + ), + GenerateOutput( + output='second', + input_tokens=10, + output_tokens=6, + total_tokens=16, + logprobs=None, + time=1.5, + stop_reason='max_tokens', + ), + ] + monkeypatch.setattr(modelscope_module, 'batched_generate', lambda _input: responses) + + output = _make_api().generate( + input=[ChatMessageUser(content='hi')], + tools=[], + tool_choice='none', + config=GenerateConfig(), + ) + + assert [choice.message.text for choice in output.choices] == ['first', 'second'] + # usage must aggregate over all returned choices, not just the last one + assert output.usage.input_tokens == 20 + assert output.usage.output_tokens == 10 + assert output.usage.total_tokens == 30 + assert output.time == pytest.approx(1.5) + assert output.message.perf_metrics.input_tokens == 20 + assert output.message.perf_metrics.output_tokens == 10 diff --git a/tests/models/test_openai_stream_finish_reason.py b/tests/models/test_openai_stream_finish_reason.py new file mode 100644 index 000000000..4dba02fb0 --- /dev/null +++ b/tests/models/test_openai_stream_finish_reason.py @@ -0,0 +1,62 @@ +import asyncio +from typing import List, Optional, Tuple + +from openai.types.chat import ChatCompletionChunk + +from evalscope.models.utils.openai import async_collect_stream_response, collect_stream_response + + +def _packed_chunk(choices: List[Tuple[int, Optional[str], Optional[str]]]) -> ChatCompletionChunk: + """Build a chunk packing several choices (as servers do for n > 1). + + ``choices`` is a list of ``(index, content, finish_reason)`` tuples. + """ + return ChatCompletionChunk.model_validate({ + 'id': 'completion-id', + 'created': 1, + 'model': 'test-model', + 'object': 'chat.completion.chunk', + 'choices': [ + { + 'index': index, + 'finish_reason': finish_reason, + 'delta': {'content': content}, + } for index, content, finish_reason in choices + ], + }) + + +def test_sync_finish_reason_restored_for_every_packed_choice() -> None: + stream = [ + _packed_chunk([(0, 'first', None), (1, 'second', None)]), + _packed_chunk([(0, None, 'stop'), (1, None, 'length')]), + ] + + completion, _ttft = collect_stream_response(stream) + + finish_reasons = {choice.index: choice.finish_reason for choice in completion.choices} + assert finish_reasons == {0: 'stop', 1: 'length'} + + +def test_async_finish_reason_restored_for_every_packed_choice() -> None: + async def stream() -> None: + yield _packed_chunk([(0, 'first', None), (1, 'second', None)]) + yield _packed_chunk([(0, None, 'tool_calls'), (1, None, 'content_filter')]) + + completion, _ttft = asyncio.run(async_collect_stream_response(stream())) + + finish_reasons = {choice.index: choice.finish_reason for choice in completion.choices} + assert finish_reasons == {0: 'tool_calls', 1: 'content_filter'} + + +def test_sync_finish_reason_matches_choice_beyond_first_position() -> None: + # the matching choice is not the first element of chunk.choices + stream = [ + _packed_chunk([(1, 'second', None), (0, 'first', None)]), + _packed_chunk([(1, None, 'length'), (0, None, 'stop')]), + ] + + completion, _ttft = collect_stream_response(stream) + + finish_reasons = {choice.index: choice.finish_reason for choice in completion.choices} + assert finish_reasons == {0: 'stop', 1: 'length'}