fix(models): correct cache, usage, streaming, and image outputs

AGENTS.md

@@ -156,7 +156,7 @@ An adapter must **never** call `self.llm_judge.judge()` or parse a judge reply i
 - `repeats`: duplicates items for k-metrics. `generation_config.n` is deprecated and mapped.
 - Use `generation_config` for runtime params. `TaskConfig.timeout` / `stream` are deprecated — forwarded with a warning.
 - `dataset_args` merges into `BenchmarkMeta._update()` (supports `local_path`, `filters` OrderedDict prepended).
-- Models are memoized by `(name, config, base_url, api_key, args)`.
+- Models are memoized by `(name, eval_type, config, base_url, api_key, args)`.
 - Use `@thread_safe` for model creation, `run_in_threads_with_progress` for concurrent eval.
 - Outputs land in `outputs/<timestamp>/{logs,predictions,reviews,reports,configs}/` (see `OutputsStructure`). `use_cache` resumes runs; `rerun_review` recomputes scores only.
 - `evalscope app` CLI command is **deprecated** (see `evalscope/cli/start_app.py`) — use `evalscope service` for the Web dashboard.

evalscope/api/model/model.py

@@ -368,6 +368,7 @@ def get_model(
     if memoize:
         model_cache_key = (
             model
+            + str(eval_type)
             + str(role)
             + config.model_dump_json(exclude_none=True)
             + str(base_url)

evalscope/models/image_edit_model.py

@@ -98,6 +98,8 @@ def generate(
         config: GenerateConfig,
     ) -> ModelOutput:
 
+        start_time = time.monotonic()
+
         # prepare generator
         kwargs: Dict[str, Any] = {}
         if config.num_inference_steps is not None:
@@ -117,10 +119,12 @@ def generate(
         output = self.model(image=input_image, prompt=prompt, **kwargs)
         image = output.images[0]
 
-        image_base64 = PIL_to_base64(image)
+        # Emit a data URI so downstream consumers (e.g. an LLM-judge request)
+        # treat the image as inline base64 instead of a local file path.
+        image_base64 = PIL_to_base64(image, add_header=True)
 
         return ModelOutput(
             model=self.model_name,
             choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])],
-            time=time.time(),
+            time=time.monotonic() - start_time,
         )

evalscope/models/litellm_compatible.py

@@ -67,7 +67,9 @@ def generate(
 
         request: Dict[str, Any] = {}
 
-        messages = openai_chat_messages(input)
+        messages = openai_chat_messages(
+            input, reasoning_format=(config.reasoning_history or 'reasoning_field'), base_url=self.base_url
+        )
         completion_params = openai_completion_params(
             model=self.model_name,
             config=config,
@@ -139,7 +141,9 @@ async def generate_async(
         """
         import litellm
 
-        messages = openai_chat_messages(input)
+        messages = openai_chat_messages(
+            input, reasoning_format=(config.reasoning_history or 'reasoning_field'), base_url=self.base_url
+        )
         completion_params = openai_completion_params(
             model=self.model_name,
             config=config,

evalscope/models/modelscope.py

@@ -223,24 +223,30 @@ def generate(
             )
             choices.append(choice)
 
+        # Aggregate usage over all returned choices: with n > 1 the loop above
+        # leaves `response` bound to the last choice only, undercounting tokens.
+        input_tokens = sum(r.input_tokens for r in responses)
+        output_tokens = sum(r.output_tokens for r in responses)
+        total_time = max(r.time for r in responses)
+
         # return output
         output = ModelOutput(
             model=self.model_name,
             choices=choices,
             usage=ModelUsage(
-                input_tokens=response.input_tokens,
-                output_tokens=response.output_tokens,
-                total_tokens=response.total_tokens,
+                input_tokens=input_tokens,
+                output_tokens=output_tokens,
+                total_tokens=input_tokens + output_tokens,
             ),
-            time=response.time,
+            time=total_time,
         )
         # Populate PerformanceMetrics from the already-available fields.
         # Local models do not produce TTFT (no streaming chunks), so ttft stays None.
         output.message.perf_metrics = PerformanceMetrics(
-            latency=response.time,
+            latency=total_time,
             ttft=None,
-            input_tokens=response.input_tokens,
-            output_tokens=response.output_tokens,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
         )
         return output
 

evalscope/models/text2image_model.py

@@ -97,6 +97,8 @@ def generate(
         config: GenerateConfig,
     ) -> ModelOutput:
 
+        start_time = time.monotonic()
+
         # prepare generator
         kwargs: Dict[str, Any] = {}
         if config.height is not None:
@@ -115,10 +117,12 @@ def generate(
         # get the first image as output
         image = self.model(prompt=prompt, **kwargs).images[0]
 
-        image_base64 = PIL_to_base64(image)
+        # Emit a data URI so downstream consumers (e.g. an LLM-judge request)
+        # treat the image as inline base64 instead of a local file path.
+        image_base64 = PIL_to_base64(image, add_header=True)
 
         return ModelOutput(
             model=self.model_name,
             choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])],
-            time=time.time(),
+            time=time.monotonic() - start_time,
         )

evalscope/models/utils/openai.py

@@ -904,8 +904,10 @@ def collect_stream_response(
         # use the finish_reason from the last chunk that generated this choice
         finish_reason = None
         for chunk in reversed(collected_chunks):
-            if chunk.choices and chunk.choices[0].index == index:
-                finish_reason = chunk.choices[0].finish_reason
+            # a single chunk may pack several choices (n > 1); match by index
+            matched = next((c for c in chunk.choices if c.index == index), None)
+            if matched is not None:
+                finish_reason = matched.finish_reason
                 break
 
         message_kwargs = {'role': 'assistant', 'content': full_reply_content}
@@ -1035,8 +1037,10 @@ async def async_collect_stream_response(
         # use the finish_reason from the last chunk that generated this choice
         finish_reason = None
         for chunk in reversed(collected_chunks):
-            if chunk.choices and chunk.choices[0].index == index:
-                finish_reason = chunk.choices[0].finish_reason
+            # a single chunk may pack several choices (n > 1); match by index
+            matched = next((c for c in chunk.choices if c.index == index), None)
+            if matched is not None:
+                finish_reason = matched.finish_reason
                 break
 
         message_kwargs = {'role': 'assistant', 'content': full_reply_content}

tests/models/test_image_model_output.py

@@ -0,0 +1,138 @@
+from typing import Any, List
+
+import pytest
+from PIL import Image
+
+from evalscope.api.messages import ChatMessageUser, ContentImage, ContentText
+from evalscope.api.model import GenerateConfig
+from evalscope.models.image_edit_model import ImageEditAPI
+from evalscope.models.text2image_model import Text2ImageAPI
+from evalscope.models.utils.openai import openai_chat_completion_part
+from evalscope.utils.io_utils import PIL_to_base64
+
+
+class _FakePipelineResult:
+
+    def __init__(self, image: Image.Image) -> None:
+        self.images = [image]
+
+
+def _install_fake_pipeline(
+    monkeypatch: Any,
+    attr_name: str,
+    image: Image.Image,
+    on_call: Any = None,
+) -> None:
+    class FakePipeline:
+
+        @classmethod
+        def from_pretrained(cls, *args: Any, **kwargs: Any) -> 'FakePipeline':
+            return cls()
+
+        def to(self, device: Any) -> 'FakePipeline':
+            return self
+
+        def __call__(self, *args: Any, **kwargs: Any) -> _FakePipelineResult:
+            if on_call is not None:
+                on_call()
+            return _FakePipelineResult(image)
+
+    import modelscope
+
+    monkeypatch.setattr(modelscope, attr_name, FakePipeline, raising=False)
+
+
+def _generated_image_content(output: Any) -> ContentImage:
+    content = output.choices[0].message.content
+    assert isinstance(content, list) and isinstance(content[0], ContentImage)
+    return content[0]
+
+
+def test_text2image_output_is_usable_as_openai_chat_input(monkeypatch: Any) -> None:
+    image = Image.new('RGB', (8, 8), color='red')
+    _install_fake_pipeline(monkeypatch, 'DiffusionPipeline', image)
+
+    api = Text2ImageAPI(model_name='test-diffusion')
+    output = api.generate(
+        input=[ChatMessageUser(content='a red square')],
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(),
+    )
+
+    part = openai_chat_completion_part(_generated_image_content(output))
+    url = part['image_url']['url']
+    assert url.startswith('data:image/')
+
+
+def test_image_edit_output_is_usable_as_openai_chat_input(monkeypatch: Any) -> None:
+    generated = Image.new('RGB', (8, 8), color='blue')
+    _install_fake_pipeline(monkeypatch, 'QwenImageEditPipeline', generated)
+
+    source_image = PIL_to_base64(Image.new('RGB', (8, 8), color='green'), format='PNG', add_header=True)
+    api = ImageEditAPI(model_name='Qwen-Image-Edit-test')
+    output = api.generate(
+        input=[
+            ChatMessageUser(content=[
+                ContentText(text='make it blue'),
+                ContentImage(image=source_image),
+            ])
+        ],
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(),
+    )
+
+    part = openai_chat_completion_part(_generated_image_content(output))
+    url = part['image_url']['url']
+    assert url.startswith('data:image/')
+
+
+def test_text2image_time_is_elapsed_seconds(monkeypatch: Any) -> None:
+    clock = [100.0]
+    monkeypatch.setattr('evalscope.models.text2image_model.time.monotonic', lambda: clock[0])
+    image = Image.new('RGB', (8, 8), color='red')
+
+    def _advance_clock() -> None:
+        clock[0] = 100.25
+
+    _install_fake_pipeline(monkeypatch, 'DiffusionPipeline', image, on_call=_advance_clock)
+
+    api = Text2ImageAPI(model_name='test-diffusion')
+    output = api.generate(
+        input=[ChatMessageUser(content='a red square')],
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(),
+    )
+
+    # elapsed seconds since generate() started, not a wall-clock epoch timestamp
+    assert output.time == pytest.approx(0.25)
+
+
+def test_image_edit_time_is_elapsed_seconds(monkeypatch: Any) -> None:
+    clock = [200.0]
+    monkeypatch.setattr('evalscope.models.image_edit_model.time.monotonic', lambda: clock[0])
+    generated = Image.new('RGB', (8, 8), color='blue')
+
+    def _advance_clock() -> None:
+        clock[0] = 200.5
+
+    _install_fake_pipeline(monkeypatch, 'QwenImageEditPipeline', generated, on_call=_advance_clock)
+
+    source_image = PIL_to_base64(Image.new('RGB', (8, 8), color='green'), format='PNG', add_header=True)
+    api = ImageEditAPI(model_name='Qwen-Image-Edit-test')
+    output = api.generate(
+        input=[
+            ChatMessageUser(content=[
+                ContentText(text='make it blue'),
+                ContentImage(image=source_image),
+            ])
+        ],
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(),
+    )
+
+    # elapsed seconds since generate() started, not a wall-clock epoch timestamp
+    assert output.time == pytest.approx(0.5)

tests/models/test_litellm_reasoning_history.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, List
+
+import litellm
+from openai.types.chat import ChatCompletion
+
+from evalscope.api.messages import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageUser,
+    ContentAudio,
+    ContentReasoning,
+    ContentText,
+)
+from evalscope.api.model import GenerateConfig
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.models.litellm_compatible import LiteLLMAPI
+
+
+def _completion_response() -> ChatCompletion:
+    return ChatCompletion.model_validate({
+        'id': 'completion-id',
+        'created': 1,
+        'model': 'test-model',
+        'object': 'chat.completion',
+        'choices': [{
+            'index': 0,
+            'finish_reason': 'stop',
+            'message': {'role': 'assistant', 'content': 'answer'},
+        }],
+        'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2},
+    })
+
+
+def _conversation() -> List[ChatMessage]:
+    return [
+        ChatMessageUser(content='question'),
+        ChatMessageAssistant(content=[ContentReasoning(reasoning='prior thoughts'), ContentText(text='prior answer')]),
+    ]
+
+
+def _capturing_completion(calls: List[Dict[str, Any]]) -> Any:
+
+    def _completion(**request: Any) -> ChatCompletion:
+        calls.append(request)
+        return _completion_response()
+
+    return _completion
+
+
+def test_reasoning_history_defaults_to_reasoning_field(monkeypatch: Any) -> None:
+    calls: List[Dict[str, Any]] = []
+    monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls))
+
+    api = LiteLLMAPI(model_name='openai/test-model')
+    api.generate(input=_conversation(), tools=[], tool_choice='none', config=GenerateConfig())
+
+    assistant_payload = calls[0]['messages'][1]
+    # parity with OpenAICompatibleAPI: reasoning lives in the top-level field,
+    # not smuggled into the content as a <think> tag
+    assert assistant_payload['reasoning_content'] == 'prior thoughts'
+    assert '<think>' not in assistant_payload['content']
+    assert assistant_payload['content'].strip() == 'prior answer'
+
+
+def test_reasoning_history_none_is_honored(monkeypatch: Any) -> None:
+    calls: List[Dict[str, Any]] = []
+    monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls))
+
+    api = LiteLLMAPI(model_name='openai/test-model')
+    api.generate(
+        input=_conversation(),
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(reasoning_history='none'),
+    )
+
+    assistant_payload = calls[0]['messages'][1]
+    assert 'reasoning_content' not in assistant_payload
+    assert '<think>' not in assistant_payload['content']
+    assert assistant_payload['content'].strip() == 'prior answer'
+
+
+def test_base_url_forwarded_for_dashscope_audio_encoding(monkeypatch: Any) -> None:
+    calls: List[Dict[str, Any]] = []
+    monkeypatch.setattr(litellm, 'completion', _capturing_completion(calls))
+
+    api = LiteLLMAPI(model_name='openai/test-model', base_url='https://dashscope.aliyuncs.com/compatible-mode/v1')
+    api.generate(
+        input=[
+            ChatMessageUser(content=[
+                ContentAudio(audio='data:audio/wav;base64,YXVkaW8=', format='wav'),
+                ContentText(text='transcribe'),
+            ])
+        ],
+        tools=[],
+        tool_choice='none',
+        config=GenerateConfig(),
+    )
+
+    audio_part = calls[0]['messages'][0]['content'][0]
+    # DashScope endpoints require the data-URI prefix on input audio
+    assert audio_part['input_audio']['data'] == 'data:audio/wav;base64,YXVkaW8='