From 7a06dba414cc70e5b804b2a5689d493b4e7ea3e6 Mon Sep 17 00:00:00 2001 From: pjgao Date: Fri, 22 May 2026 09:04:40 +0800 Subject: [PATCH 1/4] feat(perf): add --max-turn-tokens for per-turn max_tokens override in multi-turn mode Support specifying different max_tokens per turn in multi-turn stress test mode. This is essential for simulating agent tool-calling scenarios where early turns produce short outputs (e.g., 150 tokens) and the final turn produces a longer response (e.g., 1000 tokens). Usage: evalscope perf --multi-turn --max-turn-tokens 150 150 1000 If the list is shorter than the actual turn count, the last value is reused for remaining turns. Changes: - Arguments: new max_turn_tokens field with validation and CLI arg - MultiTurnStrategy: pass turn_index to api_plugin.build_request() - ApiPluginBase: add turn_index param to build_request signature - OpenaiPlugin: honor max_turn_tokens when composing request - OpenAIResponsesPlugin: same for Responses API - All other API plugins: accept turn_index for signature compat --- evalscope/perf/arguments.py | 31 +++++++++++++++++++ evalscope/perf/core/strategies/multi_turn.py | 2 +- evalscope/perf/plugin/api/base.py | 4 ++- evalscope/perf/plugin/api/custom_api.py | 2 +- evalscope/perf/plugin/api/dashscope_api.py | 2 +- evalscope/perf/plugin/api/openai_api.py | 16 +++++++--- .../perf/plugin/api/openai_embedding_api.py | 2 +- .../perf/plugin/api/openai_rerank_api.py | 2 +- .../perf/plugin/api/openai_responses_api.py | 11 ++++--- 9 files changed, 57 insertions(+), 15 deletions(-) diff --git a/evalscope/perf/arguments.py b/evalscope/perf/arguments.py index b6efcef93..03cd5ee59 100644 --- a/evalscope/perf/arguments.py +++ b/evalscope/perf/arguments.py @@ -275,6 +275,19 @@ def total_count(self) -> int: Accepts an int or a ``[min, max]`` list for uniform sampling per request. """ + max_turn_tokens: Optional[List[int]] = None + """Per-turn max_tokens override for multi-turn mode. + + A list of integers specifying max_tokens for each turn index (0-based). + Example: ``[150, 150, 150, 150, 150, 150, 150, 150, 150, 1000]`` for a + 10-turn conversation where the first 9 turns are limited to 150 tokens + and the final turn allows 1000 tokens. + + When set, this overrides ``--max-tokens`` on a per-turn basis in + ``--multi-turn`` mode. If the list is shorter than the actual turn count, + the last element is reused for remaining turns. + """ + min_tokens: Optional[int] = None """Minimum number of tokens in the response.""" @@ -360,6 +373,18 @@ def _validate_max_tokens(cls, v): raise ValueError(f'--max-tokens range values must be >= 0, got {v}') return v + @field_validator('max_turn_tokens', mode='before') + @classmethod + def _validate_max_turn_tokens(cls, v): + if v is None: + return v + if isinstance(v, list): + if not v: + raise ValueError('--max-turn-tokens must contain at least one value') + if any(x < 1 for x in v): + raise ValueError(f'--max-turn-tokens values must be >= 1, got {v}') + return v + @field_validator('multi_turn_args', mode='before') @classmethod def _validate_multi_turn_args(cls, v): @@ -642,6 +667,12 @@ def add_argument(parser: argparse.ArgumentParser): parser.add_argument( '--max-tokens', type=int, nargs='+', help='The maximum number of tokens that can be generated. ' 'Accepts 1 value (fixed) or 2 values min max for uniform sampling per request.', default=2048) + parser.add_argument( + '--max-turn-tokens', type=int, nargs='+', default=None, + help='Per-turn max_tokens override for multi-turn mode. ' + 'Pass a list of integers, one per turn (0-based). ' + 'If shorter than the turn count, the last value is reused. ' + 'Example: --max-turn-tokens 150 150 150 150 150 150 150 150 150 1000') parser.add_argument( '--min-tokens', type=int, help='The minimum number of tokens that can be generated', default=None) parser.add_argument('--n-choices', type=int, help='How many completion choices to generate', default=None) diff --git a/evalscope/perf/core/strategies/multi_turn.py b/evalscope/perf/core/strategies/multi_turn.py index 3bd30fa57..3fadcaad3 100644 --- a/evalscope/perf/core/strategies/multi_turn.py +++ b/evalscope/perf/core/strategies/multi_turn.py @@ -116,7 +116,7 @@ async def _worker(self, worker_id: int) -> None: await asyncio.sleep(interval) # Send the turn. - request = self.api_plugin.build_request(list(context)) + request = self.api_plugin.build_request(list(context), turn_index=turn_idx) benchmark_data = await self.client.post(request) # Inject multi-turn specific metadata. diff --git a/evalscope/perf/plugin/api/base.py b/evalscope/perf/plugin/api/base.py index d1d57779d..f988d6f4d 100644 --- a/evalscope/perf/plugin/api/base.py +++ b/evalscope/perf/plugin/api/base.py @@ -13,12 +13,14 @@ def __init__(self, param: Arguments) -> None: self.model_path = param.tokenizer_path @abstractmethod - def build_request(self, messages: Union[List[Dict], str], param: Optional[Arguments] = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str], param: Optional[Arguments] = None, turn_index: Optional[int] = None) -> Dict: """Build a api request body. Args: messages (List[Dict]): The messages generated by dataset. param (QueryParameters): The query parameters. + turn_index (int, optional): Current turn index in multi-turn mode. + Used for per-turn max_tokens override via ``--max-turn-tokens``. Raises: NotImplementedError: Not implemented. diff --git a/evalscope/perf/plugin/api/custom_api.py b/evalscope/perf/plugin/api/custom_api.py index 93e80f6d7..88a0bc689 100644 --- a/evalscope/perf/plugin/api/custom_api.py +++ b/evalscope/perf/plugin/api/custom_api.py @@ -37,7 +37,7 @@ def __init__(self, param: Arguments): else: self.tokenizer = None - def build_request(self, messages: Union[List[Dict], str], param: Arguments = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: """Build a custom API request body based on the input messages and parameters. This method formats the input messages into the expected request format diff --git a/evalscope/perf/plugin/api/dashscope_api.py b/evalscope/perf/plugin/api/dashscope_api.py index fa44c5bf0..d9dbf1f99 100644 --- a/evalscope/perf/plugin/api/dashscope_api.py +++ b/evalscope/perf/plugin/api/dashscope_api.py @@ -17,7 +17,7 @@ class DashScopeApiPlugin(ApiPluginBase): def __init__(self, param: Arguments): super().__init__(param) - def build_request(self, messages: List[Dict], param: Arguments = None) -> Dict: + def build_request(self, messages: List[Dict], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: """Build the openai format request based on prompt, dataset Args: diff --git a/evalscope/perf/plugin/api/openai_api.py b/evalscope/perf/plugin/api/openai_api.py index 6f665e168..ad015afc1 100644 --- a/evalscope/perf/plugin/api/openai_api.py +++ b/evalscope/perf/plugin/api/openai_api.py @@ -33,7 +33,7 @@ def __init__(self, param: Arguments): else: self.tokenizer = None - def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param: Arguments = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: """Build the openai format request based on prompt, dataset Args: @@ -41,6 +41,8 @@ def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param When param.tokenize_prompt is True, this may also be a list of token IDs (List[int]) produced by the random dataset plugin. param (QueryParameters): The query parameters. + turn_index (int, optional): Current turn index in multi-turn mode. + Used for per-turn max_tokens override via ``--max-turn-tokens``. Raises: Exception: NotImplemented @@ -55,7 +57,7 @@ def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param if param.tokenize_prompt and not isinstance(messages, dict): token_ids = self._messages_to_token_ids(messages, param) query = {'prompt': token_ids} - return self.__compose_query_from_parameter(query, param) + return self.__compose_query_from_parameter(query, param, turn_index) if param.query_template is not None: if param.query_template.startswith('@'): @@ -76,7 +78,7 @@ def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param query = {'prompt': messages} else: query = {'messages': messages} - return self.__compose_query_from_parameter(query, param) + return self.__compose_query_from_parameter(query, param, turn_index) except Exception as e: logger.exception(e) return None @@ -112,9 +114,13 @@ def _messages_to_token_ids(self, messages: Union[List[Dict], str, List[int]], pa logger.warning(f'_messages_to_token_ids: unexpected messages type {type(messages)}, returning []') return [] - def __compose_query_from_parameter(self, payload: Dict, param: Arguments): + def __compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None): payload['model'] = param.model - if param.max_tokens is not None: + if param.max_turn_tokens is not None and turn_index is not None: + # Per-turn max_tokens override for multi-turn mode. + idx = min(turn_index, len(param.max_turn_tokens) - 1) + payload['max_tokens'] = param.max_turn_tokens[idx] + elif param.max_tokens is not None: payload['max_tokens'] = _sample_int_or_range(param.max_tokens) if param.min_tokens is not None: payload['min_tokens'] = param.min_tokens diff --git a/evalscope/perf/plugin/api/openai_embedding_api.py b/evalscope/perf/plugin/api/openai_embedding_api.py index 692e99102..44f3a791d 100644 --- a/evalscope/perf/plugin/api/openai_embedding_api.py +++ b/evalscope/perf/plugin/api/openai_embedding_api.py @@ -41,7 +41,7 @@ def __init__(self, param: Arguments): else: self.tokenizer = None - def build_request(self, messages: Union[List[Dict], str, List[str]], param: Arguments = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str, List[str]], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: """Build the OpenAI embedding format request. Args: diff --git a/evalscope/perf/plugin/api/openai_rerank_api.py b/evalscope/perf/plugin/api/openai_rerank_api.py index 93b648ade..32428dc2c 100644 --- a/evalscope/perf/plugin/api/openai_rerank_api.py +++ b/evalscope/perf/plugin/api/openai_rerank_api.py @@ -42,7 +42,7 @@ def __init__(self, param: Arguments): else: self.tokenizer = None - def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: """Build the rerank format request. Args: diff --git a/evalscope/perf/plugin/api/openai_responses_api.py b/evalscope/perf/plugin/api/openai_responses_api.py index 55c86cb37..ef7299e98 100644 --- a/evalscope/perf/plugin/api/openai_responses_api.py +++ b/evalscope/perf/plugin/api/openai_responses_api.py @@ -30,7 +30,7 @@ def __init__(self, param: Arguments): else: self.tokenizer = None - def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments = None) -> Dict: + def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: param = param or self.param try: if param.query_template is not None: @@ -42,7 +42,7 @@ def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments query['input'] = normalize_responses_input(query.pop('messages')) else: query = {'input': normalize_responses_input(messages)} - return self._compose_query_from_parameter(query, param) + return self._compose_query_from_parameter(query, param, turn_index) except Exception as e: logger.exception(e) return None @@ -190,9 +190,12 @@ def _set_cached_tokens(output: Any, usage: Dict[str, Any]) -> None: if cached is not None: output.real_cached_tokens = cached - def _compose_query_from_parameter(self, payload: Dict, param: Arguments) -> Dict: + def _compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None) -> Dict: payload['model'] = param.model - if param.max_tokens is not None: + if param.max_turn_tokens is not None and turn_index is not None: + idx = min(turn_index, len(param.max_turn_tokens) - 1) + payload['max_output_tokens'] = param.max_turn_tokens[idx] + elif param.max_tokens is not None: payload['max_output_tokens'] = _sample_int_or_range(param.max_tokens) if param.stream is not None: payload['stream'] = param.stream From 3a8be9e8c1f4628173a1b9a7ea7dff07c7d97ef0 Mon Sep 17 00:00:00 2001 From: pjgao Date: Fri, 22 May 2026 09:09:49 +0800 Subject: [PATCH 2/4] docs: add --max-turn-tokens documentation for multi-turn mode Update Chinese and English docs for multi_turn and parameters pages: - multi_turn.md: new section explaining per-turn output length control with a concrete tool-call simulation example (150/1000 tokens) - parameters.md: new row for --max-turn-tokens parameter --- docs/en/user_guides/stress_test/multi_turn.md | 39 +++++++++++++++++++ docs/en/user_guides/stress_test/parameters.md | 1 + docs/zh/user_guides/stress_test/multi_turn.md | 39 +++++++++++++++++++ docs/zh/user_guides/stress_test/parameters.md | 1 + 4 files changed, 80 insertions(+) diff --git a/docs/en/user_guides/stress_test/multi_turn.md b/docs/en/user_guides/stress_test/multi_turn.md index 64f3a23e4..5085dca2b 100644 --- a/docs/en/user_guides/stress_test/multi_turn.md +++ b/docs/en/user_guides/stress_test/multi_turn.md @@ -19,6 +19,7 @@ The multi-turn conversation benchmark allows you to test a model service in real | `--min-turns` | `int` | Minimum number of user turns per conversation; used by `random_multi_turn` only | `1` | | `--max-turns` | `int` | Maximum number of user turns per conversation; **required** for `random_multi_turn`; optional for ShareGPT / `custom_multi_turn` datasets to truncate long conversations; for `swe_smith` live construction, the per-conversation turn count is sampled from `[min_turns, max_turns]` | `None` | | `--dataset-offset` | `int` | Skip the first N conversations in the dataset; useful for sharded testing or avoiding cache hits | `0` | +| `--max-turn-tokens` | `list[int]` | Per-turn `max_tokens` override; accepts a list of integers specifying the maximum output tokens for each turn by index (0-based). When the list is shorter than the actual turn count, the last value is reused. Only effective in `--multi-turn` mode | `None` | ### `multi_turn_args` (swe_smith-specific parameters) @@ -266,6 +267,44 @@ Runtime context structure (when sending turn 2): > **Note**: The `assistant` messages in the dataset are used only to identify conversation structure and are **never** sent directly to the model. At runtime, workers always append the model's actual output to the context to ensure accurate history. +### Per-turn Output Length Control (`--max-turn-tokens`) + +When simulating Agent tool-calling performance, an open-source model cannot produce tool-call structured outputs like the actual model, resulting in different per-turn output lengths. `--max-turn-tokens` allows you to limit the model's output length on a per-turn basis, approximating the context growth behavior of the real model. + +**Usage example**: A 10-turn conversation where the first 9 turns simulate tool calls (150 tokens each) and the final turn produces a complete answer (1000 tokens). + +First, prepare a JSONL data file (one 10-turn conversation per line, with a system prompt of ~4000 tokens): + +```json +[{"role": "system", "content": "<4000 token system prompt>"}, {"role": "user", "content": "Analyze this code"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Continue"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "Provide the final answer"}] +``` + +> **Note**: The `assistant` messages only define the conversation structure and are replaced by the model's real outputs at runtime. + +Then run the benchmark: + +```bash +evalscope perf \\ + --model YOUR_MODEL \\ + --url OPENAI_API_COMPAT_URL \\ + --api openai \\ + --dataset custom_multi_turn \\ + --dataset-path /path/to/tool_call_sim.jsonl \\ + --multi-turn \\ + --max-turn-tokens 150 150 150 150 150 150 150 150 150 1000 \\ + --number 50 \\ + --parallel 10 \\ + --extra-args '{"ignore_eos": true}' +``` + +| Turn | `max_tokens` | Simulated behavior | +|------|-------------|--------------------| +| Turn 1 | 150 | Simulate initial tool call | +| Turns 2-9 | 150 | Simulate intermediate tool calls | +| Turn 10 | 1000 | Final complete answer | + +> **Tip**: The list is automatically extended by reusing the last value. For example, `--max-turn-tokens 150 1000` in a 10-turn conversation results in `[150, 150, 150, 150, 150, 150, 150, 150, 150, 1000]`. + **Usage example**: You have conversation data already in OpenAI messages format and want to benchmark directly without any format conversion. First, prepare the JSONL data file (one conversation per line): diff --git a/docs/en/user_guides/stress_test/parameters.md b/docs/en/user_guides/stress_test/parameters.md index e2a0d3a19..649810d5f 100644 --- a/docs/en/user_guides/stress_test/parameters.md +++ b/docs/en/user_guides/stress_test/parameters.md @@ -143,6 +143,7 @@ Must be used with `--multi-turn`. See the [Multi-turn Benchmark Guide](./multi_t | `--frequency-penalty` | `float` | frequency_penalty value | - | | `--logprobs` | `bool` | Whether to return logarithmic probabilities | - | | `--max-tokens` | `int` | Maximum number of tokens that can be generated | - | +| `--max-turn-tokens` | `int list` | **Multi-turn mode only**: Per-turn override of `max_tokens`
• Accepts a list of integers specifying max tokens per turn (0-based index)
• Last value is reused if the list is shorter than the actual turn count
• Only effective in `--multi-turn` mode
• Example: `--max-turn-tokens 150 150 150 1000` | `None` | | `--min-tokens` | `int` | Minimum number of tokens to generate
Note: Not all model services support this parameter
For `vLLM>=0.8.1`, you need to additionally set
`--extra-args '{"ignore_eos": true}'` | - | | `--n-choices` | `int` | Number of completion choices to generate | - | | `--seed` | `int` | Random seed | `None` | diff --git a/docs/zh/user_guides/stress_test/multi_turn.md b/docs/zh/user_guides/stress_test/multi_turn.md index ec6cb1819..bf143295f 100644 --- a/docs/zh/user_guides/stress_test/multi_turn.md +++ b/docs/zh/user_guides/stress_test/multi_turn.md @@ -19,6 +19,7 @@ | `--min-turns` | `int` | 每个对话最少用户轮数，仅 `random_multi_turn` 使用 | `1` | | `--max-turns` | `int` | 每个对话最多用户轮数；`random_multi_turn` **必须设置**；ShareGPT / `custom_multi_turn` 等数据集可选，用于截断过长对话；`swe_smith` live 构建时每条对话轮次从 `[min_turns, max_turns]` 随机采样 | `None` | | `--dataset-offset` | `int` | 跳过数据集前 N 条对话，用于分片测试或避免缓存命中 | `0` | +| `--max-turn-tokens` | `list[int]` | 逐轮 `max_tokens` 覆盖值；接受一个整数列表，按 turn index（从 0 开始）指定每轮的最大输出 token 数。列表短于实际轮数时，复用最后一个值。仅在 `--multi-turn` 模式下生效 | `None` | ### `multi_turn_args`（`swe_smith` 专属参数） @@ -266,6 +267,44 @@ evalscope perf \ > **说明**：数据集中的 `assistant` 消息仅用于标识对话结构，**不会**被直接发送给模型。运行时 worker 始终将模型的实际输出追加到上下文，保证历史准确。 +### 逐轮控制输出长度（`--max-turn-tokens`） + +在模拟 Agent 工具调用性能的场景中，开源模型无法像实际模型那样输出工具调用结构，导致每轮输出长度与实际模型不同。通过 `--max-turn-tokens` 可以逐轮限制模型的输出长度，从而近似模拟实际模型的上下文增长行为。 + +**使用示例**：10 轮对话，前 9 轮模拟工具调用（各 150 token），最后一轮输出完整回答（1000 token）。 + +首先准备 JSONL 数据文件（每行一条 10 轮对话，system prompt 约 4000 token）： + +```json +[{"role": "system", "content": "<4000 token 的系统提示>"}, {"role": "user", "content": "帮我分析这段代码"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "继续"}, {"role": "assistant", "content": "x"}, {"role": "user", "content": "请给出完整的最终回答"}] +``` + +> **说明**：assistant 消息仅定义对话结构，实际运行中会被模型的真实输出替换。 + +然后运行压测： + +```bash +evalscope perf \\ + --model YOUR_MODEL \\ + --url OPENAI_API_COMPAT_URL \\ + --api openai \\ + --dataset custom_multi_turn \\ + --dataset-path /path/to/tool_call_sim.jsonl \\ + --multi-turn \\ + --max-turn-tokens 150 150 150 150 150 150 150 150 150 1000 \\ + --number 50 \\ + --parallel 10 \\ + --extra-args '{"ignore_eos": true}' +``` + +| 轮次 | `max_tokens` | 模拟效果 | +|------|-------------|---------| +| 第 1 轮 | 150 | 模拟首次工具调用 | +| 第 2-9 轮 | 150 | 模拟中间轮工具调用 | +| 第 10 轮 | 1000 | 最终完整回答 | + +> **提示**：列表长度不足时自动复用最后一个值。例如 `--max-turn-tokens 150 1000` 在 10 轮对话中效果为 `[150, 150, 150, 150, 150, 150, 150, 150, 150, 1000]`。 + **使用示例**：适用场景：已有 OpenAI messages 格式的对话数据，直接用于多轮压测，无需转换格式。 首先准备 JSONL 数据文件（每行一条对话）： diff --git a/docs/zh/user_guides/stress_test/parameters.md b/docs/zh/user_guides/stress_test/parameters.md index 79e2db1da..a180c5fc3 100644 --- a/docs/zh/user_guides/stress_test/parameters.md +++ b/docs/zh/user_guides/stress_test/parameters.md @@ -144,6 +144,7 @@ SLA自动调优功能使用详见[自动调优指南](./sla_auto_tune.md)。 | `--frequency-penalty` | `float` | frequency_penalty值 | - | | `--logprobs` | `bool` | 是否返回对数概率 | - | | `--max-tokens` | `int` 或 `int int` | 可以生成的最大token数量
• 单个整数：固定值，如 `--max-tokens 2048`
• 两个整数：`最小值 最大值`，每次请求从该范围均匀随机采样，如 `--max-tokens 512 2048` | `2048` | +| `--max-turn-tokens` | `int list` | **多轮模式专属**：逐轮覆盖 `max_tokens`
• 接受整数列表，按 turn index（0-based）指定每轮的最大输出 token 数
• 列表短于实际轮数时，复用最后一个值
• 仅在 `--multi-turn` 模式下生效，否则忽略
• 示例：`--max-turn-tokens 150 150 150 1000` | `None` | | `--min-tokens` | `int` | 生成的最少token数量
注意：并非所有模型服务都支持
对于`vLLM>=0.8.1`，需额外设置
`--extra-args '{"ignore_eos": true}'` | - | | `--n-choices` | `int` | 生成的补全选择数量 | - | | `--seed` | `int` | 随机种子 | `None` | From 9b28a0fedf665ecbe7d5ca90c53e8248fe09d7d0 Mon Sep 17 00:00:00 2001 From: pjgao Date: Fri, 22 May 2026 10:08:36 +0800 Subject: [PATCH 3/4] fix: address code review comments - Optional imports, validator, doc examples - Add missing Optional import to openai_api, openai_responses_api, dashscope_api, custom_api, openai_embedding_api, openai_rerank_api - Improve max_turn_tokens validator: allow >= 0 for consistency with max_tokens, coerce single int to list for programmatic API - Fix doc tip examples: correct the list extension behavior description in both en and zh multi_turn.md --- docs/en/user_guides/stress_test/multi_turn.md | 2 +- docs/zh/user_guides/stress_test/multi_turn.md | 2 +- evalscope/perf/arguments.py | 7 +++++-- evalscope/perf/plugin/api/custom_api.py | 2 +- evalscope/perf/plugin/api/dashscope_api.py | 2 +- evalscope/perf/plugin/api/openai_api.py | 2 +- evalscope/perf/plugin/api/openai_embedding_api.py | 2 +- evalscope/perf/plugin/api/openai_rerank_api.py | 2 +- evalscope/perf/plugin/api/openai_responses_api.py | 2 +- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/en/user_guides/stress_test/multi_turn.md b/docs/en/user_guides/stress_test/multi_turn.md index 5085dca2b..0ef8eaf45 100644 --- a/docs/en/user_guides/stress_test/multi_turn.md +++ b/docs/en/user_guides/stress_test/multi_turn.md @@ -303,7 +303,7 @@ evalscope perf \\ | Turns 2-9 | 150 | Simulate intermediate tool calls | | Turn 10 | 1000 | Final complete answer | -> **Tip**: The list is automatically extended by reusing the last value. For example, `--max-turn-tokens 150 1000` in a 10-turn conversation results in `[150, 150, 150, 150, 150, 150, 150, 150, 150, 1000]`. +> **Tip**: The list is automatically extended by reusing the last value for all subsequent turns. For example, `--max-turn-tokens 150 1000` in a 10-turn conversation results in `[150, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]` (the first turn is 150, and all subsequent turns are 1000). **Usage example**: You have conversation data already in OpenAI messages format and want to benchmark directly without any format conversion. diff --git a/docs/zh/user_guides/stress_test/multi_turn.md b/docs/zh/user_guides/stress_test/multi_turn.md index bf143295f..eb4b14851 100644 --- a/docs/zh/user_guides/stress_test/multi_turn.md +++ b/docs/zh/user_guides/stress_test/multi_turn.md @@ -303,7 +303,7 @@ evalscope perf \\ | 第 2-9 轮 | 150 | 模拟中间轮工具调用 | | 第 10 轮 | 1000 | 最终完整回答 | -> **提示**：列表长度不足时自动复用最后一个值。例如 `--max-turn-tokens 150 1000` 在 10 轮对话中效果为 `[150, 150, 150, 150, 150, 150, 150, 150, 150, 1000]`。 +> **提示**：列表长度不足时自动复用最后一个值给后续所有轮次。例如 `--max-turn-tokens 150 1000` 在 10 轮对话中效果为 `[150, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]`（第一轮为 150，后续均为 1000）。 **使用示例**：适用场景：已有 OpenAI messages 格式的对话数据，直接用于多轮压测，无需转换格式。 diff --git a/evalscope/perf/arguments.py b/evalscope/perf/arguments.py index 03cd5ee59..99137c6b5 100644 --- a/evalscope/perf/arguments.py +++ b/evalscope/perf/arguments.py @@ -378,11 +378,14 @@ def _validate_max_tokens(cls, v): def _validate_max_turn_tokens(cls, v): if v is None: return v + # Coerce single int to list for programmatic API support + if isinstance(v, (int, float)): + v = [int(v)] if isinstance(v, list): if not v: raise ValueError('--max-turn-tokens must contain at least one value') - if any(x < 1 for x in v): - raise ValueError(f'--max-turn-tokens values must be >= 1, got {v}') + if any(x < 0 for x in v): + raise ValueError(f'--max-turn-tokens values must be >= 0, got {v}') return v @field_validator('multi_turn_args', mode='before') diff --git a/evalscope/perf/plugin/api/custom_api.py b/evalscope/perf/plugin/api/custom_api.py index 88a0bc689..20a59c514 100644 --- a/evalscope/perf/plugin/api/custom_api.py +++ b/evalscope/perf/plugin/api/custom_api.py @@ -1,6 +1,6 @@ import aiohttp import json -from typing import Any, AsyncGenerator, Dict, List, Tuple, Union +from typing import Any, AsyncGenerator, Dict, List, Tuple, Union, Optional from evalscope.perf.arguments import Arguments from evalscope.perf.multi_turn_args import _sample_int_or_range diff --git a/evalscope/perf/plugin/api/dashscope_api.py b/evalscope/perf/plugin/api/dashscope_api.py index d9dbf1f99..de4070fec 100644 --- a/evalscope/perf/plugin/api/dashscope_api.py +++ b/evalscope/perf/plugin/api/dashscope_api.py @@ -1,6 +1,6 @@ import json import os -from typing import Any, Dict, Iterator, List +from typing import Any, Dict, Iterator, List, Optional from evalscope.perf.arguments import Arguments from evalscope.perf.multi_turn_args import _sample_int_or_range diff --git a/evalscope/perf/plugin/api/openai_api.py b/evalscope/perf/plugin/api/openai_api.py index ad015afc1..3696aac58 100644 --- a/evalscope/perf/plugin/api/openai_api.py +++ b/evalscope/perf/plugin/api/openai_api.py @@ -2,7 +2,7 @@ import math import os from collections import defaultdict -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union, Optional from evalscope.perf.arguments import Arguments from evalscope.perf.multi_turn_args import _sample_int_or_range diff --git a/evalscope/perf/plugin/api/openai_embedding_api.py b/evalscope/perf/plugin/api/openai_embedding_api.py index 44f3a791d..befa5f345 100644 --- a/evalscope/perf/plugin/api/openai_embedding_api.py +++ b/evalscope/perf/plugin/api/openai_embedding_api.py @@ -4,7 +4,7 @@ import sys import time import traceback -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union, Optional from evalscope.perf.arguments import Arguments from evalscope.perf.plugin.api.base import ApiPluginBase diff --git a/evalscope/perf/plugin/api/openai_rerank_api.py b/evalscope/perf/plugin/api/openai_rerank_api.py index 32428dc2c..a55c5614b 100644 --- a/evalscope/perf/plugin/api/openai_rerank_api.py +++ b/evalscope/perf/plugin/api/openai_rerank_api.py @@ -3,7 +3,7 @@ import sys import time import traceback -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union, Optional from evalscope.perf.arguments import Arguments from evalscope.perf.plugin.api.base import ApiPluginBase diff --git a/evalscope/perf/plugin/api/openai_responses_api.py b/evalscope/perf/plugin/api/openai_responses_api.py index ef7299e98..e6e89231a 100644 --- a/evalscope/perf/plugin/api/openai_responses_api.py +++ b/evalscope/perf/plugin/api/openai_responses_api.py @@ -1,7 +1,7 @@ import json import time from collections import defaultdict -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union, Optional from evalscope.models.utils.openai_responses import ( normalize_responses_input, From f06de3619e140d700536b60a30d4a7bcac47fb18 Mon Sep 17 00:00:00 2001 From: pjgao Date: Fri, 22 May 2026 12:56:40 +0800 Subject: [PATCH 4/4] feat(perf): support tools definitions in custom_multi_turn dataset - custom_multi_turn: extract tools from JSON data and embed into first turn - openai_api: extract embedded tools and inject into request payload - openai_responses_api: same tools support for Responses API - Supports JSON format: {"messages": [...], "tools": [...]} - Backward compatible: works without tools definitions --- evalscope/perf/plugin/api/openai_api.py | 28 ++- .../perf/plugin/api/openai_responses_api.py | 22 ++- evalscope/perf/plugin/datasets/custom.py | 187 +++++++++++++++--- 3 files changed, 201 insertions(+), 36 deletions(-) diff --git a/evalscope/perf/plugin/api/openai_api.py b/evalscope/perf/plugin/api/openai_api.py index 3696aac58..905c076ff 100644 --- a/evalscope/perf/plugin/api/openai_api.py +++ b/evalscope/perf/plugin/api/openai_api.py @@ -14,6 +14,23 @@ logger = get_logger() +_TOOL_CONTEXT_KEY = "__evalscope_tools__" + + +def _extract_tools(messages) -> Optional[List[Dict]]: + """Extract tools definitions from messages if embedded by the dataset plugin. + + Scans the first message for the internal tools key. If found, removes it + from the message to keep the payload clean before sending. + """ + if not isinstance(messages, list): + return None + for msg in messages: + if isinstance(msg, dict) and _TOOL_CONTEXT_KEY in msg: + tools = msg.pop(_TOOL_CONTEXT_KEY) + return tools + return None + @register_api(['openai', 'local_vllm', 'local']) class OpenaiPlugin(DefaultApiPlugin): @@ -52,12 +69,15 @@ def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param """ param = param or self.param try: + # Extract tools definitions embedded by the dataset plugin. + tools = _extract_tools(messages) + # --tokenize-prompt path: convert messages/text/token-IDs to a token-ID list # and send as a /v1/completions request with `prompt=[int, ...]`. if param.tokenize_prompt and not isinstance(messages, dict): token_ids = self._messages_to_token_ids(messages, param) query = {'prompt': token_ids} - return self.__compose_query_from_parameter(query, param, turn_index) + return self.__compose_query_from_parameter(query, param, turn_index, tools) if param.query_template is not None: if param.query_template.startswith('@'): @@ -78,7 +98,7 @@ def build_request(self, messages: Union[List[Dict], str, List[int], Dict], param query = {'prompt': messages} else: query = {'messages': messages} - return self.__compose_query_from_parameter(query, param, turn_index) + return self.__compose_query_from_parameter(query, param, turn_index, tools) except Exception as e: logger.exception(e) return None @@ -114,8 +134,10 @@ def _messages_to_token_ids(self, messages: Union[List[Dict], str, List[int]], pa logger.warning(f'_messages_to_token_ids: unexpected messages type {type(messages)}, returning []') return [] - def __compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None): + def __compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None, tools: Optional[List[Dict]] = None): payload['model'] = param.model + if tools: + payload['tools'] = tools if param.max_turn_tokens is not None and turn_index is not None: # Per-turn max_tokens override for multi-turn mode. idx = min(turn_index, len(param.max_turn_tokens) - 1) diff --git a/evalscope/perf/plugin/api/openai_responses_api.py b/evalscope/perf/plugin/api/openai_responses_api.py index e6e89231a..ca512b587 100644 --- a/evalscope/perf/plugin/api/openai_responses_api.py +++ b/evalscope/perf/plugin/api/openai_responses_api.py @@ -18,6 +18,19 @@ logger = get_logger() +_TOOL_CONTEXT_KEY = "__evalscope_tools__" + + +def _extract_tools(messages) -> Optional[List[Dict]]: + """Extract tools definitions from messages if embedded by the dataset plugin.""" + if not isinstance(messages, list): + return None + for msg in messages: + if isinstance(msg, dict) and _TOOL_CONTEXT_KEY in msg: + tools = msg.pop(_TOOL_CONTEXT_KEY) + return tools + return None + @register_api(['openai_responses', 'openai_response', 'responses']) class OpenAIResponsesPlugin(DefaultApiPlugin): @@ -33,6 +46,9 @@ def __init__(self, param: Arguments): def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments = None, turn_index: Optional[int] = None) -> Dict: param = param or self.param try: + # Extract tools definitions embedded by the dataset plugin. + tools = _extract_tools(messages) + if param.query_template is not None: query = self._load_query_template(param.query_template) query['input'] = normalize_responses_input(messages) @@ -42,7 +58,7 @@ def build_request(self, messages: Union[List[Dict], str, Dict], param: Arguments query['input'] = normalize_responses_input(query.pop('messages')) else: query = {'input': normalize_responses_input(messages)} - return self._compose_query_from_parameter(query, param, turn_index) + return self._compose_query_from_parameter(query, param, turn_index, tools) except Exception as e: logger.exception(e) return None @@ -190,8 +206,10 @@ def _set_cached_tokens(output: Any, usage: Dict[str, Any]) -> None: if cached is not None: output.real_cached_tokens = cached - def _compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None) -> Dict: + def _compose_query_from_parameter(self, payload: Dict, param: Arguments, turn_index: Optional[int] = None, tools: Optional[List[Dict]] = None) -> Dict: payload['model'] = param.model + if tools: + payload['tools'] = tools if param.max_turn_tokens is not None and turn_index is not None: idx = min(turn_index, len(param.max_turn_tokens) - 1) payload['max_output_tokens'] = param.max_turn_tokens[idx] diff --git a/evalscope/perf/plugin/datasets/custom.py b/evalscope/perf/plugin/datasets/custom.py index ca936705f..191692a3c 100644 --- a/evalscope/perf/plugin/datasets/custom.py +++ b/evalscope/perf/plugin/datasets/custom.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict, Iterator, List +from typing import Any, Dict, Iterator, List, Optional, Tuple from evalscope.perf.arguments import Arguments from evalscope.perf.plugin.datasets.base import DatasetPluginBase, Message, Messages @@ -8,6 +8,11 @@ logger = get_logger() +# Internal key used to carry tools definitions through the conversation pipeline. +# The dataset plugin embeds tools in the first message; the API plugin extracts +# and injects them into the request body, then strips the key before sending. +_TOOL_CONTEXT_KEY = "__evalscope_tools__" + @register_dataset('custom') class CustomDatasetPlugin(DatasetPluginBase): @@ -57,14 +62,21 @@ def __init__(self, query_parameters: Arguments): def _split_into_turns(self, messages: List[Message]) -> List[Messages]: """Split a flat message list into per-turn delta lists. - Uses ``assistant`` messages as turn boundaries. Each run of - non-assistant messages before an ``assistant`` message (or at the end - of the conversation) forms one turn's delta. + Turn boundaries are ``assistant`` messages that mark the end of a + conversational turn (i.e., followed by a ``user`` message or at the + end of the conversation). ``assistant`` messages that are part of a + tool-calling chain (followed by ``tool`` responses) are retained in the + current turn so that ``tool`` messages have their corresponding + ``assistant`` ``tool_calls`` in the context. - Example:: + Example (standard conversation):: [system, user_1, assistant_ref, user_2, assistant_ref, user_3] -> [[system, user_1], [user_2], [user_3]] + Example (agent with tool calls):: + [system, user, assistant(tool_calls), tool, assistant(content)] + -> [[system, user], [assistant(tool_calls), tool, assistant(content)]] + Args: messages: Flat list of OpenAI message dicts. @@ -73,50 +85,62 @@ def _split_into_turns(self, messages: List[Message]) -> List[Messages]: """ turns: List[Messages] = [] current: Messages = [] - for msg in messages: + for idx, msg in enumerate(messages): if msg.get('role') == 'assistant': - if current: - turns.append(current) - current = [] - # assistant message acts as boundary only; content is discarded + # Check if this assistant is followed by a tool response. + # If so, it's part of a tool-calling chain and should be kept. + next_msg = messages[idx + 1] if idx + 1 < len(messages) else None + if next_msg and next_msg.get('role') == 'tool': + # Part of tool-calling chain – keep in current turn. + current.append(msg) + else: + # Turn boundary – finalize current turn and discard this + # assistant (the runner will append the model's real output). + if current: + turns.append(current) + current = [] else: current.append(msg) if current: turns.append(current) return turns + +def _embed_tools(turn_delta: Messages, tools: List[Dict]) -> None: + """Embed tools definitions into the first message of a turn delta. + + Uses an internal key that the API plugin extracts and injects into the + request body, then strips before sending. + """ + if turn_delta: + first_msg = turn_delta[0] + first_msg[_TOOL_CONTEXT_KEY] = tools + def build_messages(self) -> Iterator[List[Messages]]: - """Yield complete conversations as ``List[Messages]`` from the JSONL file. + """Yield complete conversations from a JSONL or JSON file. + + Supported file formats: + - **JSONL**: one JSON array per line (existing behavior). + - **JSON**: a top-level JSON array of conversations, one per element. Each yielded item is a ``List[Messages]`` where every ``Messages`` contains the delta for one turn. The multi-turn benchmark runner extends the growing context with each delta and appends the model's real response after each turn. + + If tools definitions are present in the source data, they are embedded + into the first message of the first turn using an internal key that is + later extracted by the API plugin and injected into the request body. """ max_turns = self.query_parameters.max_turns - for line in self.dataset_line_by_line(self.query_parameters.dataset_path): - line = line.strip() - if not line: - continue - - try: - messages = json.loads(line) - except json.JSONDecodeError as e: - logger.warning(f'Skipping malformed JSON line: {e}') - continue - - if not isinstance(messages, list) or not messages: - logger.warning('Skipping line: expected a non-empty JSON array.') - continue - - # Validate that every element has role and content fields - if not all(isinstance(m, dict) and 'role' in m and 'content' in m for m in messages): - logger.warning('Skipping line: each message must have "role" and "content" fields.') - continue - + for messages, tools in self._iter_conversations(): turns = self._split_into_turns(messages) + # Embed tools into the first turn so they flow through the pipeline. + if tools and turns: + _embed_tools(turns[0], tools) + # Apply max_turns truncation at the dataset layer if max_turns is not None: turns = turns[:max_turns] @@ -141,6 +165,107 @@ def build_messages(self) -> Iterator[List[Messages]]: if is_valid: yield turns + def _iter_conversations(self) -> Iterator[Tuple[List[Message], Optional[List[Dict]]]]: + """Iterate (messages, tools) tuples from the dataset file, auto-detecting format. + + Supported formats (auto-detected): + 1. **Single JSON object**: ``{"model": "...", "messages": [...], "tools": [...]}`` + – extracts ``messages`` and ``tools``. + 2. **JSON array of conversation objects**: ``[{"messages": [...], "tools": [...]}, ...]``. + 3. **JSON array of message arrays**: ``[[...], [...]]`` – no tools. + 4. **JSONL**: one JSON array (or conversation object) per line. + """ + path = self.query_parameters.dataset_path + + try: + with open(path, 'r') as f: + data = json.load(f) + except (json.JSONDecodeError, UnicodeDecodeError): + # Not valid JSON – fall back to JSONL + yield from self._iter_jsonl(path) + return + + if isinstance(data, dict): + # Single conversation object + messages = data.get('messages') + if messages and isinstance(messages, list): + tools = data.get('tools') + if tools and isinstance(tools, list): + yield messages, tools + else: + yield messages, None + return + + if isinstance(data, list): + if not data: + return + first = data[0] + + if isinstance(first, dict): + # Array of conversation objects: each may have "messages" and "tools" + for item in data: + if isinstance(item, dict): + messages = item.get('messages') + if messages and isinstance(messages, list): + tools = item.get('tools') + if tools and isinstance(tools, list): + yield messages, tools + else: + yield messages, None + elif isinstance(item, list) and item: + yield item, None + return + + if isinstance(first, list): + # Array of message arrays + for conv in data: + if isinstance(conv, list) and conv: + yield conv, None + return + + # Fallback + logger.warning(f'Unsupported JSON structure in {path}, falling back to JSONL') + yield from self._iter_jsonl(path) + + def _iter_jsonl(self) -> Iterator[Tuple[List[Message], Optional[List[Dict]]]]: + """Read JSONL file: one JSON array (or conversation object) per line.""" + path = self.query_parameters.dataset_path + for line in self.dataset_line_by_line(path): + line = line.strip() + if not line: + continue + + try: + item = json.loads(line) + except json.JSONDecodeError as e: + logger.warning(f'Skipping malformed JSON line: {e}') + continue + + # Support both raw message array and conversation object in JSONL + if isinstance(item, dict): + messages = item.get('messages') + if not messages or not isinstance(messages, list): + continue + tools = item.get('tools') + if not tools or not isinstance(tools, list): + tools = None + elif isinstance(item, list): + messages = item + tools = None + else: + continue + + if not messages: + logger.warning('Skipping line: empty messages array.') + continue + + # Validate that every element has role and content fields + if not all(isinstance(m, dict) and 'role' in m and 'content' in m for m in messages): + logger.warning('Skipping line: each message must have "role" and "content" fields.') + continue + + yield messages, tools + if __name__ == '__main__': from evalscope.perf.arguments import Arguments