diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 2fa4491e..c34195d9 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -2,6 +2,9 @@ name: Lint
on: [push, pull_request]
+permissions:
+ contents: read
+
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
@@ -29,33 +32,5 @@ jobs:
run: |
python .github/scripts/check_imports.py
- - name: Run pre-commit (auto-fix)
- id: pre_commit_auto_fix
- run: |
- # 运行 pre-commit，允许自动修复，不因修复而失败
- pre-commit run --all-files || true
-
- - name: Check for changes
- id: check_changes
- run: |
- if [[ -n $(git status --porcelain) ]]; then
- echo "changed=true" >> $GITHUB_OUTPUT
- echo "📝 Files were modified by pre-commit auto-fix"
- else
- echo "changed=false" >> $GITHUB_OUTPUT
- echo "✅ No auto-fix changes"
- fi
-
- - name: Commit auto-fix changes
- if: steps.check_changes.outputs.changed == 'true' && github.event_name == 'push'
- run: |
- git config --local user.email "action@github.com"
- git config --local user.name "GitHub Action"
- git add -A
- git commit -m "🎨 Auto-format code with pre-commit"
- git push
-
- - name: Run pre-commit (final check)
- run: |
- # 再次运行 pre-commit，这次如果有错误就真的失败
- pre-commit run --all-files
+ - name: Run pre-commit
+ run: pre-commit run --all-files
diff --git a/AGENTS.md b/AGENTS.md
index 088b0898..aaa0d698 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -49,6 +49,8 @@ dingo/
│ │ ├── rule/ ← Rule-based evaluators (80+ built-in)
│ │ │ ├── base.py ← BaseRule
│ │ │ ├── rule_common.py ← Common rules (text quality, format, PII, etc.)
+│ │ │ ├── guobiao/
+│ │ │ │ └── rule_tc609_quality.py ← TC609 quality metrics and placeholders
│ │ │ ├── rule_search_ranking.py ← IR ranking metrics (NDCG, MRR, Recall, Precision, MAP, HitRate)
│ │ │ └── utils/ ← Shared utilities (normalize, ngrams, etc.)
│ │ └── llm/ ← LLM-based evaluators
diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py
index 7ec09736..d6cc310d 100644
--- a/dingo/config/input_args.py
+++ b/dingo/config/input_args.py
@@ -25,6 +25,7 @@ class DatasetSqlArgs(BaseModel):
port: str = ''
database: str = ''
connect_args: str = '' # 连接参数，如 ?charset=utf8mb4
+ engine_args: str = '' # SQLAlchemy 引擎参数，如 pool_recycle=1800&pool_pre_ping=true
class DatasetExcelArgs(BaseModel):
@@ -193,6 +194,3 @@ class InputArgs(BaseModel):
dataset: DatasetArgs = DatasetArgs()
executor: ExecutorArgs = ExecutorArgs()
evaluator: List[EvalPipline] = []
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
diff --git a/dingo/data/converter/base.py b/dingo/data/converter/base.py
index 8707bb79..bc88d107 100644
--- a/dingo/data/converter/base.py
+++ b/dingo/data/converter/base.py
@@ -238,6 +238,23 @@ def _convert(raw: Union[str, Dict]):
return _convert
+@BaseConverter.register("md")
+class MarkdownConverter(BaseConverter):
+ """Markdown file converter."""
+
+ def __init__(self):
+ super().__init__()
+
+ @classmethod
+ def convertor(cls, input_args: InputArgs) -> Callable:
+ def _convert(raw: Union[str, Dict]):
+ if isinstance(raw, Dict):
+ return Data(**raw)
+ return Data(**{"id": "", "content": str(raw)})
+
+ return _convert
+
+
@BaseConverter.register("jsonl")
class JsonLineConverter(BaseConverter):
"""Json line file converter."""
diff --git a/dingo/data/datasource/local.py b/dingo/data/datasource/local.py
index fb5a5f97..a8176b7a 100644
--- a/dingo/data/datasource/local.py
+++ b/dingo/data/datasource/local.py
@@ -32,7 +32,7 @@ def to_dict(self) -> Dict[str, Any]:
"config_name": self.config_name,
}
- def load(self, **kwargs) -> Generator[str, None, None]:
+ def load(self, **kwargs) -> Generator[Any, None, None]:
"""Load the local file dataset based on `LocalDataSource`.
Args:
kwargs: Additional keyword arguments used for loading the dataset.
@@ -51,10 +51,39 @@ def _find_all_files(self, path: str, file_list: List[str]):
for _f in os.listdir(path):
f = os.path.join(path, _f)
if os.path.isfile(f):
- file_list.append(f)
+ if getattr(self.input_args.dataset, "format", None) == "jsonl":
+ if f.lower().endswith(".jsonl"):
+ file_list.append(f)
+ elif getattr(self.input_args.dataset, "format", None) == "json":
+ if f.lower().endswith(".json"):
+ file_list.append(f)
+ elif getattr(self.input_args.dataset, "format", None) == "md":
+ if f.lower().endswith(".md"):
+ file_list.append(f)
+ else:
+ file_list.append(f)
if os.path.isdir(f):
self._find_all_files(f, file_list)
+ @staticmethod
+ def _load_md_file(path: str) -> Dict[str, str]:
+ try:
+ with open(path, "r", encoding="utf-8") as md_file:
+ return {
+ "id": os.path.basename(path),
+ "content": md_file.read(),
+ }
+ except UnicodeDecodeError as decode_error:
+ raise RuntimeError(
+ f'Failed to read markdown file "{path}": Unsupported file encoding. '
+ f'Markdown files must be UTF-8 encoded. Original error: {str(decode_error)}'
+ )
+ except Exception as e:
+ raise RuntimeError(
+ f'Unexpected error reading markdown file "{path}": {str(e)}. '
+ f'Please check if the file exists and is readable.'
+ )
+
def _load_excel_file_xlsx(self, path: str) -> Generator[str, None, None]:
"""
Load an .xlsx Excel file and return its contents row by row as JSON strings.
@@ -377,7 +406,7 @@ def _load_excel_file_xls(self, path: str) -> Generator[str, None, None]:
if wb:
wb.release_resources()
- def _load_local_file(self) -> Generator[str, None, None]:
+ def _load_local_file(self) -> Generator[Any, None, None]:
"""
Load a local file and return its contents.
@@ -390,14 +419,27 @@ def _load_local_file(self) -> Generator[str, None, None]:
raise RuntimeError(f'"{self.path}" is not a valid path')
f_list = []
+ input_is_file = os.path.isfile(self.path)
if os.path.exists(self.path) and os.path.isfile(self.path):
f_list = [self.path]
elif os.path.exists(self.path) and os.path.isdir(self.path):
self._find_all_files(self.path, f_list)
- by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2"]
+ by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2", "md"]
for f in f_list:
+ if self.input_args.dataset.format == "md":
+ if f.lower().endswith(".md"):
+ yield self._load_md_file(f)
+ elif input_is_file:
+ raise RuntimeError(
+ f'Input file "{self.path}" is not a markdown file. '
+ f'Please provide a ".md" file or a directory containing ".md" files when dataset.format is "md".'
+ )
+ else:
+ continue
+ continue
+
# Check if file is CSV
if f.endswith('.csv'):
if self.input_args.dataset.format != 'csv':
diff --git a/dingo/data/datasource/sql.py b/dingo/data/datasource/sql.py
index ef8d7d90..7b4c4df2 100644
--- a/dingo/data/datasource/sql.py
+++ b/dingo/data/datasource/sql.py
@@ -1,7 +1,8 @@
from typing import Any, Dict, Generator, Optional
+from urllib.parse import parse_qsl
from sqlalchemy import create_engine, text
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import URL, Engine
from dingo.config import InputArgs
from dingo.data.datasource.base import DataSource
@@ -9,6 +10,14 @@
@DataSource.register()
class SqlDataSource(DataSource):
+ _ENGINE_ARG_TYPES = {
+ "pool_pre_ping": "bool",
+ "pool_recycle": "int",
+ "pool_size": "int",
+ "max_overflow": "int",
+ "pool_timeout": "int",
+ }
+
def __init__(
self,
input_args: InputArgs = None,
@@ -33,37 +42,120 @@ def _get_engine(sql_config) -> Engine:
"must be set when using SQL datasource."
)
- # 构建数据库连接URL
- # SQLite 格式: sqlite:///path/to/database.db
- # 其他数据库格式: dialect+driver://username:password@host:port/database
- if sql_config.dialect.lower() == "sqlite":
- driver_part = f"+{sql_config.driver}" if sql_config.driver else ""
- connection_url = f"{sql_config.dialect}{driver_part}:///{sql_config.database}"
- else:
- # 对于非 SQLite 数据库，需要用户名、密码和主机
- if not sql_config.username or not sql_config.host:
+ dialect = sql_config.dialect.lower()
+ query_args = SqlDataSource._parse_connect_args(sql_config.connect_args)
+
+ connection_url = SqlDataSource._build_connection_url(sql_config, query_args)
+
+ engine_kwargs: Dict[str, Any] = {"pool_pre_ping": True}
+ if dialect in {"mysql", "mariadb"}:
+ engine_kwargs["pool_recycle"] = 1800
+ engine_kwargs.update(SqlDataSource._parse_engine_args(sql_config.engine_args))
+
+ engine = create_engine(connection_url, **engine_kwargs)
+ return engine
+
+ @staticmethod
+ def _build_driver_name(sql_config) -> str:
+ return (
+ f"{sql_config.dialect}+{sql_config.driver}"
+ if sql_config.driver
+ else sql_config.dialect
+ )
+
+ @staticmethod
+ def _parse_connect_args(connect_args: str) -> Dict[str, str]:
+ return SqlDataSource._parse_query_arg_string(connect_args)
+
+ @staticmethod
+ def _parse_query_arg_string(raw_arg_string: str) -> Dict[str, str]:
+ if not raw_arg_string:
+ return {}
+ normalized = raw_arg_string.strip()
+ if normalized.startswith("?"):
+ normalized = normalized[1:]
+ if not normalized:
+ return {}
+ return {
+ key: value
+ for key, value in parse_qsl(normalized, keep_blank_values=False)
+ if key
+ }
+
+ @staticmethod
+ def _parse_bool_value(raw_value: str, key: str) -> bool:
+ normalized = raw_value.strip().lower()
+ if normalized == "true":
+ return True
+ if normalized == "false":
+ return False
+ raise RuntimeError(
+ f"SQL engine arg '{key}' expects 'true' or 'false', got: {raw_value}."
+ )
+
+ @staticmethod
+ def _parse_engine_args(engine_args: str) -> Dict[str, Any]:
+ raw_engine_args = SqlDataSource._parse_query_arg_string(engine_args)
+ if not raw_engine_args:
+ return {}
+
+ parsed_engine_args: Dict[str, Any] = {}
+ for key, raw_value in raw_engine_args.items():
+ expected_type = SqlDataSource._ENGINE_ARG_TYPES.get(key)
+ if expected_type is None:
+ allowed = ", ".join(sorted(SqlDataSource._ENGINE_ARG_TYPES.keys()))
raise RuntimeError(
- f"For {sql_config.dialect}, username and host must be set."
+ f"Unsupported SQL engine arg '{key}'. Allowed keys: {allowed}."
)
- driver_part = f"+{sql_config.driver}" if sql_config.driver else ""
- port_part = f":{sql_config.port}" if sql_config.port else ""
- password_part = f":{sql_config.password}" if sql_config.password else ""
+ if expected_type == "int":
+ try:
+ parsed_engine_args[key] = int(raw_value)
+ except ValueError as exc:
+ raise RuntimeError(
+ f"SQL engine arg '{key}' expects an integer value, got: {raw_value}."
+ ) from exc
+ elif expected_type == "bool":
+ parsed_engine_args[key] = SqlDataSource._parse_bool_value(raw_value, key)
+
+ return parsed_engine_args
+
+ @staticmethod
+ def _parse_port(port: str) -> Optional[int]:
+ if not port:
+ return None
+ try:
+ return int(port)
+ except ValueError as exc:
+ raise RuntimeError("SQL connection parameter 'port' must be an integer.") from exc
- connection_url = (
- f"{sql_config.dialect}{driver_part}://"
- f"{sql_config.username}{password_part}@"
- f"{sql_config.host}{port_part}/{sql_config.database}"
+ @staticmethod
+ def _build_connection_url(sql_config, query_args: Dict[str, str]) -> URL:
+ driver_name = SqlDataSource._build_driver_name(sql_config)
+ query = query_args or None
+
+ if sql_config.dialect.lower() == "sqlite":
+ return URL.create(
+ drivername=driver_name,
+ database=sql_config.database,
+ query=query,
)
- # 添加连接参数（如 ?charset=utf8mb4）
- if sql_config.connect_args:
- # 确保参数以 ? 开头
- args_part = sql_config.connect_args if sql_config.connect_args.startswith('?') else f"?{sql_config.connect_args}"
- connection_url = f"{connection_url}{args_part}"
+ # 对于非 SQLite 数据库，需要用户名、密码和主机
+ if not sql_config.username or not sql_config.host:
+ raise RuntimeError(
+ f"For {sql_config.dialect}, username and host must be set."
+ )
- engine = create_engine(connection_url)
- return engine
+ return URL.create(
+ drivername=driver_name,
+ username=sql_config.username,
+ password=sql_config.password or None,
+ host=sql_config.host,
+ port=SqlDataSource._parse_port(sql_config.port),
+ database=sql_config.database,
+ query=query,
+ )
@staticmethod
def get_source_type() -> str:
diff --git a/dingo/exec/local.py b/dingo/exec/local.py
index ff48aaa5..7321015b 100644
--- a/dingo/exec/local.py
+++ b/dingo/exec/local.py
@@ -115,8 +115,8 @@ def execute(self) -> SummaryModel:
# 统计eval_details，第一层key是字段名组合，第二层value是List[EvalDetail]
# 错误类型从EvalDetail.label中获取
for field_key, eval_detail_list in result_info.eval_details.items():
- if field_key not in self.summary.type_ratio:
- self.summary.type_ratio[field_key] = {}
+ if field_key not in self.summary.type_count:
+ self.summary.type_count[field_key] = {}
# 遍历 List[EvalDetail]，同时收集指标分数和标签
label_set = set()
@@ -131,14 +131,26 @@ def execute(self) -> SummaryModel:
label_set.add(label)
for label in label_set:
- self.summary.type_ratio[field_key].setdefault(label, 0)
- self.summary.type_ratio[field_key][label] += 1
+ self.summary.type_count[field_key].setdefault(label, 0)
+ self.summary.type_count[field_key][label] += 1
+
+ for field_key, eval_detail_list in result_info.token_usage_details.items():
+ for eval_detail in eval_detail_list:
+ if eval_detail.usage is not None and eval_detail.metric:
+ self.summary.add_token_usage(
+ field_key, eval_detail.metric, eval_detail.usage
+ )
if result_info.eval_status:
self.summary.num_bad += 1
else:
self.summary.num_good += 1
self.summary.total += 1
+ # Keep score updated during execution so get_summary()
+ # remains meaningful even if the task stops early.
+ self.summary.score = round(
+ self.summary.num_good / self.summary.total * 100, 2
+ )
self.write_single_data(
self.summary.output_path, self.input_args, result_info
@@ -147,7 +159,7 @@ def execute(self) -> SummaryModel:
self.write_summary(
self.summary.output_path,
self.input_args,
- self.summarize(self.summary),
+ self.summarize(self.summary, refresh_type_ratio=False),
)
log.debug("[Summary]: " + str(self.summary))
@@ -205,6 +217,9 @@ def evaluate_single_data(self, dingo_id: str, eval_fields: dict, eval_type: str,
# Set result_info fields
join_fields = ','.join(eval_fields.values()) if eval_fields else 'default'
+ usage_detail_list = [mr for mr in eval_detail_list if mr.usage is not None]
+ if usage_detail_list:
+ result_info.token_usage_details = {join_fields: usage_detail_list}
# 根据配置决定保存哪些结果
if self.input_args.executor.result_save.all_labels or self.input_args.executor.result_save.merge:
@@ -237,23 +252,26 @@ def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInf
# 第一层是字段名，如果不存在，则直接赋值
else:
existing_item.eval_details[key] = value
+
+ for key, value in new_item.token_usage_details.items():
+ if key in existing_item.token_usage_details:
+ existing_item.token_usage_details[key].extend(value)
+ else:
+ existing_item.token_usage_details[key] = value
else:
existing_list.append(new_item)
return existing_list
- def summarize(self, summary: SummaryModel) -> SummaryModel:
+ def summarize(
+ self, summary: SummaryModel, refresh_type_ratio: bool = True
+ ) -> SummaryModel:
new_summary = copy.deepcopy(summary)
if new_summary.total == 0:
return new_summary
new_summary.score = round(new_summary.num_good / new_summary.total * 100, 2)
-
- # type_ratio是两层结构：第一层是字段名，第二层是具体错误类型
- for field_name in new_summary.type_ratio:
- for eval_details in new_summary.type_ratio[field_name]:
- new_summary.type_ratio[field_name][eval_details] = round(
- new_summary.type_ratio[field_name][eval_details] / new_summary.total, 6
- )
+ if refresh_type_ratio:
+ new_summary = self.refresh_type_ratio(new_summary)
# 计算指标分数的平均值、最小值、最大值、标准差等
new_summary.calculate_metrics_score_averages()
@@ -261,6 +279,20 @@ def summarize(self, summary: SummaryModel) -> SummaryModel:
new_summary.finish_time = time.strftime("%Y%m%d_%H%M%S", time.localtime())
return new_summary
+ def refresh_type_ratio(self, summary: SummaryModel) -> SummaryModel:
+ if summary.total <= 0:
+ summary.type_ratio = {}
+ return summary
+
+ summary.type_ratio = {}
+ for field_name, label_counts in summary.type_count.items():
+ summary.type_ratio[field_name] = {}
+ for label, count in label_counts.items():
+ summary.type_ratio[field_name][label] = round(
+ count / summary.total, 6
+ )
+ return summary
+
@staticmethod
def _json_default(value):
if isinstance(value, Decimal):
@@ -380,6 +412,7 @@ def write_summary(self, path: str, input_args: InputArgs, summary: SummaryModel)
)
def get_summary(self):
+ self.summary = self.refresh_type_ratio(self.summary)
return self.summary
def get_info_list(self, high_quality: bool) -> list:
diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py
index bf93d1c3..60d070c7 100644
--- a/dingo/io/input/required_field.py
+++ b/dingo/io/input/required_field.py
@@ -7,3 +7,9 @@ class RequiredField(Enum):
CONTEXT = "context"
IMAGE = "image"
METADATA = "metadata"
+ TYPE = "type"
+ DT = "dt"
+ SOURCE = "source"
+ SOURCE_DETAILS = "source_details"
+ ANNOTATION = "annotation"
+ DATA_CONTENT = "data_content"
diff --git a/dingo/io/output/eval_detail.py b/dingo/io/output/eval_detail.py
index f2073dca..03592d35 100644
--- a/dingo/io/output/eval_detail.py
+++ b/dingo/io/output/eval_detail.py
@@ -1,6 +1,6 @@
-from typing import Any, Dict, List, Optional
+from typing import Optional
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
class QualityLabel:
@@ -9,6 +9,20 @@ class QualityLabel:
QUALITY_BAD_PREFIX = "QUALITY_BAD_" # Indicates not pass the quality check
+class TokenUsage(BaseModel):
+ """Token usage returned by an LLM provider for one evaluator call."""
+
+ prompt_tokens: Optional[int] = None
+ completion_tokens: Optional[int] = None
+ total_tokens: Optional[int] = None
+ reasoning_tokens: Optional[int] = None
+ cached_tokens: Optional[int] = None
+ model: Optional[str] = None
+ provider: Optional[str] = None
+ calls: int = 1
+ source: str = "provider"
+
+
class EvalDetail(BaseModel):
metric: str
status: bool = False
@@ -16,3 +30,4 @@ class EvalDetail(BaseModel):
score: Optional[float] = None
label: Optional[list[str]] = None
reason: Optional[list] = None
+ usage: Optional[TokenUsage] = None
diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py
index 6a4a9549..6adc4abf 100644
--- a/dingo/io/output/result_info.py
+++ b/dingo/io/output/result_info.py
@@ -5,7 +5,7 @@
from decimal import Decimal
from typing import Any, Dict, List, Optional
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
from dingo.io.output.eval_detail import EvalDetail
@@ -15,6 +15,17 @@ class ResultInfo(BaseModel):
raw_data: Dict = {}
eval_status: bool = False
eval_details: Dict[str, List[EvalDetail]] = {}
+ token_usage_details: Dict[str, List[EvalDetail]] = Field(
+ default_factory=dict,
+ exclude=True,
+ )
+
+ @staticmethod
+ def _eval_detail_to_dict(model_res: EvalDetail) -> Dict[str, Any]:
+ detail = model_res.model_dump()
+ if detail.get('usage') is None:
+ detail.pop('usage', None)
+ return detail
@staticmethod
def _apply_field_filter(output_data: Dict[str, Any], field_list: Optional[List[str]]) -> Dict[str, Any]:
@@ -82,7 +93,7 @@ def to_dict(self, field_list: Optional[List[str]] = None):
'raw_data': self._normalize_value(self.raw_data),
'eval_status': self.eval_status,
'eval_details': {
- k: [model_res.model_dump() for model_res in v]
+ k: [self._eval_detail_to_dict(model_res) for model_res in v]
for k, v in self.eval_details.items()
},
}
@@ -112,7 +123,7 @@ def move_conflict_field(field_name: str):
dingo_result = {
'eval_status': self.eval_status,
'eval_details': {
- k: [model_res.model_dump() for model_res in v]
+ k: [self._eval_detail_to_dict(model_res) for model_res in v]
for k, v in self.eval_details.items()
},
}
diff --git a/dingo/io/output/summary_model.py b/dingo/io/output/summary_model.py
index 3d231df7..cb2a2fe9 100644
--- a/dingo/io/output/summary_model.py
+++ b/dingo/io/output/summary_model.py
@@ -3,6 +3,8 @@
from pydantic import BaseModel, Field
+from dingo.io.output.eval_detail import TokenUsage
+
class SummaryModel(BaseModel):
task_id: str = ''
@@ -16,11 +18,13 @@ class SummaryModel(BaseModel):
num_good: int = 0
num_bad: int = 0
total: int = 0
- type_ratio: Dict[str, Dict[str, int]] = {}
+ type_count: Dict[str, Dict[str, int]] = Field(default_factory=dict)
+ type_ratio: Dict[str, Dict[str, float]] = Field(default_factory=dict)
# 新增：指标分数统计（用于RAG等评估场景）
# 结构：{field_key: {metric_name: {scores, score_average, ...}}}
metrics_score_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict)
+ token_usage_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict)
def add_metric_score(self, field_key: str, metric_name: str, score: float):
"""
@@ -45,6 +49,59 @@ def add_metric_score(self, field_key: str, metric_name: str, score: float):
metric_stats['scores'].append(score)
metric_stats['score_count'] += 1
+ def add_token_usage(self, field_key: str, metric_name: str, usage: TokenUsage):
+ """
+ 添加 LLM token 使用量到统计中
+
+ Args:
+ field_key: 字段名（如 'user_input,response'）
+ metric_name: 指标名称（如 LLMTextQualityV5）
+ usage: 单次 LLM 调用的 token 使用量
+ """
+ usage_stats = self.token_usage_stats.setdefault(field_key, {}).setdefault(
+ metric_name,
+ {
+ 'prompt_tokens': 0,
+ 'completion_tokens': 0,
+ 'total_tokens': 0,
+ 'reasoning_tokens': 0,
+ 'cached_tokens': 0,
+ 'calls': 0,
+ 'records': 0,
+ 'models': {},
+ 'providers': {},
+ 'sources': {},
+ },
+ )
+
+ for token_field in [
+ 'prompt_tokens',
+ 'completion_tokens',
+ 'total_tokens',
+ 'reasoning_tokens',
+ 'cached_tokens',
+ ]:
+ value = getattr(usage, token_field, None)
+ if value is not None:
+ usage_stats[token_field] += int(value)
+
+ calls = int(usage.calls or 1)
+ usage_stats['calls'] += calls
+ usage_stats['records'] += 1
+
+ if usage.model:
+ usage_stats['models'][usage.model] = (
+ usage_stats['models'].get(usage.model, 0) + calls
+ )
+ if usage.provider:
+ usage_stats['providers'][usage.provider] = (
+ usage_stats['providers'].get(usage.provider, 0) + calls
+ )
+ if usage.source:
+ usage_stats['sources'][usage.source] = (
+ usage_stats['sources'].get(usage.source, 0) + calls
+ )
+
def calculate_metrics_score_averages(self):
"""
计算所有字段和指标分数的平均值、最小值、最大值、标准差
@@ -117,6 +174,7 @@ def to_dict(self):
'num_good': self.num_good,
'num_bad': self.num_bad,
'total': self.total,
+ 'type_count': self.type_count,
'type_ratio': self.type_ratio,
}
@@ -131,4 +189,7 @@ def to_dict(self):
for field_key, metrics in self.metrics_score_stats.items()
}
+ if self.token_usage_stats:
+ result['token_usage'] = self.token_usage_stats
+
return result
diff --git a/dingo/model/llm/agent/agent_hallucination.py b/dingo/model/llm/agent/agent_hallucination.py
index fc22ba56..889cd07d 100644
--- a/dingo/model/llm/agent/agent_hallucination.py
+++ b/dingo/model/llm/agent/agent_hallucination.py
@@ -20,6 +20,7 @@
from dingo.io.output.eval_detail import EvalDetail, QualityLabel
from dingo.model import Model
from dingo.model.llm.agent.base_agent import BaseAgent
+from dingo.model.llm.base import llm_response_content
from dingo.utils import log
@@ -339,7 +340,7 @@ def _extract_claims(cls, input_data: Data) -> List[str]:
# Call LLM
messages = [{"role": "user", "content": prompt}]
- response = cls.send_messages(messages)
+ response = llm_response_content(cls.send_messages(messages))
# Parse JSON response
# Handle markdown code blocks
diff --git a/dingo/model/llm/agent/base_agent.py b/dingo/model/llm/agent/base_agent.py
index d3db23d2..8308ccbd 100644
--- a/dingo/model/llm/agent/base_agent.py
+++ b/dingo/model/llm/agent/base_agent.py
@@ -16,6 +16,7 @@
from dingo.io.input import Data, RequiredField
from dingo.io.output.eval_detail import EvalDetail, QualityLabel
from dingo.model.llm.agent.tools import ToolRegistry
+from dingo.model.llm.base import llm_response_content
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils import log
@@ -251,7 +252,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
prompt = step.get('prompt', '')
# Use parent's send_messages method
messages = [{"role": "user", "content": prompt}]
- response = cls.send_messages(messages)
+ response = llm_response_content(cls.send_messages(messages))
results.append(response)
else:
diff --git a/dingo/model/llm/agent/tools/claims_extractor.py b/dingo/model/llm/agent/tools/claims_extractor.py
index f3204b96..d8dc786c 100644
--- a/dingo/model/llm/agent/tools/claims_extractor.py
+++ b/dingo/model/llm/agent/tools/claims_extractor.py
@@ -23,8 +23,10 @@
from pydantic import Field
+from dingo.io.output.eval_detail import TokenUsage
from dingo.model.llm.agent.tools.base_tool import BaseTool, ToolConfig
from dingo.model.llm.agent.tools.tool_registry import tool_register
+from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils import log
@@ -351,10 +353,11 @@ def execute(
# Extract claims from each chunk
all_claims = []
+ token_usage: TokenUsage | None = None
for i, chunk_data in enumerate(chunks):
- log.debug(f"Processing chunk {i+1}/{len(chunks)}")
+ log.debug(f"Processing chunk {i + 1}/{len(chunks)}")
- chunk_claims = cls._extract_claims_from_chunk(
+ chunk_claims, chunk_usage = cls._extract_claims_from_chunk(
client,
chunk_data['text'],
chunk_data['start_pos'],
@@ -362,6 +365,7 @@ def execute(
include_context
)
all_claims.extend(chunk_claims)
+ token_usage = BaseOpenAI._merge_token_usage(token_usage, chunk_usage)
# Deduplicate and merge similar claims
unique_claims = cls._deduplicate_claims(all_claims)
@@ -377,6 +381,8 @@ def execute(
# Build metadata
metadata = cls._build_metadata(unique_claims)
+ if token_usage is not None:
+ metadata['token_usage'] = token_usage.model_dump()
result = {
'success': True,
@@ -460,7 +466,7 @@ def _extract_claims_from_chunk(
start_pos: int,
claim_types: List[str],
include_context: bool
- ) -> List[Dict]:
+ ) -> tuple[List[Dict], TokenUsage | None]:
"""
Extract claims from a single text chunk using LLM.
@@ -472,7 +478,7 @@ def _extract_claims_from_chunk(
include_context: Whether to include context
Returns:
- List of extracted claims
+ Tuple of extracted claims and token usage
"""
# Build user prompt
user_prompt = f"""Extract verifiable claims from the following text.
@@ -496,6 +502,11 @@ def _extract_claims_from_chunk(
temperature=cls.config.temperature,
response_format={"type": "json_object"} # Force JSON output
)
+ token_usage = BaseOpenAI._extract_token_usage(
+ response,
+ model_name=cls.config.model,
+ provider="openai",
+ )
output_text = response.choices[0].message.content
@@ -520,14 +531,14 @@ def _extract_claims_from_chunk(
filtered_claims.append(claim)
- return filtered_claims
+ return filtered_claims, token_usage
except json.JSONDecodeError as e:
log.warning(f"Failed to parse LLM output as JSON: {e}")
- return []
+ return [], token_usage if 'token_usage' in locals() else None
except Exception as e:
log.error(f"LLM call failed: {e}")
- return []
+ return [], None
@classmethod
def _deduplicate_claims(cls, claims: List[Dict]) -> List[Dict]:
diff --git a/dingo/model/llm/base.py b/dingo/model/llm/base.py
index 440193e2..e0fb53cd 100644
--- a/dingo/model/llm/base.py
+++ b/dingo/model/llm/base.py
@@ -1,8 +1,20 @@
-from typing import List
+from typing import List, Optional
from dingo.config.input_args import EvaluatorLLMArgs
from dingo.io import Data
-from dingo.io.output.eval_detail import EvalDetail
+from dingo.io.output.eval_detail import EvalDetail, TokenUsage
+
+
+class LLMCallResult:
+ def __init__(self, content: str, usage: Optional[TokenUsage] = None):
+ self.content = content
+ self.usage = usage
+
+
+def llm_response_content(response) -> str:
+ if isinstance(response, LLMCallResult):
+ return response.content
+ return str(response)
class BaseLLM:
diff --git a/dingo/model/llm/base_litellm.py b/dingo/model/llm/base_litellm.py
index bcf92528..b3ffbe93 100644
--- a/dingo/model/llm/base_litellm.py
+++ b/dingo/model/llm/base_litellm.py
@@ -1,6 +1,7 @@
from typing import List
from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.model.llm.base import LLMCallResult
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils.exception import ExceedMaxTokens
@@ -106,4 +107,11 @@ def send_messages(cls, messages: List) -> str:
)
content = choice.message.content # type: ignore[union-attr]
- return str(content) if content is not None else ""
+ return LLMCallResult(
+ content=str(content) if content is not None else "",
+ usage=cls._extract_token_usage(
+ response,
+ model_name=model_name,
+ provider="litellm",
+ ),
+ )
diff --git a/dingo/model/llm/base_lmdeploy_apiclient.py b/dingo/model/llm/base_lmdeploy_apiclient.py
index 7cc10eb9..2ac5a7b4 100644
--- a/dingo/model/llm/base_lmdeploy_apiclient.py
+++ b/dingo/model/llm/base_lmdeploy_apiclient.py
@@ -23,7 +23,13 @@ class BaseLmdeployApiClient(BaseLLM):
@classmethod
def create_client(cls):
- from lmdeploy.serve.openai.api_client import APIClient
+ try:
+ from lmdeploy.serve.openai.api_client import APIClient
+ except ImportError as exc:
+ raise ImportError(
+ "lmdeploy is required for the lmdeploy API client. "
+ "Install with: pip install dingo-python[lmdeploy] (or pip install lmdeploy)"
+ ) from exc
if not cls.dynamic_config.api_url:
raise ValueError("api_url cannot be empty in llm config.")
diff --git a/dingo/model/llm/base_openai.py b/dingo/model/llm/base_openai.py
index c3911699..a644fa3a 100644
--- a/dingo/model/llm/base_openai.py
+++ b/dingo/model/llm/base_openai.py
@@ -6,8 +6,8 @@
from dingo.config.input_args import EvaluatorLLMArgs
from dingo.io.input import Data, RequiredField
-from dingo.io.output.eval_detail import EvalDetail, QualityLabel
-from dingo.model.llm.base import BaseLLM
+from dingo.io.output.eval_detail import EvalDetail, QualityLabel, TokenUsage
+from dingo.model.llm.base import BaseLLM, LLMCallResult
from dingo.model.response.response_class import ResponseScoreReason
from dingo.utils import log
from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens
@@ -96,7 +96,121 @@ def send_messages(cls, messages: List):
f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}"
)
- return str(completions.choices[0].message.content)
+ return LLMCallResult(
+ content=str(completions.choices[0].message.content),
+ usage=cls._extract_token_usage(
+ completions,
+ model_name=model_name,
+ provider="openai",
+ ),
+ )
+
+ @staticmethod
+ def _usage_value(data, key: str):
+ if data is None:
+ return None
+ if isinstance(data, dict):
+ return data.get(key)
+ return getattr(data, key, None)
+
+ @staticmethod
+ def _coerce_optional_int(value):
+ if value is None:
+ return None
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return None
+
+ @classmethod
+ def _extract_token_usage(
+ cls,
+ completion,
+ model_name: str,
+ provider: str = "openai",
+ ) -> TokenUsage | None:
+ raw_usage = getattr(completion, "usage", None)
+ if raw_usage is None:
+ return None
+
+ if hasattr(raw_usage, "model_dump"):
+ usage_data = raw_usage.model_dump()
+ elif isinstance(raw_usage, dict):
+ usage_data = raw_usage
+ else:
+ usage_data = raw_usage
+
+ completion_details = cls._usage_value(
+ usage_data, "completion_tokens_details"
+ )
+ prompt_details = cls._usage_value(usage_data, "prompt_tokens_details")
+
+ return TokenUsage(
+ prompt_tokens=cls._coerce_optional_int(
+ cls._usage_value(usage_data, "prompt_tokens")
+ ),
+ completion_tokens=cls._coerce_optional_int(
+ cls._usage_value(usage_data, "completion_tokens")
+ ),
+ total_tokens=cls._coerce_optional_int(
+ cls._usage_value(usage_data, "total_tokens")
+ ),
+ reasoning_tokens=cls._coerce_optional_int(
+ cls._usage_value(completion_details, "reasoning_tokens")
+ ),
+ cached_tokens=cls._coerce_optional_int(
+ cls._usage_value(prompt_details, "cached_tokens")
+ ),
+ model=model_name,
+ provider=provider,
+ source="provider",
+ )
+
+ @staticmethod
+ def _copy_token_usage(usage: TokenUsage) -> TokenUsage:
+ if hasattr(usage, "model_copy"):
+ return usage.model_copy(deep=True)
+ return usage.copy(deep=True)
+
+ @classmethod
+ def _merge_token_usage(
+ cls,
+ current: TokenUsage | None,
+ new_usage: TokenUsage | None,
+ ) -> TokenUsage | None:
+ if new_usage is None:
+ return current
+ if current is None:
+ return cls._copy_token_usage(new_usage)
+
+ def _sum_optional(left, right):
+ if left is None and right is None:
+ return None
+ return int(left or 0) + int(right or 0)
+
+ current.prompt_tokens = _sum_optional(
+ current.prompt_tokens, new_usage.prompt_tokens
+ )
+ current.completion_tokens = _sum_optional(
+ current.completion_tokens, new_usage.completion_tokens
+ )
+ current.total_tokens = _sum_optional(
+ current.total_tokens, new_usage.total_tokens
+ )
+ current.reasoning_tokens = _sum_optional(
+ current.reasoning_tokens, new_usage.reasoning_tokens
+ )
+ current.cached_tokens = _sum_optional(
+ current.cached_tokens, new_usage.cached_tokens
+ )
+ current.calls += int(new_usage.calls or 1)
+ if current.model != new_usage.model:
+ current.model = current.model or new_usage.model
+ if current.provider != new_usage.provider:
+ current.provider = current.provider or new_usage.provider
+ if current.source != new_usage.source:
+ current.source = current.source or new_usage.source
+ return current
@classmethod
def validate_numeric_range(cls, value, min_val, max_val, param_name):
@@ -191,10 +305,16 @@ def eval(cls, input_data: Data) -> EvalDetail:
attempts = 0
except_msg = ""
except_name = Exception.__class__.__name__
+ usage: TokenUsage | None = None
while attempts < 3:
try:
response = cls.send_messages(messages)
- res: EvalDetail = cls.process_response(response)
+ if isinstance(response, LLMCallResult):
+ usage = cls._merge_token_usage(usage, response.usage)
+ res: EvalDetail = cls.process_response(response.content)
+ res.usage = usage
+ else:
+ res: EvalDetail = cls.process_response(response)
return res
except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e:
except_msg = str(e)
@@ -216,4 +336,5 @@ def eval(cls, input_data: Data) -> EvalDetail:
res.status = True
res.label = [f"QUALITY_BAD.{except_name}"]
res.reason = [except_msg]
+ res.usage = usage
return res
diff --git a/dingo/model/llm/guobiao/__init__.py b/dingo/model/llm/guobiao/__init__.py
new file mode 100644
index 00000000..82e64589
--- /dev/null
+++ b/dingo/model/llm/guobiao/__init__.py
@@ -0,0 +1 @@
+"""LLM evaluators for TC609 high-quality dataset documentation."""
diff --git a/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py
new file mode 100644
index 00000000..9ab77dcb
--- /dev/null
+++ b/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py
@@ -0,0 +1,46 @@
+from dingo.io.input import RequiredField
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
+
+@Model.llm_register("LLM_TC609_0101_DocBasicInfoCompleteness")
+class LLM_TC609_0101_DocBasicInfoCompleteness(BaseOpenAI):
+ """Evaluate completeness of basic dataset information."""
+
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = {
+ "category": "National Standard LLM Assessment Metrics",
+ "metric_name": "LLM_TC609_0101_DocBasicInfoCompleteness",
+ "description": (
+ "Uses an LLM to assess dataset scale, format, file structure, "
+ "access channel, and technical support in dataset documentation."
+ ),
+ "paper_title": "TC609 high-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "examples": "examples/guobiao/example_doc3.py",
+ }
+ prompt = """
+# 角色
+你是高质量数据集说明文档审核专家。请评估文档的基本信息完整性（0101）。
+
+# 检查事项
+1. 数据集规模：说明样本数量、记录数量、文件数量、存储规模等可核实的规模信息。
+2. 格式规范：说明文件格式、编码、字段结构或解析方式。
+3. 文件结构：说明目录、文件组成及其组织关系。
+4. 访问渠道：说明数据集的获取、下载或访问方式。
+5. 技术支持：说明问题反馈、维护渠道或技术支持方式。
+
+# 判定规则
+逐项寻找明确证据。同义词、近义表达、表格、目录树和代码示例均可作为证据，不要求出现固定标题或关键词。
+只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。
+共5项；明确覆盖至少4项时score为1，否则为0。
+reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。
+
+# 输出格式
+只输出合法JSON，不要输出Markdown或其他文字：
+{"score": 0, "reason": "covered: ...; missing: ..."}
+
+# 待评估文档
+"""
diff --git a/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py
new file mode 100644
index 00000000..2314baba
--- /dev/null
+++ b/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py
@@ -0,0 +1,46 @@
+from dingo.io.input import RequiredField
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
+
+@Model.llm_register("LLM_TC609_0102_DocContentFeatureCompleteness")
+class LLM_TC609_0102_DocContentFeatureCompleteness(BaseOpenAI):
+ """Evaluate completeness of dataset content features."""
+
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = {
+ "category": "National Standard LLM Assessment Metrics",
+ "metric_name": "LLM_TC609_0102_DocContentFeatureCompleteness",
+ "description": (
+ "Uses an LLM to assess modality, distribution, label statistics, "
+ "sample examples, and limitations in dataset documentation."
+ ),
+ "paper_title": "TC609 high-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "examples": "examples/guobiao/example_doc3.py",
+ }
+ prompt = """
+# 角色
+你是高质量数据集说明文档审核专家。请评估文档的内容特征完整性（0102）。
+
+# 检查事项
+1. 模态类型：说明数据包含文本、图像、音频、视频或其他模态。
+2. 数据分布：说明领域、类别、语言、时间或其他维度的数据分布。
+3. 标签统计：说明标签类别及各类别的数量、占比或分布情况。
+4. 样本示例：给出能够代表实际记录结构和内容的数据样例。
+5. 局限性：说明覆盖范围、规模、偏差、适用边界或已知不足。
+
+# 判定规则
+逐项寻找明确证据。同义词、近义表达、表格和示例均可作为证据，不要求出现固定标题或关键词。
+只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。
+共5项；明确覆盖至少4项时score为1，否则为0。
+reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。
+
+# 输出格式
+只输出合法JSON，不要输出Markdown或其他文字：
+{"score": 0, "reason": "covered: ...; missing: ..."}
+
+# 待评估文档
+"""
diff --git a/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py
new file mode 100644
index 00000000..f9127fcf
--- /dev/null
+++ b/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py
@@ -0,0 +1,46 @@
+from dingo.io.input import RequiredField
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
+
+@Model.llm_register("LLM_TC609_0103_DocConstructionProcessCompleteness")
+class LLM_TC609_0103_DocConstructionProcessCompleteness(BaseOpenAI):
+ """Evaluate completeness of the dataset construction process."""
+
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = {
+ "category": "National Standard LLM Assessment Metrics",
+ "metric_name": "LLM_TC609_0103_DocConstructionProcessCompleteness",
+ "description": (
+ "Uses an LLM to assess source, collection, processing, annotation, "
+ "and version control in dataset documentation."
+ ),
+ "paper_title": "TC609 high-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "examples": "examples/guobiao/example_doc3.py",
+ }
+ prompt = """
+# 角色
+你是高质量数据集说明文档审核专家。请评估文档的数据集构建过程完整性（0103）。
+
+# 检查事项
+1. 数据来源：说明数据来自何处，包括人工编写、公开数据、业务系统等来源。
+2. 采集方法：说明数据如何采集、收集、生成或选取。
+3. 加工处理流程：说明清洗、转换、去重、审核、质量控制等处理步骤。
+4. 标注规范：说明标签定义、标注方式、标注人员或一致性要求。
+5. 版本控制：说明版本号、发布日期、变更记录或版本管理方式。
+
+# 判定规则
+逐项寻找明确证据。同义词、近义表达、流程列表和表格均可作为证据，不要求出现固定标题或关键词。
+只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。
+共5项；明确覆盖至少4项时score为1，否则为0。
+reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。
+
+# 输出格式
+只输出合法JSON，不要输出Markdown或其他文字：
+{"score": 0, "reason": "covered: ...; missing: ..."}
+
+# 待评估文档
+"""
diff --git a/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py
new file mode 100644
index 00000000..3f3d2084
--- /dev/null
+++ b/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py
@@ -0,0 +1,46 @@
+from dingo.io.input import RequiredField
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
+
+@Model.llm_register("LLM_TC609_0104_DocApplicationCompleteness")
+class LLM_TC609_0104_DocApplicationCompleteness(BaseOpenAI):
+ """Evaluate completeness of dataset application information."""
+
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = {
+ "category": "National Standard LLM Assessment Metrics",
+ "metric_name": "LLM_TC609_0104_DocApplicationCompleteness",
+ "description": (
+ "Uses an LLM to assess license, target scenarios, evaluation "
+ "method, benchmark results, and typical cases."
+ ),
+ "paper_title": "TC609 high-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "examples": "examples/guobiao/example_doc3.py",
+ }
+ prompt = """
+# 角色
+你是高质量数据集说明文档审核专家。请评估文档的应用说明完整性（0104）。
+
+# 检查事项
+1. 使用许可：说明许可证、授权协议或使用限制。
+2. 目标应用场景：说明适用和不适用的任务、用户或业务场景。
+3. 评估方法：说明如何评测该数据集或如何验证其质量。
+4. 基准结果：给出评测结果、基线结果，或明确说明不提供基准及其原因。
+5. 典型应用案例：说明一个具体使用流程、示例任务或实际应用案例。
+
+# 判定规则
+逐项寻找明确证据。同义词、近义表达、操作步骤和示例均可作为证据，不要求出现固定标题或关键词。
+只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。
+共5项；明确覆盖至少4项时score为1，否则为0。
+reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。
+
+# 输出格式
+只输出合法JSON，不要输出Markdown或其他文字：
+{"score": 0, "reason": "covered: ...; missing: ..."}
+
+# 待评估文档
+"""
diff --git a/dingo/model/llm/llm_custom_metric.py b/dingo/model/llm/llm_custom_metric.py
index 53d2968e..8624932b 100644
--- a/dingo/model/llm/llm_custom_metric.py
+++ b/dingo/model/llm/llm_custom_metric.py
@@ -7,6 +7,7 @@
from dingo.config.input_args import EvaluatorLLMArgs
from dingo.io.input import Data
from dingo.io.output.eval_detail import EvalDetail
+from dingo.model.llm.base import LLMCallResult
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.model.model import Model
from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens
@@ -112,7 +113,14 @@ def send_messages(self, messages: List):
f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}"
)
- return str(completions.choices[0].message.content)
+ return LLMCallResult(
+ content=str(completions.choices[0].message.content),
+ usage=self._extract_token_usage(
+ completions,
+ model_name=model_name,
+ provider="openai",
+ ),
+ )
def _eval_detail_from_response(self, response_json: dict) -> EvalDetail:
custom_metric = self._get_custom_metric()
@@ -189,9 +197,15 @@ def eval(self, input_data: Data) -> EvalDetail:
attempts = 0
except_msg = ""
except_name = Exception.__name__
+ usage = None
while attempts < 3:
try:
response = self.send_messages(messages)
+ if isinstance(response, LLMCallResult):
+ usage = self._merge_token_usage(usage, response.usage)
+ result = self.process_response(response.content)
+ result.usage = usage
+ return result
return self.process_response(response)
except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e:
except_msg = str(e)
@@ -203,9 +217,11 @@ def eval(self, input_data: Data) -> EvalDetail:
except_msg = str(e)
except_name = e.__class__.__name__
- return EvalDetail(
+ result = EvalDetail(
metric=self._get_custom_metric().metric,
status=True,
label=[f"QUALITY_BAD.{except_name}"],
reason=[except_msg],
)
+ result.usage = usage
+ return result
diff --git a/dingo/model/llm/llm_factcheck_public.py b/dingo/model/llm/llm_factcheck_public.py
index 966c44a1..18e0aee8 100644
--- a/dingo/model/llm/llm_factcheck_public.py
+++ b/dingo/model/llm/llm_factcheck_public.py
@@ -4,6 +4,7 @@
from dingo.io.input import Data, RequiredField
from dingo.io.output.eval_detail import EvalDetail, QualityLabel
from dingo.model import Model
+from dingo.model.llm.base import LLMCallResult, llm_response_content
from dingo.model.llm.base_openai import BaseOpenAI
@@ -199,17 +200,25 @@ def eval(cls, input_data: Data) -> EvalDetail:
cls.create_client()
# 1. 提取声明
- claims = cls._extract_claims(input_data.prompt, input_data.content)
+ usage = None
+ claims, claim_usage = cls._extract_claims_with_usage(
+ input_data.prompt, input_data.content
+ )
+ usage = cls._merge_token_usage(usage, claim_usage)
if not claims:
result = EvalDetail(metric=cls.__name__)
result.reason = ["No factual claims found"]
+ result.usage = usage
return result
# 2. 分批验证
all_results = []
for i in range(0, len(claims), cls.batch_size):
batch = claims[i:i + cls.batch_size]
- results = cls._verify_claims(input_data.prompt, input_data.content, batch)
+ results, batch_usage = cls._verify_claims_with_usage(
+ input_data.prompt, input_data.content, batch
+ )
+ usage = cls._merge_token_usage(usage, batch_usage)
all_results.extend(results)
# 3. 计算指标
@@ -218,6 +227,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
# 4. 设置评估结果
result = EvalDetail(metric=cls.__name__)
result.reason = [cls._format_reason(metrics)]
+ result.usage = usage
# 5. 根据分数设置状态
if metrics["factual_ratio"] < cls.threshold:
@@ -237,6 +247,11 @@ def eval(cls, input_data: Data) -> EvalDetail:
@classmethod
def _extract_claims(cls, prompt: str, response: str) -> List[str]:
+ claims, _ = cls._extract_claims_with_usage(prompt, response)
+ return claims
+
+ @classmethod
+ def _extract_claims_with_usage(cls, prompt: str, response: str):
"""提取事实性声明"""
messages = [
{"role": "user", "content": (cls.prompt["CLAIM_LISTING"] +
@@ -245,10 +260,12 @@ def _extract_claims(cls, prompt: str, response: str) -> List[str]:
response=response
)}
]
- result = cls.send_messages(messages)
+ response_result = cls.send_messages(messages)
+ result = llm_response_content(response_result)
try:
claims = cls._parse_json_list(result)
- return [c for c in claims if c.strip()] # 过滤空声明
+ usage = response_result.usage if isinstance(response_result, LLMCallResult) else None
+ return [c for c in claims if c.strip()], usage # 过滤空声明
except Exception as e:
raise ValueError(f"Failed to parse claims: {str(e)}")
@@ -257,6 +274,14 @@ def _verify_claims(cls,
prompt: str,
response: str,
claims: List[str]) -> List[FactCheckResult]:
+ results, _ = cls._verify_claims_with_usage(prompt, response, claims)
+ return results
+
+ @classmethod
+ def _verify_claims_with_usage(cls,
+ prompt: str,
+ response: str,
+ claims: List[str]):
"""验证一批声明"""
messages = [
{"role": "user", "content": (cls.prompt["FACT_CHECKING"] +
@@ -266,9 +291,11 @@ def _verify_claims(cls,
claims=claims
)}
]
- result = cls.send_messages(messages)
+ response_result = cls.send_messages(messages)
+ result = llm_response_content(response_result)
try:
- return cls._parse_check_results(result)
+ usage = response_result.usage if isinstance(response_result, LLMCallResult) else None
+ return cls._parse_check_results(result), usage
except Exception as e:
raise ValueError(f"Failed to parse check results: {str(e)}")
diff --git a/dingo/model/llm/llm_perspective.py b/dingo/model/llm/llm_perspective.py
index 77876cca..a4fbb34a 100644
--- a/dingo/model/llm/llm_perspective.py
+++ b/dingo/model/llm/llm_perspective.py
@@ -20,10 +20,11 @@ class LLMPerspective(BaseLLM):
def create_client(cls):
try:
from googleapiclient import discovery
- except ImportError:
- log.warning(
- "=========== perspective register fail. Please check whether install googleapiclient. ==========="
- )
+ except ImportError as exc:
+ raise ImportError(
+ "google-api-python-client is required for LLMPerspective. "
+ "Install with: pip install google-api-python-client"
+ ) from exc
if cls.client is None:
if not cls.dynamic_config.key:
diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py
new file mode 100644
index 00000000..36f9ec42
--- /dev/null
+++ b/dingo/model/llm/llm_search_result_authority.py
@@ -0,0 +1,322 @@
+"""Rule-based search result authority grader."""
+
+from __future__ import annotations
+import html
+import math
+import re
+import statistics
+from dataclasses import dataclass
+from typing import Any
+
+from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.io.input import Data
+from dingo.io.output.eval_detail import EvalDetail
+from dingo.model import Model
+
+PRESTIGIOUS_VENUE_PATTERNS = (
+ r"^nature(?:$|\s)",
+ r"^npj(?:$|\s)",
+ r"^communications (?:biology|chemistry|earth & environment|materials|physics)$",
+ r"^scientific (?:reports|data)$",
+ r"^science$",
+ r"^science (?:advances|immunology|robotics|signaling|translational medicine)$",
+ r"^cell$",
+ r"^cell (?:reports|metabolism|systems|stem cell|chemical biology|host & microbe|genomics)$",
+ r"^(?:cancer|molecular|developmental) cell$",
+ r"^(?:the )?lancet(?:$|\s)",
+ r"^(?:the )?new england journal of medicine$",
+ r"^nejm(?:$|\s)",
+ r"^jama(?:$|\s)",
+ r"^(?:the )?bmj$",
+ r"^(?:proceedings of the national academy of sciences(?: of the united states of america)?|pnas)$",
+ r"^journal of the american chemical society$",
+ r"^physical review letters$",
+ r"^angewandte chemie(?: international edition)?$",
+ r"^(?:neurips|icml|iclr|cvpr|acl|emnlp|aaai|ijcai|sigir)$",
+ r"^advances in neural information processing systems$",
+)
+
+RECOGNIZED_VENUE_PATTERNS = (
+ r"\bieee\b",
+ r"\bacm\b",
+ r"^plos(?:$|\s)",
+ r"^frontiers in ",
+ r"^bmj(?:$|\s)",
+)
+
+RECOGNIZED_PUBLISHER_HINTS = (
+ "nature portfolio",
+ "springer nature",
+ "springer",
+ "elsevier",
+ "wiley",
+ "taylor & francis",
+ "routledge",
+ "sage publishing",
+ "oxford university press",
+ "cambridge university press",
+ "institute of electrical and electronics engineers",
+ "association for computing machinery",
+ "american chemical society",
+ "royal society of chemistry",
+ "institute of physics",
+ "iop publishing",
+ "american physical society",
+ "american institute of physics",
+ "frontiers media",
+ "public library of science",
+ "bmj publishing",
+ "wolters kluwer",
+ "lippincott williams & wilkins",
+ "american geophysical union",
+ "royal society",
+ "american association for the advancement of science",
+ "de gruyter",
+ "crc press",
+ "chapman & hall",
+ "world scientific",
+ "academic press",
+ "humana press",
+ "emerald publishing",
+)
+
+REPOSITORY_VENUE_HINTS = (
+ "arxiv",
+ "biorxiv",
+ "medrxiv",
+ "chemrxiv",
+ "ssrn",
+ "zenodo",
+ "figshare",
+ "mendeley data",
+ "open science framework",
+ "osf preprints",
+ "research square",
+ "repository",
+)
+
+
+def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
+ return max(low, min(high, value))
+
+
+def _normalize_text(text: Any) -> str:
+ value = html.unescape(str(text or ""))
+ value = re.sub(r"<[^>]+>", " ", value)
+ return " ".join(value.lower().split())
+
+
+def _venue_aliases(venue: str) -> list[str]:
+ aliases = [part.strip(" .,:;-") for part in venue.split("|") if part.strip(" .,:;-")]
+ return aliases or [venue]
+
+
+def _matches_venue_patterns(venue: str, patterns: tuple[str, ...]) -> bool:
+ return any(re.search(pattern, alias) for alias in _venue_aliases(venue) for pattern in patterns)
+
+
+def _extract_publishers(result: dict[str, Any]) -> str:
+ value = result.get("publication_publisher") or result.get("publisher") or ""
+ if isinstance(value, list):
+ return _normalize_text(" | ".join(str(item) for item in value if item))
+ return _normalize_text(value)
+
+
+def _has_venue_issn(result: dict[str, Any]) -> bool:
+ value = result.get("publication_venue_issn") or result.get("issn") or []
+ values = value if isinstance(value, list) else [value]
+ return any(re.fullmatch(r"\d{4}-?\d{3}[\dXx]", str(item or "").strip()) for item in values)
+
+
+def extract_venue(result: dict[str, Any]) -> str:
+ return str(
+ result.get("publication_venue_name_unified")
+ or result.get("publication_venue_name")
+ or result.get("venue")
+ or result.get("source")
+ or ""
+ )
+
+
+def extract_citations(result: dict[str, Any], key: str = "citation_count") -> float:
+ try:
+ return float(result.get(key) or 0)
+ except (TypeError, ValueError):
+ return 0.0
+
+
+def _has_doi_in_locations(locations: Any) -> bool:
+ """Return whether a valid locations list contains a DOI URL or value."""
+ if not isinstance(locations, list):
+ return False
+
+ for location in locations:
+ if isinstance(location, dict):
+ values = (
+ location.get("doi"),
+ location.get("url"),
+ location.get("landing_page_url"),
+ )
+ elif isinstance(location, str):
+ values = (location,)
+ else:
+ continue
+
+ if any("doi.org/" in str(value or "").lower() for value in values):
+ return True
+ return False
+
+
+@dataclass
+class AuthorityGrade:
+ """Structured authority score for one search result."""
+
+ score: float = 0.0
+ citation_score: float = 0.0
+ influential_citation_score: float = 0.0
+ venue_score: float = 0.0
+ doi_score: float = 0.0
+ reason: str = ""
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "score": round(self.score, 5),
+ "citation_score": round(self.citation_score, 5),
+ "influential_citation_score": round(self.influential_citation_score, 5),
+ "venue_score": round(self.venue_score, 5),
+ "doi_score": round(self.doi_score, 5),
+ "reason": self.reason,
+ }
+
+
+@dataclass
+class AuthoritySummary:
+ mean_score: float = 0.0
+ median_score: float = 0.0
+ mean_citation_score: float = 0.0
+ mean_influential_citation_score: float = 0.0
+ mean_venue_score: float = 0.0
+ mean_doi_score: float = 0.0
+ graded_pairs: int = 0
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "authority_mean_score": round(self.mean_score, 5),
+ "authority_median_score": round(self.median_score, 5),
+ "authority_mean_citation_score": round(self.mean_citation_score, 5),
+ "authority_mean_influential_citation_score": round(self.mean_influential_citation_score, 5),
+ "authority_mean_venue_score": round(self.mean_venue_score, 5),
+ "authority_mean_doi_score": round(self.mean_doi_score, 5),
+ "authority_graded_pairs": self.graded_pairs,
+ }
+
+
+@Model.llm_register("LLMSearchResultAuthority")
+class LLMSearchResultAuthority:
+ """Authority scorer based on citation impact, venue, and DOI metadata."""
+
+ dynamic_config = EvaluatorLLMArgs()
+ default_threshold = 0.15
+
+ def grade(self, *, result: dict[str, Any]) -> AuthorityGrade:
+ venue = _normalize_text(extract_venue(result))
+ venue_type = _normalize_text(result.get("publication_venue_type") or "")
+ publishers = _extract_publishers(result)
+ citations = extract_citations(result, "citation_count")
+ influential = extract_citations(result, "influential_citation_count")
+
+ citation_score = _clamp(math.log1p(citations) / math.log1p(500.0))
+ influential_score = _clamp(math.log1p(influential) / math.log1p(50.0))
+
+ venue_score = 0.25
+ reason = "unknown_or_low_signal_venue"
+ is_repository = "repository" in venue_type or "preprint" in venue_type or any(
+ hint in venue for hint in REPOSITORY_VENUE_HINTS
+ )
+ is_academic_book = "book series" in venue_type or "ebook platform" in venue_type or "ebooks" in venue
+ if is_repository:
+ venue_score = 0.45
+ reason = "repository_or_preprint"
+ elif is_academic_book:
+ venue_score = 0.55
+ reason = "academic_book_series"
+ elif _matches_venue_patterns(venue, PRESTIGIOUS_VENUE_PATTERNS):
+ venue_score = 0.85
+ reason = "prestigious_venue_family"
+ elif _matches_venue_patterns(venue, RECOGNIZED_VENUE_PATTERNS) or any(
+ hint in publishers for hint in RECOGNIZED_PUBLISHER_HINTS
+ ):
+ venue_score = 0.75
+ reason = "recognized_scholarly_publisher_or_venue"
+ elif "journal" in venue_type or "conference" in venue_type or _has_venue_issn(result):
+ venue_score = 0.65
+ reason = "structured_journal_or_conference"
+ elif venue:
+ venue_score = 0.40
+ reason = "named_venue"
+
+ doi_score = 1.0 if result.get("doi") or _has_doi_in_locations(result.get("locations")) else 0.0
+ score = (
+ 0.45 * citation_score
+ + 0.20 * influential_score
+ + 0.25 * venue_score
+ + 0.10 * doi_score
+ )
+ return AuthorityGrade(
+ score=_clamp(score),
+ citation_score=citation_score,
+ influential_citation_score=influential_score,
+ venue_score=venue_score,
+ doi_score=doi_score,
+ reason=reason,
+ )
+
+ @classmethod
+ def _config_value(cls, name: str, default: Any = None) -> Any:
+ return getattr(cls.dynamic_config, name, default)
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ """Executor entry point: evaluate one flattened search result row."""
+ result = getattr(input_data, "search_result", None)
+ if not isinstance(result, dict):
+ result = input_data.to_dict()
+
+ grade = cls().grade(result=result)
+ threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold)
+
+ labels: list[str] = []
+ if grade.score < threshold:
+ labels.append("Authority.Error_Authority_Low")
+ if grade.citation_score <= 0.0:
+ labels.append("Authority.Error_Citation_Miss")
+ if grade.venue_score <= 0.25:
+ labels.append("Authority.Error_Venue_Low_Signal")
+ if grade.doi_score <= 0.0:
+ labels.append("Authority.Error_DOI_Miss")
+
+ status = bool(labels)
+ if not labels:
+ labels = ["QUALITY_GOOD"]
+
+ return EvalDetail(
+ metric=cls.__name__,
+ status=status,
+ score=round(grade.score, 5),
+ label=labels,
+ reason=[grade.to_dict()],
+ )
+
+
+def aggregate_grades(grades: list[AuthorityGrade]) -> AuthoritySummary:
+ if not grades:
+ return AuthoritySummary()
+ return AuthoritySummary(
+ mean_score=statistics.mean(g.score for g in grades),
+ median_score=statistics.median(g.score for g in grades),
+ mean_citation_score=statistics.mean(g.citation_score for g in grades),
+ mean_influential_citation_score=statistics.mean(g.influential_citation_score for g in grades),
+ mean_venue_score=statistics.mean(g.venue_score for g in grades),
+ mean_doi_score=statistics.mean(g.doi_score for g in grades),
+ graded_pairs=len(grades),
+ )
diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py
new file mode 100644
index 00000000..fcf25f2c
--- /dev/null
+++ b/dingo/model/llm/llm_search_result_effectiveness.py
@@ -0,0 +1,800 @@
+"""Search result effectiveness grader.
+
+This grader scores whether a returned search result has usable bibliographic
+content for a user to judge and consume it. Field-presence checks are
+deterministic. Readability and corruption checks can be delegated to an LLM
+judge to avoid over-penalizing normal academic formulas, units, and symbols.
+
+It intentionally does not judge topical relevance; use
+``LLMSearchResultRelevance`` for that.
+"""
+
+from __future__ import annotations
+import json
+import logging
+import re
+import statistics
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.io.input import Data
+from dingo.io.output.eval_detail import EvalDetail, TokenUsage
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
+logger = logging.getLogger(__name__)
+
+
+# Require a tag name immediately after ``<`` (or ```` as HTML markup.
+HTML_TAG_PATTERN = r"]*?)?\s*/?>"
+
+RULE_SPECIAL_CHARACTER_PATTERNS = (
+ r"u200e",
+ r"÷|\? :",
+ r"[锟解枴閿熻В鏋碷�]|\{\/U\}",
+ r"U\+26[0-F][0-D]|U\+273[3-4]|U\+1F[3-6][0-4][0-F]|U\+1F6[8-F][0-F]",
+ r"<\|.*?\|>",
+ HTML_TAG_PATTERN,
+)
+RULE_INVISIBLE_CHAR_PATTERN = r"[\u0080-\u009F\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]"
+RULE_ABNORMAL_CHAR_THRESHOLD = 0.01
+MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}"
+UTF8_LATIN1_SEQUENCE_PATTERN = re.compile(r"[\u00C2\u00C3\u00D0\u00D1][\u0080-\u00BF]")
+C1_CONTROL_PATTERN = re.compile(r"[\u0080-\u009F]")
+UNICODE_REPLACEMENT_CHARACTER = "\ufffd"
+
+
+LLM_FIELD_QUALITY_SYSTEM_PROMPT = """You are a strict but practical data quality evaluator for
+academic search result metadata.
+
+Judge whether each supplied metadata field is readable and clean enough to show to users.
+Focus on real text-quality problems:
+- missing or empty field
+- invisible/control characters
+- mojibake or garbled encoding, such as replacement characters, unreadable CJK mojibake,
+ or UTF-8 text decoded as Latin-1 with repeated sequences like Ð... or Ñ...
+- raw HTML/XML markup leaked into visible text, such as ...
+- suspicious special-character noise that materially hurts readability
+
+Do NOT penalize normal academic content:
+- mathematical formulas, LaTeX, chemical symbols, units, Greek letters
+- punctuation, pipes used as separators, parentheses, slashes, hyphens
+- mixed Chinese/English titles, journal names, abbreviations, DOI-like text
+
+Return compact JSON only. Do not use markdown. Keep each reason within 12 words
+and do not use double quotes inside reasons.
+Schema:
+{
+ "fields": {
+ "title": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."},
+ "abstract": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."},
+ "keywords": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."},
+ "venue": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."},
+ "author": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}
+ },
+ "overall_issues": ["..."],
+ "reason": "short overall reason"
+}
+
+Use issue names from:
+- missing_field
+- invisible_char
+- mojibake
+- html_tag
+- unreadable_text
+- special_char_noise
+- none
+
+Scoring guidance:
+- 1.0: clean, readable field; normal formulas and units are allowed.
+- 0.7: mostly readable with minor display artifacts.
+- 0.4: readable but contains visible markup or notable noise requiring cleanup.
+- 0.1: unreadable garbled text, heavy mojibake, or severe invisible/control-character corruption.
+- 0.0: missing/empty field.
+"""
+
+
+def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
+ return max(low, min(high, value))
+
+
+def _presence_quality(value: Any) -> float:
+ """Score field presence without using content length as a quality proxy."""
+ return 1.0 if str(value or "").strip() else 0.0
+
+
+def _suspicious_latin1_mojibake_count(text: str) -> int:
+ return len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(text)) + len(C1_CONTROL_PATTERN.findall(text))
+
+
+def _looks_like_utf8_latin1_mojibake(text: str) -> bool:
+ """Detect UTF-8 text accidentally decoded as Latin-1.
+
+ Literal characters such as ``Ð`` and ``Ñ`` can be valid text, so they are
+ not sufficient evidence by themselves. A value is treated as suspicious
+ when it contains repeated UTF-8/Latin-1 byte-shaped sequences or C1 control
+ characters and a reversible Latin-1-to-UTF-8 repair removes that evidence.
+ """
+ value = str(text or "")
+ if not value:
+ return False
+
+ suspicious_before = _suspicious_latin1_mojibake_count(value)
+ if suspicious_before == 0:
+ return False
+
+ c1_count = len(C1_CONTROL_PATTERN.findall(value))
+ repeated_sequences = len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(value)) >= 2
+ if not repeated_sequences and c1_count / max(1, len(value)) < RULE_ABNORMAL_CHAR_THRESHOLD:
+ return False
+
+ try:
+ repaired = value.encode("latin-1").decode("utf-8")
+ except (UnicodeEncodeError, UnicodeDecodeError):
+ # Mixed-language metadata may contain legitimate non-Latin-1 text next
+ # to a corrupted fragment. Repeated byte-shaped pairs plus C1 controls
+ # are enough to send that field to the LLM judge for confirmation.
+ return repeated_sequences and c1_count > 0
+
+ suspicious_after = _suspicious_latin1_mojibake_count(repaired)
+ return repaired != value and suspicious_after < suspicious_before
+
+
+def _has_mojibake_evidence(text: str) -> bool:
+ value = str(text or "")
+ return (
+ UNICODE_REPLACEMENT_CHARACTER in value
+ or bool(re.search(MOJIBAKE_EVIDENCE_PATTERN, value))
+ or _looks_like_utf8_latin1_mojibake(value)
+ )
+
+
+def _rule_abnormal_char_issues(text: str) -> list[str]:
+ value = str(text or "")
+ if not value:
+ return []
+
+ issues: list[str] = []
+ special_matches: list[str] = []
+ for pattern in RULE_SPECIAL_CHARACTER_PATTERNS:
+ special_matches.extend(re.findall(pattern, value))
+ has_html_tag = bool(re.search(HTML_TAG_PATTERN, value))
+ if has_html_tag or len(special_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD:
+ issues.append("RuleSpecialCharacter")
+
+ has_mojibake = _has_mojibake_evidence(value)
+ if has_mojibake:
+ issues.append("RuleMojibake")
+
+ invisible_matches = re.findall(RULE_INVISIBLE_CHAR_PATTERN, value)
+ if not has_mojibake and len(invisible_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD:
+ issues.append("RuleInvisibleChar")
+ return issues
+
+
+def _has_confirmed_llm_issue(issues: list[str] | None) -> bool:
+ if not issues:
+ return False
+ ignored = {"none", "missing_field"}
+ return any(str(issue).split(":")[-1].strip().lower() not in ignored for issue in issues)
+
+
+def _filter_llm_field_issues(field: str, value: str, issues: list[str]) -> list[str]:
+ """Keep only LLM issues supported by field-level evidence."""
+ text = str(value or "")
+ filtered: list[str] = []
+ for issue in issues:
+ issue_type = str(issue).split(":")[-1].strip().lower()
+ keep = False
+ if issue_type == "html_tag":
+ keep = bool(re.search(HTML_TAG_PATTERN, text))
+ elif issue_type == "invisible_char":
+ keep = bool(re.search(RULE_INVISIBLE_CHAR_PATTERN, text))
+ elif issue_type in {"mojibake", "unreadable_text"}:
+ keep = _has_mojibake_evidence(text)
+ elif issue_type == "special_char_noise":
+ keep = bool(_rule_abnormal_char_issues(text))
+ else:
+ keep = True
+
+ if keep and issue not in filtered:
+ filtered.append(issue)
+ return filtered
+
+
+def _strip_json_fence(text: str) -> str:
+ value = (text or "").strip()
+ if value.startswith("```json"):
+ value = value[7:].strip()
+ elif value.startswith("```"):
+ value = value[3:].strip()
+ if value.endswith("```"):
+ value = value[:-3].strip()
+ return value
+
+
+def _extract_json_object(text: str) -> str:
+ value = _strip_json_fence(text)
+ start = value.find("{")
+ end = value.rfind("}")
+ if start >= 0 and end > start:
+ return value[start:end + 1]
+ return value
+
+
+def _safe_float(value: Any, default: float = 1.0) -> float:
+ try:
+ return _clamp(float(value))
+ except (TypeError, ValueError):
+ return default
+
+
+def _normalize_issues(value: Any) -> list[str]:
+ if not value:
+ return []
+ if isinstance(value, str):
+ return [] if value.lower() == "none" else [value]
+ if isinstance(value, list):
+ issues = []
+ for item in value:
+ item_text = str(item).strip()
+ if item_text and item_text.lower() != "none":
+ issues.append(item_text)
+ return issues
+ return [str(value)]
+
+
+EFFECTIVENESS_LABEL_MAP = {
+ "missing_title": "Effectiveness.Error_Title_Miss",
+ "missing_abstract": "Effectiveness.Error_Abstract_Miss",
+ "missing_keywords": "Effectiveness.Error_Keywords_Miss",
+ "missing_author": "Effectiveness.Error_Author_Miss",
+ "html_tag": "Effectiveness.Error_HTML_Tag",
+ "mojibake": "Effectiveness.Error_Mojibake",
+ "invisible_char": "Effectiveness.Error_Invisible_Char",
+ "unreadable_text": "Effectiveness.Error_Unreadable_Text",
+ "special_char_noise": "Effectiveness.Error_Special_Char_Noise",
+ "llm_quality_parse_error": "Effectiveness.Error_LLM_Quality_Parse",
+ "RuleSpecialCharacter": "Effectiveness.Error_Rule_Special_Character",
+ "RuleInvisibleChar": "Effectiveness.Error_Rule_Invisible_Char",
+ "RuleMojibake": "Effectiveness.Error_Mojibake",
+}
+
+
+def _issue_to_label(issue: str) -> str | None:
+ issue_text = str(issue or "").strip()
+ if not issue_text:
+ return None
+ issue_type = issue_text.split(":")[-1]
+ return EFFECTIVENESS_LABEL_MAP.get(issue_type)
+
+
+def _issues_to_labels(issues: list[str] | None) -> list[str]:
+ """Map issues to final business labels.
+
+ RuleSpecialCharacter and RuleInvisibleChar are candidate triggers. When LLM
+ confirms a concrete issue such as title:html_tag, keep the concrete
+ business label and suppress the intermediate rule label to avoid duplicate
+ output files for the same problem.
+ """
+ labels: list[str] = []
+ normalized_issues = [str(issue or "").strip() for issue in (issues or []) if str(issue or "").strip()]
+ has_confirmed_quality_issue = any(
+ ":" in issue and _issue_to_label(issue) is not None
+ for issue in normalized_issues
+ )
+
+ for issue in normalized_issues:
+ issue_type = issue.split(":")[-1]
+ if has_confirmed_quality_issue and issue_type in {
+ "RuleSpecialCharacter",
+ "RuleInvisibleChar",
+ "RuleMojibake",
+ }:
+ continue
+ label = _issue_to_label(issue)
+ if label and label not in labels:
+ labels.append(label)
+ return labels
+
+
+def _truncate_for_llm(value: str, max_chars: int = 1000) -> str:
+ text = str(value or "")
+ if len(text) <= max_chars:
+ return text
+ return text[:max_chars] + "...[truncated]"
+
+
+def extract_keywords(result: dict[str, Any]) -> list[str]:
+ value = result.get("keywords") or result.get("keyword") or result.get("concepts") or []
+ if isinstance(value, str):
+ return [item.strip() for item in re.split(r"[,;|]", value) if item.strip()]
+ if isinstance(value, list):
+ keywords: list[str] = []
+ for item in value:
+ if isinstance(item, dict):
+ name = item.get("name") or item.get("display_name") or item.get("keyword")
+ if name:
+ keywords.append(str(name))
+ elif item not in (None, ""):
+ keywords.append(str(item))
+ return keywords
+ return []
+
+
+def extract_venue(result: dict[str, Any]) -> str:
+ return str(
+ result.get("publication_venue_name_unified")
+ or result.get("publication_venue_name")
+ or result.get("venue")
+ or result.get("source")
+ or ""
+ )
+
+
+def extract_authors(result: dict[str, Any]) -> list[str]:
+ """Extract author names from common search API response shapes."""
+ value = result.get("author") or result.get("authors") or []
+ if isinstance(value, str):
+ return [item.strip() for item in re.split(r"[;|]", value) if item.strip()]
+ if isinstance(value, dict):
+ value = [value]
+ if not isinstance(value, list):
+ return []
+
+ authors: list[str] = []
+ for item in value:
+ if isinstance(item, dict):
+ name = item.get("name") or item.get("display_name") or item.get("author_name")
+ if name:
+ authors.append(str(name).strip())
+ elif item not in (None, ""):
+ authors.append(str(item).strip())
+ return [author for author in authors if author]
+
+
+@dataclass
+class LLMFieldQuality:
+ """LLM readability and corruption judgment for one search result."""
+
+ title_score: float = 1.0
+ abstract_score: float = 1.0
+ keywords_score: float = 1.0
+ venue_score: float = 1.0
+ author_score: float = 1.0
+ issues: list[str] | None = None
+ reason: str = ""
+ error: str = ""
+ usage: TokenUsage | None = None
+
+ def field_score(self, field: str) -> float:
+ return {
+ "title": self.title_score,
+ "abstract": self.abstract_score,
+ "keywords": self.keywords_score,
+ "venue": self.venue_score,
+ "author": self.author_score,
+ }.get(field, 1.0)
+
+
+def _parse_llm_field_quality_response(text: str) -> LLMFieldQuality:
+ candidate = _extract_json_object(text)
+ try:
+ data = json.loads(candidate)
+ except json.JSONDecodeError as e:
+ return LLMFieldQuality(error=f"JSON parse failed: {e}. Text: {text[:200]}")
+
+ fields = data.get("fields") if isinstance(data, dict) else {}
+ if not isinstance(fields, dict):
+ return LLMFieldQuality(error=f"Missing fields object. Text: {candidate[:200]}")
+
+ issues: list[str] = []
+ scores: dict[str, float] = {}
+ reasons: list[str] = []
+ for field in ("title", "abstract", "keywords", "venue", "author"):
+ field_data = fields.get(field) or {}
+ if not isinstance(field_data, dict):
+ field_data = {}
+ scores[field] = _safe_float(field_data.get("score"), default=1.0)
+ for issue in _normalize_issues(field_data.get("issues")):
+ issues.append(f"{field}:{issue}")
+ reason = str(field_data.get("reason") or "").strip()
+ if reason:
+ reasons.append(f"{field}: {reason}")
+
+ for issue in _normalize_issues(data.get("overall_issues")):
+ issues.append(issue)
+
+ return LLMFieldQuality(
+ title_score=scores["title"],
+ abstract_score=scores["abstract"],
+ keywords_score=scores["keywords"],
+ venue_score=scores["venue"],
+ author_score=scores["author"],
+ issues=issues,
+ reason=str(data.get("reason") or "; ".join(reasons))[:500],
+ )
+
+
+@dataclass
+class EffectivenessGrade:
+ """Structured score for one search result."""
+
+ score: float = 0.0
+ title_score: float = 0.0
+ abstract_score: float = 0.0
+ keywords_score: float = 0.0
+ venue_score: float = 0.0
+ author_score: float = 0.0
+ issues: list[str] | None = None
+ llm_quality_reason: str = ""
+ llm_quality_error: str = ""
+ usage: TokenUsage | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "score": round(self.score, 5),
+ "title_score": round(self.title_score, 5),
+ "abstract_score": round(self.abstract_score, 5),
+ "keywords_score": round(self.keywords_score, 5),
+ "venue_score": round(self.venue_score, 5),
+ "author_score": round(self.author_score, 5),
+ "issues": self.issues or [],
+ "llm_quality_reason": self.llm_quality_reason,
+ "llm_quality_error": self.llm_quality_error,
+ }
+
+
+@dataclass
+class EffectivenessSummary:
+ mean_score: float = 0.0
+ median_score: float = 0.0
+ mean_title_score: float = 0.0
+ mean_abstract_score: float = 0.0
+ mean_keywords_score: float = 0.0
+ mean_venue_score: float = 0.0
+ mean_author_score: float = 0.0
+ graded_pairs: int = 0
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "effectiveness_mean_score": round(self.mean_score, 5),
+ "effectiveness_median_score": round(self.median_score, 5),
+ "effectiveness_mean_title_score": round(self.mean_title_score, 5),
+ "effectiveness_mean_abstract_score": round(self.mean_abstract_score, 5),
+ "effectiveness_mean_keywords_score": round(self.mean_keywords_score, 5),
+ "effectiveness_mean_venue_score": round(self.mean_venue_score, 5),
+ "effectiveness_mean_author_score": round(self.mean_author_score, 5),
+ "effectiveness_graded_pairs": self.graded_pairs,
+ }
+
+
+@Model.llm_register("LLMSearchResultEffectiveness")
+class LLMSearchResultEffectiveness:
+ """Effectiveness scorer for title, abstract, keywords, and authors.
+
+ Venue text is still scanned for corruption, but venue presence and quality
+ belong to the authority metric and do not affect the effectiveness score.
+ """
+
+ dynamic_config = EvaluatorLLMArgs()
+ default_threshold = 0.15
+
+ def __init__(
+ self,
+ *,
+ model: str | None = None,
+ api_key: str | None = None,
+ api_url: str | None = None,
+ max_tokens: int = 512,
+ temperature: float = 0.0,
+ timeout: float | None = None,
+ enable_llm_quality: bool = False,
+ ):
+ self.model = model or "gpt-4o"
+ self.api_key = api_key
+ self.api_url = api_url
+ self.max_tokens = max_tokens
+ self.temperature = temperature
+ self.timeout = timeout
+ self.enable_llm_quality = enable_llm_quality
+ self._client = None
+
+ def _get_client(self):
+ if self._client is None:
+ from openai import OpenAI
+
+ kwargs: dict[str, Any] = {}
+ if self.api_key:
+ kwargs["api_key"] = self.api_key
+ if self.api_url:
+ kwargs["base_url"] = self.api_url
+ self._client = OpenAI(**kwargs)
+ return self._client
+
+ def _build_llm_quality_user_message(
+ self,
+ *,
+ title: str,
+ abstract: str,
+ keywords: list[str],
+ venue: str,
+ authors: list[str],
+ candidate_fields: set[str] | None = None,
+ ) -> str:
+ all_fields = {
+ "title": title,
+ "abstract": abstract,
+ "keywords": " | ".join(keywords),
+ "venue": venue,
+ "author": " | ".join(authors),
+ }
+ selected = candidate_fields or set(all_fields)
+ payload = {
+ field: _truncate_for_llm(value)
+ for field, value in all_fields.items()
+ if field in selected
+ }
+ return (
+ "Evaluate only the supplied fields for readability and corruption. "
+ "Omitted fields should not be judged. Return compact JSON.\n\n"
+ f"{json.dumps(payload, ensure_ascii=False, indent=2)}"
+ )
+
+ def _judge_llm_field_quality(
+ self,
+ *,
+ title: str,
+ abstract: str,
+ keywords: list[str],
+ venue: str,
+ authors: list[str],
+ candidate_fields: set[str] | None = None,
+ ) -> LLMFieldQuality:
+ if not self.enable_llm_quality:
+ return LLMFieldQuality()
+ client = self._get_client()
+ last_result = LLMFieldQuality(error="LLM field quality judgment failed")
+ usage: TokenUsage | None = None
+ for attempt in range(3):
+ try:
+ completion = client.chat.completions.create(
+ model=self.model,
+ messages=[
+ {"role": "system", "content": LLM_FIELD_QUALITY_SYSTEM_PROMPT},
+ {
+ "role": "user",
+ "content": self._build_llm_quality_user_message(
+ title=title,
+ abstract=abstract,
+ keywords=keywords,
+ venue=venue,
+ authors=authors,
+ candidate_fields=candidate_fields,
+ ),
+ },
+ ],
+ temperature=self.temperature,
+ max_tokens=self.max_tokens,
+ timeout=self.timeout,
+ )
+ usage = BaseOpenAI._merge_token_usage(
+ usage,
+ BaseOpenAI._extract_token_usage(
+ completion,
+ model_name=self.model,
+ provider="openai",
+ ),
+ )
+ response_text = completion.choices[0].message.content or ""
+ last_result = _parse_llm_field_quality_response(response_text)
+ last_result.usage = usage
+ if not last_result.error:
+ return last_result
+ error: Exception | str = last_result.error
+ except Exception as exc:
+ error = exc
+ last_result = LLMFieldQuality(error=str(exc))
+ last_result.usage = usage
+
+ logger.warning(
+ "LLM field quality attempt %s/3 failed for title=%r: %s",
+ attempt + 1,
+ title,
+ error,
+ )
+ if attempt < 2:
+ time.sleep(attempt + 1)
+ return last_result
+
+ def grade(
+ self,
+ *,
+ title: str = "",
+ abstract: str = "",
+ keywords: list[str] | str | None = None,
+ venue: str = "",
+ authors: list[str] | str | None = None,
+ result: dict[str, Any] | None = None,
+ ) -> EffectivenessGrade:
+ if result is not None:
+ title = str(result.get("title") or result.get("display_name") or title or "")
+ abstract = str(result.get("abstract") or abstract or "")
+ keywords = extract_keywords(result) if keywords is None else keywords
+ venue = extract_venue(result) or venue
+ authors = extract_authors(result) if authors is None else authors
+
+ keyword_items = (
+ [item.strip() for item in re.split(r"[,;|]", keywords) if item.strip()]
+ if isinstance(keywords, str)
+ else [str(item).strip() for item in (keywords or []) if str(item).strip()]
+ )
+ author_items = (
+ [item.strip() for item in re.split(r"[;|]", authors) if item.strip()]
+ if isinstance(authors, str)
+ else [str(item).strip() for item in (authors or []) if str(item).strip()]
+ )
+
+ title_score = _presence_quality(title)
+ abstract_score = _presence_quality(abstract)
+ keywords_score = 1.0 if keyword_items else 0.0
+ venue_score = _presence_quality(venue)
+ author_score = 1.0 if author_items else 0.0
+
+ issues: list[str] = []
+ if not str(title or "").strip():
+ issues.append("missing_title")
+ if not str(abstract or "").strip():
+ issues.append("missing_abstract")
+ if not keyword_items:
+ issues.append("missing_keywords")
+ if not author_items:
+ issues.append("missing_author")
+
+ field_values = {
+ "title": str(title or ""),
+ "abstract": str(abstract or ""),
+ "keywords": " | ".join(keyword_items),
+ "venue": str(venue or ""),
+ "author": " | ".join(author_items),
+ }
+ rule_candidate_issues = {
+ field: _rule_abnormal_char_issues(value)
+ for field, value in field_values.items()
+ if value
+ }
+ rule_candidate_issues = {
+ field: field_issues
+ for field, field_issues in rule_candidate_issues.items()
+ if field_issues
+ }
+
+ llm_quality = LLMFieldQuality()
+ if rule_candidate_issues and self.enable_llm_quality:
+ llm_quality = self._judge_llm_field_quality(
+ title=str(title or ""),
+ abstract=str(abstract or ""),
+ keywords=keyword_items,
+ venue=str(venue or ""),
+ authors=author_items,
+ candidate_fields=set(rule_candidate_issues),
+ )
+
+ def apply_confirmed_field_issue(field: str, score: float) -> float:
+ field_rule_issues = rule_candidate_issues.get(field) or []
+ if not field_rule_issues:
+ return score
+
+ if not self.enable_llm_quality:
+ issues.extend(field_rule_issues)
+ return min(score, 0.1)
+
+ if llm_quality.error:
+ return score
+
+ field_llm_issues = [
+ issue for issue in (llm_quality.issues or [])
+ if str(issue).startswith(f"{field}:")
+ ]
+ field_llm_issues = _filter_llm_field_issues(
+ field,
+ field_values.get(field, ""),
+ field_llm_issues,
+ )
+ llm_field_score = llm_quality.field_score(field)
+ if llm_field_score < 1.0 or _has_confirmed_llm_issue(field_llm_issues):
+ issues.extend(field_rule_issues)
+ issues.extend(field_llm_issues)
+ return min(score, llm_field_score)
+ return score
+
+ title_score = apply_confirmed_field_issue("title", title_score)
+ abstract_score = apply_confirmed_field_issue("abstract", abstract_score)
+ keywords_score = apply_confirmed_field_issue("keywords", keywords_score)
+ venue_score = apply_confirmed_field_issue("venue", venue_score)
+ author_score = apply_confirmed_field_issue("author", author_score)
+
+ if rule_candidate_issues and self.enable_llm_quality and llm_quality.error:
+ issues.append("llm_quality_parse_error")
+
+ score = (
+ 0.30 * title_score
+ + 0.50 * abstract_score
+ + 0.10 * keywords_score
+ + 0.10 * author_score
+ )
+ return EffectivenessGrade(
+ score=_clamp(score),
+ title_score=_clamp(title_score),
+ abstract_score=_clamp(abstract_score),
+ keywords_score=_clamp(keywords_score),
+ venue_score=_clamp(venue_score),
+ author_score=_clamp(author_score),
+ issues=issues,
+ llm_quality_reason=llm_quality.reason,
+ llm_quality_error=llm_quality.error,
+ usage=llm_quality.usage,
+ )
+
+ @classmethod
+ def _config_value(cls, name: str, default: Any = None) -> Any:
+ return getattr(cls.dynamic_config, name, default)
+
+ @classmethod
+ def _build_from_config(cls) -> "LLMSearchResultEffectiveness":
+ return cls(
+ model=cls.dynamic_config.model,
+ api_key=cls.dynamic_config.key,
+ api_url=cls.dynamic_config.api_url,
+ max_tokens=int(cls._config_value("max_tokens", 512) or 512),
+ temperature=float(cls._config_value("temperature", 0.0) or 0.0),
+ timeout=cls._config_value("timeout", None),
+ enable_llm_quality=bool(cls._config_value("enable_llm_quality", False)),
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ """Executor entry point: evaluate one flattened search result row."""
+ result = getattr(input_data, "search_result", None)
+ if not isinstance(result, dict):
+ result = input_data.to_dict()
+
+ grader = cls._build_from_config()
+ grade = grader.grade(result=result)
+ threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold)
+
+ labels = _issues_to_labels(grade.issues)
+
+ if grade.score < threshold and "Effectiveness.Error_Effectiveness_Low" not in labels:
+ labels.append("Effectiveness.Error_Effectiveness_Low")
+
+ status = bool(labels)
+ if not labels:
+ labels = ["QUALITY_GOOD"]
+
+ return EvalDetail(
+ metric=cls.__name__,
+ status=status,
+ score=round(grade.score, 5),
+ label=labels,
+ reason=[grade.to_dict()],
+ usage=grade.usage,
+ )
+
+
+def aggregate_grades(grades: list[EffectivenessGrade]) -> EffectivenessSummary:
+ if not grades:
+ return EffectivenessSummary()
+ return EffectivenessSummary(
+ mean_score=statistics.mean(g.score for g in grades),
+ median_score=statistics.median(g.score for g in grades),
+ mean_title_score=statistics.mean(g.title_score for g in grades),
+ mean_abstract_score=statistics.mean(g.abstract_score for g in grades),
+ mean_keywords_score=statistics.mean(g.keywords_score for g in grades),
+ mean_venue_score=statistics.mean(g.venue_score for g in grades),
+ mean_author_score=statistics.mean(g.author_score for g in grades),
+ graded_pairs=len(grades),
+ )
diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py
index d3f1a62b..a15c2d0a 100644
--- a/dingo/model/llm/llm_search_result_relevance.py
+++ b/dingo/model/llm/llm_search_result_relevance.py
@@ -17,25 +17,41 @@
from __future__ import annotations
import json
import logging
+import re
import statistics
+import time
from dataclasses import dataclass
from typing import Any
+from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.io.input import Data
+from dingo.io.output.eval_detail import EvalDetail, TokenUsage
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+
logger = logging.getLogger(__name__)
-STANDARD_SYSTEM_PROMPT = """\
-You are a helpful assistant that grades the relevance of search results for given queries.
-Your task is to assign a relevance score between 0.0 and 1.0 to each result, based on
-how good a result is for the query.
+HTML_TAG_PATTERN = r"<[^>]+>"
+MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}|�{2,}"
+INVISIBLE_CHAR_PATTERN = r"[\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]"
+DOI_PATTERN = re.compile(r"(?i)(?:https?://(?:dx\.)?doi\.org/)?(10\.\d{4,9}/[^\s]+)")
-For each search result, carefully read the query and the result. Assign a value for
-each criterion as follows:
-- Provide a brief explanation of your reasoning.
+STANDARD_SYSTEM_PROMPT = """\
+Grade how useful a search result is for a query and assign a relevance score
+from 0.0 to 1.0.
+Read the query, title, and available content, then return these fields:
+- Brief reasoning in 20 words or fewer.
- Assign a query_relevance score between 0.0 and 1.0.
- Assign a result_quality score between 0.0 and 1.0.
-- Indicate if there are any content_issues (true/false).
+- Set content_issues to true or false.
- Assign a confidence score between 0.0 and 1.0.
-- Assign an overall score between 0.0 and 1.0."""
+- Assign an overall score between 0.0 and 1.0.
+
+Set content_issues=true only when mojibake, raw HTML/XML, parser residue, invisible/control
+characters, or similar corruption makes visible content materially unreadable. Missing or short
+abstracts and truncated previews are not issues when the visible title/snippet remains readable.
+
+Return one valid JSON object only. Do not use double quotes inside reasoning."""
DETAILED_SYSTEM_PROMPT = """\
You are a helpful assistant that grades the relevance of search results for given queries.
@@ -73,9 +89,11 @@
2. result_quality: The authority, accuracy, and trustworthiness of the result. High-quality \
results come from reputable sources, are well-written, and are not spammy or misleading.
-3. content_issues: A boolean indicating whether there are problems with the content, such as \
-truncation, missing information, or improper parsing. If the result is incomplete or garbled, \
-set this to true.
+3. content_issues: A boolean indicating severe content corruption only. Set this to true \
+when the visible title/content has garbled or mojibake text, raw HTML/XML or parser residue \
+that materially hurts readability, invisible/control characters, or unreadable text. Do NOT \
+set this to true merely because the abstract is missing, the snippet is short, or the preview \
+is truncated, if the visible title/snippet is still readable enough to judge relevance.
4. confidence: How certain you are about your grading. If the result snippet is clear and \
directly answers the query, confidence should be high. If you need external information to \
@@ -86,15 +104,18 @@
For each search result, carefully read the query and the result. Assign a value for \
each criterion as follows:
-- Provide a brief explanation of your reasoning.
+- Provide a brief explanation of your reasoning in 20 words or fewer.
- Assign a query_relevance score between 0.0 and 1.0.
- Assign a result_quality score between 0.0 and 1.0.
-- Indicate if there are any content_issues (true/false).
+- Indicate if there are severe content_issues (true/false).
- Assign a confidence score between 0.0 and 1.0.
- Assign an overall score between 0.0 and 1.0.
Be consistent and use decimal points for fine-grained differentiation. If you are unsure \
-due to missing or unclear information, lower your confidence and make a best guess as to the score."""
+due to missing or unclear information, lower your confidence and make a best guess as to the score.
+
+Return only one valid JSON object. Keep reasoning short and do not use double quotes inside \
+the reasoning string. If you need quotation marks in reasoning, use single quotes."""
@dataclass
@@ -107,6 +128,7 @@ class RelevanceGrade:
confidence: float = 0.0
reasoning: str = ""
error: str = ""
+ usage: TokenUsage | None = None
def to_dict(self) -> dict[str, Any]:
d: dict[str, Any] = {
@@ -179,11 +201,13 @@ def _build_user_message(
'"result_quality": 0.0-1.0, "content_issues": true/false, '
'"confidence": 0.0-1.0, "score": 0.0-1.0}'
)
+ parts.append(
+ "Return JSON only. Keep reasoning under 20 words and do not use double quotes inside reasoning."
+ )
return "\n".join(parts)
-def _parse_grade_response(response_text: str) -> RelevanceGrade:
- """Parse LLM JSON response into a RelevanceGrade."""
+def _strip_json_fence(response_text: str) -> str:
text = response_text.strip()
if text.startswith("```json"):
text = text[7:]
@@ -191,28 +215,209 @@ def _parse_grade_response(response_text: str) -> RelevanceGrade:
text = text[3:]
if text.endswith("```"):
text = text[:-3]
- text = text.strip()
+ return text.strip()
- try:
- data = json.loads(text)
- except json.JSONDecodeError:
- return RelevanceGrade(error=f"JSON parse failed: {text[:200]}")
+def _extract_json_object(text: str) -> str | None:
+ start = text.find("{")
+ end = text.rfind("}")
+ if start == -1 or end == -1 or end <= start:
+ return None
+ return text[start:end + 1].strip()
+
+
+def _repair_unescaped_quotes_in_reasoning(text: str) -> str:
+ """Escape stray double quotes inside the reasoning JSON string.
+
+ Some models emit otherwise valid JSON such as:
+ {"reasoning": "The query "PBPK" matches", "score": 0.9, ...}
+ The inner quotes break json.loads. This repair scopes the change to the
+ reasoning value and leaves the following JSON keys untouched.
+ """
+ start_match = re.search(r'("reasoning"\s*:\s*")', text)
+ if not start_match:
+ return text
+
+ value_start = start_match.end()
+ next_key = re.search(
+ r'"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"\s*:|})',
+ text[value_start:],
+ flags=re.DOTALL,
+ )
+ if not next_key:
+ return text
+
+ value_end = value_start + next_key.start()
+ value = text[value_start:value_end]
+ repaired_value = re.sub(r'(? float:
try:
- if not isinstance(data, dict):
- return RelevanceGrade(error=f"JSON is not a dictionary: {text[:200]}")
- return RelevanceGrade(
- score=float(data.get("score", 0.0)),
- query_relevance=float(data.get("query_relevance", 0.0)),
- result_quality=float(data.get("result_quality", 0.0)),
- content_issues=bool(data.get("content_issues", False)),
- confidence=float(data.get("confidence", 0.0)),
- reasoning=str(data.get("reasoning", "")),
+ return float(value)
+ except (TypeError, ValueError):
+ return 0.0
+
+
+def _clamp_score(value: Any) -> float:
+ return max(0.0, min(1.0, _coerce_float(value)))
+
+
+def _grade_from_dict(data: dict[str, Any]) -> RelevanceGrade:
+ return RelevanceGrade(
+ score=_clamp_score(data.get("score", 0.0)),
+ query_relevance=_clamp_score(data.get("query_relevance", 0.0)),
+ result_quality=_clamp_score(data.get("result_quality", 0.0)),
+ content_issues=bool(data.get("content_issues", False)),
+ confidence=_clamp_score(data.get("confidence", 0.0)),
+ reasoning=str(data.get("reasoning", "")),
+ )
+
+
+def _parse_grade_fields_lenient(text: str) -> RelevanceGrade | None:
+ number_fields = {}
+ for field in ("query_relevance", "result_quality", "confidence", "score"):
+ match = re.search(
+ rf'"{field}"\s*:\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))',
+ text,
+ flags=re.IGNORECASE,
)
- except (ValueError, TypeError) as e:
- return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {text[:200]}")
+ if match:
+ number_fields[field] = _clamp_score(match.group(1))
+
+ if "score" not in number_fields and "query_relevance" not in number_fields:
+ return None
+
+ issue_match = re.search(r'"content_issues"\s*:\s*(true|false)', text, flags=re.IGNORECASE)
+ reasoning = ""
+ reasoning_match = re.search(
+ r'"reasoning"\s*:\s*"(.*?)"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"|})',
+ text,
+ flags=re.DOTALL,
+ )
+ if reasoning_match:
+ reasoning = " ".join(reasoning_match.group(1).split())
+
+ return RelevanceGrade(
+ score=number_fields.get("score", number_fields.get("query_relevance", 0.0)),
+ query_relevance=number_fields.get("query_relevance", 0.0),
+ result_quality=number_fields.get("result_quality", 0.0),
+ content_issues=issue_match.group(1).lower() == "true" if issue_match else False,
+ confidence=number_fields.get("confidence", 0.0),
+ reasoning=reasoning,
+ )
+
+
+def _parse_grade_response(response_text: str) -> RelevanceGrade:
+ """Parse LLM JSON response into a RelevanceGrade."""
+ text = _strip_json_fence(response_text)
+ candidates = [text]
+ extracted = _extract_json_object(text)
+ if extracted and extracted != text:
+ candidates.append(extracted)
+
+ repaired_candidates = []
+ for candidate in candidates:
+ repaired = _repair_unescaped_quotes_in_reasoning(candidate)
+ if repaired != candidate:
+ repaired_candidates.append(repaired)
+ candidates.extend(repaired_candidates)
+
+ for candidate in candidates:
+ try:
+ data = json.loads(candidate)
+ except json.JSONDecodeError:
+ continue
+
+ try:
+ if not isinstance(data, dict):
+ return RelevanceGrade(error=f"JSON is not a dictionary: {candidate[:200]}")
+ return _grade_from_dict(data)
+ except (ValueError, TypeError) as e:
+ return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {candidate[:200]}")
+
+ lenient = _parse_grade_fields_lenient(extracted or text)
+ if lenient:
+ return lenient
+ return RelevanceGrade(error=f"JSON parse failed: {text[:200]}")
+
+
+def _content_issue_evidence(title: str, abstract: str) -> list[str]:
+ text = "\n".join([str(title or ""), str(abstract or "")])
+ issues: list[str] = []
+ if re.search(MOJIBAKE_EVIDENCE_PATTERN, text):
+ issues.append("mojibake_or_garbled_text")
+ if re.search(INVISIBLE_CHAR_PATTERN, text):
+ issues.append("invisible_or_control_char")
+ html_matches = re.findall(HTML_TAG_PATTERN, text)
+ if html_matches:
+ issues.append("html_or_xml_tag_residue")
+ if re.search(r"(/docserver/|<\?xml| bool:
+ return bool(_content_issue_evidence(title, abstract))
+
+
+def _normalize_doi(value: Any) -> str:
+ """Extract and normalize a DOI from a query or result field."""
+ text = str(value or "").strip()
+ match = DOI_PATTERN.search(text)
+ if not match:
+ return ""
+ return match.group(1).rstrip(".,;:)]}").lower()
+
+
+def is_doi_query(query: str) -> bool:
+ return bool(_normalize_doi(query))
+
+
+def _extract_result_dois(result: dict[str, Any]) -> list[str]:
+ candidates: list[Any] = [result.get("doi"), result.get("unique_id")]
+ locations = result.get("locations")
+ if isinstance(locations, list):
+ for location in locations:
+ if isinstance(location, dict):
+ candidates.extend([location.get("doi"), location.get("url"), location.get("landing_page_url")])
+
+ dois: list[str] = []
+ for candidate in candidates:
+ values = candidate if isinstance(candidate, list) else [candidate]
+ for value in values:
+ doi = _normalize_doi(value)
+ if doi and doi not in dois:
+ dois.append(doi)
+ return dois
+
+
+def _grade_doi_result(query: str, result: dict[str, Any]) -> RelevanceGrade | None:
+ """Use deterministic identifier matching when the query is a DOI."""
+ query_doi = _normalize_doi(query)
+ if not query_doi:
+ return None
+
+ result_dois = _extract_result_dois(result)
+ exact_match = query_doi in result_dois
+ score = 1.0 if exact_match else 0.0
+ result_text = ", ".join(result_dois) if result_dois else "missing"
+ return RelevanceGrade(
+ score=score,
+ query_relevance=score,
+ result_quality=1.0 if exact_match else 0.0,
+ content_issues=False,
+ confidence=1.0,
+ reasoning=(
+ f"Exact DOI match: {query_doi}."
+ if exact_match
+ else f"DOI mismatch: expected {query_doi}; result {result_text}."
+ ),
+ )
+@Model.llm_register("LLMSearchResultRelevance")
class LLMSearchResultRelevance:
"""Exa-style pointwise search result relevance grader.
@@ -220,6 +425,9 @@ class LLMSearchResultRelevance:
``BaseOpenAI`` evaluator hierarchy.
"""
+ dynamic_config = EvaluatorLLMArgs()
+ default_threshold = 0.15
+
def __init__(
self,
*,
@@ -228,12 +436,18 @@ def __init__(
api_url: str | None = None,
prompt_mode: str = "standard",
expected_criteria: str | None = None,
+ max_tokens: int = 1024,
+ temperature: float = 0.0,
+ timeout: float | None = None,
):
self.model = model or "gpt-4o"
self.api_key = api_key
self.api_url = api_url
self.prompt_mode = prompt_mode
self.expected_criteria = expected_criteria
+ self.max_tokens = max_tokens
+ self.temperature = temperature
+ self.timeout = timeout
self._client = None
def _get_client(self):
@@ -261,22 +475,123 @@ def grade(
expected_criteria=expected_criteria or self.expected_criteria,
)
- try:
- client = self._get_client()
- completion = client.chat.completions.create(
- model=self.model,
- messages=[
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_message},
- ],
- temperature=0.0,
- max_tokens=512,
+ client = self._get_client()
+ last_grade = RelevanceGrade(error="LLM grading failed")
+ usage: TokenUsage | None = None
+ for attempt in range(3):
+ try:
+ completion = client.chat.completions.create(
+ model=self.model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_message},
+ ],
+ temperature=self.temperature,
+ max_tokens=self.max_tokens,
+ timeout=self.timeout,
+ )
+ usage = BaseOpenAI._merge_token_usage(
+ usage,
+ BaseOpenAI._extract_token_usage(
+ completion,
+ model_name=self.model,
+ provider="openai",
+ ),
+ )
+ response_text = completion.choices[0].message.content or ""
+ last_grade = _parse_grade_response(response_text)
+ last_grade.usage = usage
+ if not last_grade.error:
+ return last_grade
+ error: Exception | str = last_grade.error
+ except Exception as exc:
+ error = exc
+ last_grade = RelevanceGrade(error=str(exc))
+ last_grade.usage = usage
+
+ logger.warning(
+ "LLM grading attempt %s/3 failed for query=%r title=%r: %s",
+ attempt + 1,
+ query,
+ title,
+ error,
)
- response_text = completion.choices[0].message.content or ""
- return _parse_grade_response(response_text)
- except Exception as e:
- logger.warning("LLM grading failed for query=%r title=%r: %s", query, title, e)
- return RelevanceGrade(error=str(e))
+ if attempt < 2:
+ time.sleep(attempt + 1)
+ return last_grade
+
+ @classmethod
+ def _config_value(cls, name: str, default: Any = None) -> Any:
+ return getattr(cls.dynamic_config, name, default)
+
+ @classmethod
+ def _build_from_config(cls) -> "LLMSearchResultRelevance":
+ return cls(
+ model=cls.dynamic_config.model,
+ api_key=cls.dynamic_config.key,
+ api_url=cls.dynamic_config.api_url,
+ prompt_mode=str(cls._config_value("prompt_mode", "detailed") or "detailed"),
+ expected_criteria=cls._config_value("expected_criteria", None),
+ max_tokens=int(cls._config_value("max_tokens", 1024) or 1024),
+ temperature=float(cls._config_value("temperature", 0.0) or 0.0),
+ timeout=cls._config_value("timeout", None),
+ )
+
+ @staticmethod
+ def _extract_title(result: dict[str, Any]) -> str:
+ return str(result.get("title") or result.get("display_name") or "")
+
+ @staticmethod
+ def _extract_abstract(result: dict[str, Any]) -> str:
+ return str(result.get("abstract") or result.get("summary") or result.get("content") or "")
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ """Executor entry point: evaluate one flattened query-result pair."""
+ result = getattr(input_data, "search_result", None)
+ if not isinstance(result, dict):
+ result = input_data.to_dict()
+ query = str(getattr(input_data, "query", "") or result.get("_eval_query") or result.get("query") or "")
+
+ title = cls._extract_title(result)
+ abstract = cls._extract_abstract(result)
+ grade = _grade_doi_result(query, result)
+ if grade is None:
+ grader = cls._build_from_config()
+ grade = grader.grade(
+ query=query,
+ title=title,
+ abstract=abstract,
+ )
+ threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold)
+ content_issue_evidence = _content_issue_evidence(title, abstract) if grade.content_issues else []
+ effective_content_issues = bool(content_issue_evidence)
+
+ labels: list[str] = []
+ if grade.error:
+ labels.append("Relevance.Error_Parse")
+ if grade.score < threshold:
+ labels.append("Relevance.Error_Relevance_Low")
+ if effective_content_issues:
+ labels.append("Relevance.Error_Content_Issues")
+
+ status = bool(labels)
+ if not labels:
+ labels = ["QUALITY_GOOD"]
+
+ reason = grade.to_dict()
+ reason["raw_content_issues"] = grade.content_issues
+ reason["content_issues"] = effective_content_issues
+ reason["content_issue_evidence"] = content_issue_evidence
+
+ return EvalDetail(
+ metric=cls.__name__,
+ status=status,
+ score=round(grade.score, 5),
+ label=labels,
+ reason=[reason],
+ usage=grade.usage,
+ )
def aggregate_grades(
diff --git a/dingo/model/llm/rag/llm_rag_answer_relevancy.py b/dingo/model/llm/rag/llm_rag_answer_relevancy.py
index 86febfa7..898d31b2 100644
--- a/dingo/model/llm/rag/llm_rag_answer_relevancy.py
+++ b/dingo/model/llm/rag/llm_rag_answer_relevancy.py
@@ -13,6 +13,7 @@
from dingo.io.input import Data, RequiredField
from dingo.io.output.eval_detail import EvalDetail
from dingo.model import Model
+from dingo.model.llm.base import LLMCallResult, llm_response_content
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils import log
from dingo.utils.exception import ConvertJsonError
@@ -106,7 +107,14 @@ def build_messages(cls, input_data: Data) -> List:
@classmethod
def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[str, Any]]:
"""生成多个相关问题"""
+ questions, _ = cls._generate_multiple_questions_with_usage(input_data, n)
+ return questions
+
+ @classmethod
+ def _generate_multiple_questions_with_usage(cls, input_data: Data, n: int = 3):
+ """生成多个相关问题，同时返回 LLM token 使用量"""
questions = []
+ usage = None
# 确保客户端已经创建
if not hasattr(cls, 'client') or cls.client is None:
@@ -117,13 +125,16 @@ def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[
messages = cls.build_messages(input_data)
# 调用LLM生成问题
- response = cls.send_messages(messages)
+ response_result = cls.send_messages(messages)
+ response = llm_response_content(response_result)
+ if isinstance(response_result, LLMCallResult):
+ usage = cls._merge_token_usage(usage, response_result.usage)
# 处理响应
processed_response = cls.process_question_response(response)
questions.append(processed_response)
- return questions
+ return questions, usage
@classmethod
def process_question_response(cls, response: str) -> Dict[str, Any]:
@@ -246,7 +257,9 @@ def eval(cls, input_data: Data) -> EvalDetail:
cls.dynamic_config.temperature = 0.7
# 生成多个相关问题
- generated_questions = cls.generate_multiple_questions(input_data, cls.strictness)
+ generated_questions, usage = cls._generate_multiple_questions_with_usage(
+ input_data, cls.strictness
+ )
# 计算相关性分数和详细信息
score, details = cls.calculate_score(generated_questions, original_question)
@@ -254,6 +267,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
# 构建结果
result = EvalDetail(metric=cls.__name__)
result.score = score
+ result.usage = usage
# 根据分数判断是否通过，默认阈值为5
threshold = 5
diff --git a/dingo/model/llm/rag/llm_rag_context_precision.py b/dingo/model/llm/rag/llm_rag_context_precision.py
index a5927ec3..c1be34ed 100644
--- a/dingo/model/llm/rag/llm_rag_context_precision.py
+++ b/dingo/model/llm/rag/llm_rag_context_precision.py
@@ -10,6 +10,7 @@
from dingo.io.input import Data, RequiredField
from dingo.io.output.eval_detail import EvalDetail
from dingo.model import Model
+from dingo.model.llm.base import LLMCallResult, llm_response_content
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils import log
from dingo.utils.exception import ConvertJsonError
@@ -293,6 +294,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
# 获取所有上下文的消息
messages_list = cls.build_messages(input_data)
responses = []
+ usage = None
# 为每个上下文发送单独的请求
for item in messages_list:
@@ -302,7 +304,10 @@ def eval(cls, input_data: Data) -> EvalDetail:
while attempts < 3:
try:
- response = cls.send_messages(messages)
+ response_result = cls.send_messages(messages)
+ response = llm_response_content(response_result)
+ if isinstance(response_result, LLMCallResult):
+ usage = cls._merge_token_usage(usage, response_result.usage)
break
except Exception as e:
attempts += 1
@@ -326,4 +331,6 @@ def eval(cls, input_data: Data) -> EvalDetail:
responses.append(response)
# 处理所有响应
- return cls.process_response(responses)
+ result = cls.process_response(responses)
+ result.usage = usage
+ return result
diff --git a/dingo/model/llm/vlm_layout_quality.py b/dingo/model/llm/vlm_layout_quality.py
index 14c03fc2..812ad145 100644
--- a/dingo/model/llm/vlm_layout_quality.py
+++ b/dingo/model/llm/vlm_layout_quality.py
@@ -4,6 +4,7 @@
from dingo.io.input import Data, RequiredField
from dingo.io.output.eval_detail import EvalDetail
from dingo.model import Model
+from dingo.model.llm.base import LLMCallResult
from dingo.model.llm.base_openai import BaseOpenAI
from dingo.utils import log
from dingo.utils.image_loader import ImageLoader
@@ -159,7 +160,14 @@ def send_messages(cls, messages: List):
temperature=0.1
)
- return str(completions.choices[0].message.content)
+ return LLMCallResult(
+ content=str(completions.choices[0].message.content),
+ usage=cls._extract_token_usage(
+ completions,
+ model_name=model_name,
+ provider="openai",
+ ),
+ )
@classmethod
def process_response(cls, response: str) -> EvalDetail:
diff --git a/dingo/model/rule/guobiao/__init__.py b/dingo/model/rule/guobiao/__init__.py
new file mode 100644
index 00000000..9a8d56ca
--- /dev/null
+++ b/dingo/model/rule/guobiao/__init__.py
@@ -0,0 +1 @@
+"""Rules implementing SAC/TC609 high-quality dataset standards."""
diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py
new file mode 100644
index 00000000..aa418263
--- /dev/null
+++ b/dingo/model/rule/guobiao/rule_tc609_quality.py
@@ -0,0 +1,1584 @@
+import importlib.util
+import math
+from datetime import datetime, timezone
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io.input import Data, RequiredField
+from dingo.io.output.eval_detail import EvalDetail, QualityLabel
+from dingo.model.model import Model
+from dingo.model.rule.base import BaseRule
+from dingo.model.rule.guobiao.rule_tc609_quality_base import (TC609_DATASET_TYPE_DESCRIPTIONS, Rule_TC609_01_DocCompleteness, Rule_TC609_Composite, _tc609_metric_info, _TC609PlaceholderBase,
+ calculate_text_consistency)
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"])
+class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness):
+ """0101: Basic information completeness in dataset documentation."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "COMPLETENESS",
+ "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness",
+ "description": (
+ "Checks whether dataset documentation covers basic information "
+ "aspects such as scale, format, structure, access, and support"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "0101",
+ "coverage": "covered",
+ }
+ _default_dimension_name = "Basic information"
+ _default_aspect_keywords = {
+ "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"],
+ "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"],
+ "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"],
+ "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"],
+ "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"],
+ }
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=0.8,
+ dimension_name=_default_dimension_name,
+ aspect_keywords=_default_aspect_keywords,
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao_doc"])
+class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompleteness):
+ """0102: Content feature completeness in dataset documentation."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "COMPLETENESS",
+ "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness",
+ "description": (
+ "Checks whether dataset documentation covers content-feature aspects "
+ "such as modality, distribution, labels, examples, and limitations"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "0102",
+ "coverage": "covered",
+ }
+ _default_dimension_name = "Content feature"
+ _default_aspect_keywords = {
+ "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"],
+ "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"],
+ "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"],
+ "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"],
+ "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"],
+ }
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=0.8,
+ dimension_name=_default_dimension_name,
+ aspect_keywords=_default_aspect_keywords,
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao_doc"])
+class Rule_TC609_0103_DocConstructionProcessCompleteness(
+ Rule_TC609_01_DocCompleteness
+):
+ """0103: Construction-process completeness in dataset documentation."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "COMPLETENESS",
+ "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness",
+ "description": (
+ "Checks whether dataset documentation covers construction-process "
+ "aspects such as data source, collection, processing, annotation, "
+ "and version control"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "0103",
+ "coverage": "covered",
+ }
+ _default_dimension_name = "Construction process"
+ _default_aspect_keywords = {
+ "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"],
+ "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"],
+ "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"],
+ "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"],
+ "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"],
+ }
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=0.8,
+ dimension_name=_default_dimension_name,
+ aspect_keywords=_default_aspect_keywords,
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao_doc"])
+class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness):
+ """0104: Application-description completeness in dataset documentation."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "COMPLETENESS",
+ "metric_name": "Rule_TC609_0104_DocApplicationCompleteness",
+ "description": (
+ "Checks whether dataset documentation covers application aspects "
+ "such as license, scenarios, evaluation method, benchmark, and cases"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "0104",
+ "coverage": "covered",
+ }
+ _default_dimension_name = "Application description"
+ _default_aspect_keywords = {
+ "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"],
+ "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"],
+ "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"],
+ "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"],
+ "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"],
+ }
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=0.8,
+ dimension_name=_default_dimension_name,
+ aspect_keywords=_default_aspect_keywords,
+ )
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao_data"])
+class Rule_TC609_0201_FormatCompliance(BaseRule):
+ """Check whether a data record matches a user-provided field schema."""
+
+ _supported_types = {
+ "str": {"expected_type": str, "allow_none": False},
+ "int": {"expected_type": int, "allow_none": False},
+ "float": {"expected_type": float, "allow_none": False},
+ "bool": {"expected_type": bool, "allow_none": False},
+ "list": {"expected_type": list, "allow_none": False},
+ "dict": {"expected_type": dict, "allow_none": False},
+ "Optional[str]": {"expected_type": str, "allow_none": True},
+ "Optional[int]": {"expected_type": int, "allow_none": True},
+ "Optional[float]": {"expected_type": float, "allow_none": True},
+ "Optional[bool]": {"expected_type": bool, "allow_none": True},
+ "Optional[list]": {"expected_type": list, "allow_none": True},
+ "Optional[dict]": {"expected_type": dict, "allow_none": True},
+ }
+ dynamic_config = EvaluatorRuleArgs(
+ field_schema={
+ "id": "str",
+ "rid": "Optional[list]",
+ "data_content": "list",
+ "annotation": "Optional[dict]",
+ "original_time": "str",
+ "last_modified_time": "str",
+ "version": "str",
+ "license": "str",
+ "source": "str",
+ "source_details": "str",
+ "generated_data_indicator": "int",
+ },
+ allow_extra=True,
+ )
+ _metric_info = _tc609_metric_info(
+ "0201",
+ "Rule_TC609_0201_FormatCompliance",
+ "Checks required fields and their types against a user-provided schema.",
+ "covered",
+ )
+
+ @classmethod
+ def _validate_schema(cls, schema):
+ if not isinstance(schema, dict) or not schema:
+ raise ValueError(
+ "Rule_TC609_0201_FormatCompliance requires a non-empty "
+ "dynamic_config.field_schema"
+ )
+
+ for field_name, type_name in schema.items():
+ if not isinstance(field_name, str) or not field_name:
+ raise ValueError(
+ "Rule_TC609_0201_FormatCompliance schema field names "
+ "must be non-empty strings"
+ )
+ if (
+ not isinstance(type_name, str)
+ or type_name not in cls._supported_types
+ ):
+ supported = sorted(cls._supported_types.keys())
+ raise ValueError(
+ f"Unsupported schema type for field {field_name!r}: "
+ f"{type_name!r}. Supported types: {', '.join(supported)}"
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ schema = getattr(cls.dynamic_config, "field_schema", None)
+ cls._validate_schema(schema)
+
+ record = input_data.model_dump()
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+ for field_name, type_name in schema.items():
+ expected_type = cls._supported_types[type_name]["expected_type"]
+ allow_none = cls._supported_types[type_name]["allow_none"]
+ if field_name not in record:
+ reasons.append(f"{field_name}: required field is missing")
+ res.status = True
+ continue
+
+ value = record[field_name]
+ if allow_none and value is None:
+ continue
+
+ if type(value) is not expected_type:
+ reasons.append(
+ f"{field_name}: expected {type_name}, "
+ f"got {type(value).__name__}"
+ )
+ res.status = True
+
+ if not getattr(cls.dynamic_config, "allow_extra", True):
+ for field_name in sorted(record.keys() - schema.keys()):
+ reasons.append(f"{field_name}: unexpected field")
+ res.status = True
+
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao_data"])
+class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite):
+ """0202: Safety compliance, composed from safety and PII rules."""
+
+ dynamic_config = EvaluatorRuleArgs(
+ key_list=[
+ "制作炸弹",
+ "购买毒品",
+ "贩卖毒品",
+ "实施诈骗",
+ "洗钱教程",
+ "自杀方法",
+ "色情交易",
+ "儿童色情",
+ "恐怖袭击",
+ "非法枪支",
+ ],
+ refer_path=[],
+ )
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleUnsafeWords",
+ "dingo.model.rule.rule_common.RulePIIDetection",
+ "dingo.model.rule.rule_common.RuleIDCard",
+ )
+ _required_fields = [RequiredField.DATA_CONTENT]
+ _metric_info = _tc609_metric_info(
+ "0202",
+ "Rule_TC609_0202_SafetyCompliance",
+ "Combines unsafe-word, PII, and identity-card detection.",
+ "partial",
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+ if not isinstance(input_data.data_content, list):
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "data_content: expected list, "
+ f"got {type(input_data.data_content).__name__}"
+ ]
+ return res
+
+ texts = []
+ for index, item in enumerate(input_data.data_content):
+ if not isinstance(item, dict):
+ res.status = True
+ reasons.append(
+ f"data_content[{index}]: expected dict, "
+ f"got {type(item).__name__}"
+ )
+ continue
+ media_type = item.get("media_type")
+ if not isinstance(media_type, str) or not media_type.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].media_type: expected non-empty str"
+ )
+ continue
+ if media_type.strip().lower() != "text":
+ continue
+ content = item.get("content")
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].content: expected non-empty str"
+ )
+ continue
+ texts.append(content.strip())
+
+ if not texts:
+ res.status = True
+ reasons.append(
+ "data_content: at least one text item is required"
+ )
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ return res
+
+ rule_unsafe_words = cls._resolve_rule(cls.component_rules[0])
+ unsafe_words_config = EvaluatorRuleArgs(
+ key_list=cls.dynamic_config.key_list or [],
+ refer_path=cls.dynamic_config.refer_path or [],
+ )
+ if (
+ getattr(rule_unsafe_words, "dynamic_config", None)
+ != unsafe_words_config
+ ):
+ rule_unsafe_words._unsafe_words_list = None
+ rule_unsafe_words._unsafe_words_automaton = None
+ rule_unsafe_words.dynamic_config = unsafe_words_config
+ text_input = input_data.model_copy(
+ update={"content": "\n".join(texts)}
+ )
+ return super().eval(text_input)
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"])
+class Rule_TC609_0203_AnnotationCompliance(BaseRule):
+ """Check annotation metadata against TC609 format requirements."""
+
+ _annotation_methods = {
+ "人工标注",
+ "自动标注",
+ "半自动标注",
+ "其他",
+ }
+ _annotator_types = {
+ "普通标注员",
+ "专业标注员",
+ "行业领域专家",
+ "其他",
+ }
+ _required_fields = [RequiredField.ANNOTATION]
+ _metric_info = _tc609_metric_info(
+ "0203",
+ "Rule_TC609_0203_AnnotationCompliance",
+ "Checks annotation metadata fields, types, and enumerated values.",
+ "covered",
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+ annotation = input_data.annotation
+
+ if annotation is None:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+ if not isinstance(annotation, dict):
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "annotation: expected dict or None, "
+ f"got {type(annotation).__name__}"
+ ]
+ return res
+
+ if "label" not in annotation:
+ res.status = True
+ reasons.append("annotation.label: required field is missing")
+ elif not isinstance(annotation["label"], list):
+ res.status = True
+ reasons.append(
+ "annotation.label: expected list, "
+ f"got {type(annotation['label']).__name__}"
+ )
+ elif not annotation["label"]:
+ res.status = True
+ reasons.append("annotation.label: empty value is not allowed")
+
+ for field_name, allowed_values in (
+ ("annotation_method", cls._annotation_methods),
+ ("annotator", cls._annotator_types),
+ ):
+ qualified_name = f"annotation.{field_name}"
+ if field_name not in annotation:
+ res.status = True
+ reasons.append(f"{qualified_name}: required field is missing")
+ continue
+
+ value = annotation[field_name]
+ if value is None:
+ continue
+ if not isinstance(value, str):
+ res.status = True
+ reasons.append(
+ f"{qualified_name}: expected str or None, "
+ f"got {type(value).__name__}"
+ )
+ elif value not in allowed_values:
+ res.status = True
+ reasons.append(
+ f"{qualified_name}: unsupported value {value!r}; "
+ f"allowed values: {', '.join(sorted(allowed_values))}"
+ )
+
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"])
+class Rule_TC609_0204_StructuralCompleteness(BaseRule):
+ """Check required fields for missing, None, and empty values."""
+
+ dynamic_config = EvaluatorRuleArgs(
+ key_list=[
+ "id",
+ "data_content",
+ "original_time",
+ "last_modified_time",
+ "version",
+ "license",
+ "source",
+ "source_details",
+ "generated_data_indicator",
+ ],
+ allow_none=False,
+ allow_empty=False,
+ )
+ _metric_info = _tc609_metric_info(
+ "0204",
+ "Rule_TC609_0204_StructuralCompleteness",
+ "Checks configured fields for missing, None, and empty values.",
+ "covered",
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ key_list = cls.dynamic_config.key_list or []
+ if not key_list:
+ raise ValueError(
+ "Rule_TC609_0204_StructuralCompleteness requires a non-empty "
+ "dynamic_config.key_list"
+ )
+
+ record = input_data.model_dump()
+ allow_none = getattr(cls.dynamic_config, "allow_none", False)
+ allow_empty = getattr(cls.dynamic_config, "allow_empty", False)
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+
+ for field_name in key_list:
+ if field_name not in record:
+ reasons.append(f"{field_name}: required field is missing")
+ res.status = True
+ continue
+
+ value = record[field_name]
+ if value is None and not allow_none:
+ reasons.append(f"{field_name}: None is not allowed")
+ res.status = True
+ continue
+
+ if (
+ not allow_empty
+ and isinstance(value, (str, list, dict))
+ and len(value) == 0
+ ):
+ reasons.append(f"{field_name}: empty value is not allowed")
+ res.status = True
+
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao_data"])
+class Rule_TC609_0205_ContentAuthenticity(BaseRule):
+ """Check whether source metadata provides valid traceability information."""
+
+ _required_fields = [
+ RequiredField.SOURCE,
+ RequiredField.SOURCE_DETAILS,
+ ]
+ _metric_info = _tc609_metric_info(
+ "0205",
+ "Rule_TC609_0205_ContentAuthenticity",
+ "Checks source and source_details, including URL format when applicable.",
+ "covered",
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ source = input_data.source
+ source_details = input_data.source_details
+
+ if not isinstance(source, str) or not source.strip():
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["source: expected a non-empty string"]
+ return res
+
+ if not isinstance(source_details, str) or not source_details.strip():
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["source_details: expected a non-empty string"]
+ return res
+
+ if source.strip() == "互联网":
+ if not cls._is_valid_http_url(source_details):
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "source_details: expected a valid HTTP or HTTPS URL"
+ ]
+ return res
+
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+ @staticmethod
+ def _is_valid_http_url(value):
+ from urllib.parse import urlparse
+
+ if any(character.isspace() for character in value):
+ return False
+ try:
+ parsed = urlparse(value)
+ if (
+ parsed.scheme.lower() not in {"http", "https"}
+ or not parsed.netloc
+ or not parsed.hostname
+ ):
+ return False
+ parsed.port
+ except ValueError:
+ return False
+ return True
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"])
+class Rule_TC609_0206_ContentConsistency(BaseRule):
+ """Check semantic consistency among text items in data_content."""
+
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=0.5,
+ model="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+ device=-1,
+ batch_size=16,
+ max_length=512,
+ consensus_keep_ratio=0.8,
+ )
+ _metric_info = _tc609_metric_info(
+ "0206",
+ "Rule_TC609_0206_ContentConsistency",
+ "Checks semantic consistency among text items in data_content.",
+ "partial",
+ )
+ _required_fields = [RequiredField.DATA_CONTENT]
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ data_content = input_data.data_content
+ if not isinstance(data_content, list) or not data_content:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["data_content: expected a non-empty list"]
+ return res
+
+ texts = []
+ text_indexes = []
+ reasons = []
+ for index, item in enumerate(data_content):
+ if not isinstance(item, dict):
+ res.status = True
+ reasons.append(
+ f"data_content[{index}]: expected dict, "
+ f"got {type(item).__name__}"
+ )
+ continue
+
+ media_type = item.get("media_type")
+ if not isinstance(media_type, str) or not media_type.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].media_type: "
+ "expected a non-empty string"
+ )
+ continue
+ if media_type.strip().lower() != "text":
+ continue
+
+ content = item.get("content")
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].content: "
+ "expected a non-empty string for text media"
+ )
+ continue
+ texts.append(content)
+ text_indexes.append(index)
+
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ return res
+
+ if len(texts) < 2:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ res.reason = [
+ "Fewer than two text items; consistency comparison is not needed"
+ ]
+ return res
+
+ result = calculate_text_consistency(
+ texts=texts,
+ model_name=cls.dynamic_config.model,
+ device=cls.dynamic_config.device,
+ threshold=cls.dynamic_config.threshold,
+ batch_size=getattr(cls.dynamic_config, "batch_size", 16),
+ max_length=getattr(cls.dynamic_config, "max_length", 512),
+ consensus_keep_ratio=getattr(
+ cls.dynamic_config,
+ "consensus_keep_ratio",
+ 0.8,
+ ),
+ )
+ res.score = result["score"]
+ if not result["is_consistent"]:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "Text items in data_content are inconsistent "
+ f"(score: {res.score:.4f}, "
+ f"threshold: {cls.dynamic_config.threshold:.4f}, "
+ "outlier indexes: "
+ f"{[text_indexes[index] for index in result['outlier_indexes']]})"
+ ]
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ res.reason = [
+ "Text items in data_content are consistent "
+ f"(score: {res.score:.4f}, "
+ f"threshold: {cls.dynamic_config.threshold:.4f})"
+ ]
+ return res
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao_data"])
+class Rule_TC609_0207_DataTypeConsistency(BaseRule):
+ """Check whether text content matches the configured dataset type."""
+
+ _metric_info = _tc609_metric_info(
+ "0207",
+ "Rule_TC609_0207_DataTypeConsistency",
+ "Checks whether text content matches the configured dataset type.",
+ "partial",
+ )
+
+ _required_fields = [RequiredField.DATA_CONTENT]
+ dynamic_config = EvaluatorRuleArgs(
+ dataset_type="通识数据集",
+ threshold=0.5,
+ model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+ device=-1,
+ )
+
+ _model_name = None
+ _model_device = None
+ _classifier = None
+
+ @classmethod
+ def _get_classifier(cls, model_name, device):
+ if (
+ cls._model_name == model_name
+ and cls._model_device == device
+ and cls._classifier is not None
+ ):
+ return cls._classifier
+
+ required_packages = ("torch", "transformers")
+ missing_packages = [
+ package
+ for package in required_packages
+ if importlib.util.find_spec(package) is None
+ ]
+ if missing_packages:
+ raise ImportError(
+ "Rule_TC609_0207_DataTypeConsistency requires optional packages: "
+ f"{', '.join(missing_packages)}. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ )
+
+ from transformers import pipeline
+
+ cls._classifier = pipeline(
+ "zero-shot-classification",
+ model=model_name,
+ device=device,
+ )
+ cls._model_name = model_name
+ cls._model_device = device
+ return cls._classifier
+
+ @classmethod
+ def _classify_dataset_type(cls, content, model_name, device):
+ classifier = cls._get_classifier(model_name, device)
+ dataset_types = list(TC609_DATASET_TYPE_DESCRIPTIONS)
+ descriptions = [
+ TC609_DATASET_TYPE_DESCRIPTIONS[dataset_type]
+ for dataset_type in dataset_types
+ ]
+ result = classifier(
+ content,
+ candidate_labels=descriptions,
+ hypothesis_template="这段文本符合以下数据集类型要求：{}",
+ multi_label=False,
+ truncation=True,
+ )
+ labels = result.get("labels", [])
+ scores = result.get("scores", [])
+ if len(labels) != len(descriptions) or len(scores) != len(descriptions):
+ raise RuntimeError("Zero-shot classifier returned an invalid result")
+
+ scores_by_type = {}
+ for label, score in zip(labels, scores):
+ score = float(score)
+ if (
+ label not in descriptions
+ or not math.isfinite(score)
+ or not 0.0 <= score <= 1.0
+ ):
+ raise RuntimeError(
+ "Zero-shot classifier returned an invalid result"
+ )
+ dataset_type = dataset_types[descriptions.index(label)]
+ scores_by_type[dataset_type] = score
+ if len(scores_by_type) != len(dataset_types):
+ raise RuntimeError("Zero-shot classifier returned an invalid result")
+ predicted_type = dataset_types[descriptions.index(labels[0])]
+ return predicted_type, scores_by_type
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ dataset_type = cls.dynamic_config.dataset_type
+ if dataset_type not in TC609_DATASET_TYPE_DESCRIPTIONS:
+ raise ValueError(
+ "Rule_TC609_0207_DataTypeConsistency "
+ "dynamic_config.dataset_type must be one of: "
+ f"{', '.join(TC609_DATASET_TYPE_DESCRIPTIONS)}"
+ )
+
+ threshold = cls.dynamic_config.threshold
+ if threshold is None or not 0 < threshold <= 1:
+ raise ValueError(
+ "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]"
+ )
+
+ texts = []
+ reasons = []
+ if not isinstance(input_data.data_content, list):
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "data_content: expected list, "
+ f"got {type(input_data.data_content).__name__}"
+ ]
+ return res
+ for index, item in enumerate(input_data.data_content):
+ if not isinstance(item, dict):
+ res.status = True
+ reasons.append(
+ f"data_content[{index}]: expected dict, "
+ f"got {type(item).__name__}"
+ )
+ continue
+ media_type = item.get("media_type")
+ if not isinstance(media_type, str) or not media_type.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].media_type: "
+ "expected non-empty str"
+ )
+ continue
+ if media_type.strip().lower() != "text":
+ continue
+ content = item.get("content")
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].content: expected non-empty str"
+ )
+ continue
+ texts.append(content.strip())
+
+ if not texts:
+ res.status = True
+ reasons.append("data_content: at least one text item is required")
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ return res
+
+ predicted_type, scores_by_type = cls._classify_dataset_type(
+ "\n".join(texts),
+ cls.dynamic_config.model,
+ cls.dynamic_config.device,
+ )
+ res.score = scores_by_type[dataset_type]
+
+ if predicted_type == dataset_type and res.score >= threshold:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ res.reason = [
+ f"Text content matches dataset type {dataset_type} "
+ f"(score: {res.score:.4f}, threshold: {threshold:.4f})"
+ ]
+ else:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ f"Text content does not match dataset type {dataset_type}; "
+ f"predicted type: {predicted_type} "
+ f"(score: {res.score:.4f}, threshold: {threshold:.4f})"
+ ]
+ return res
+
+
+@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao_data"])
+class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite):
+ """0208: Content cleanliness, composed from available cleaning rules."""
+
+ dynamic_config = EvaluatorRuleArgs(
+ key_list=[
+ "版权所有",
+ "Copyright",
+ "未经授权不得转载",
+ "禁止转载",
+ "仅供学习交流",
+ ]
+ )
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleAbnormalChar",
+ "dingo.model.rule.rule_common.RuleAbnormalHtml",
+ "dingo.model.rule.rule_common.RuleDocRepeat",
+ "dingo.model.rule.rule_common.RuleContentNull",
+ "dingo.model.rule.rule_common.RuleWatermark",
+ )
+ _required_fields = [RequiredField.DATA_CONTENT]
+ _metric_info = _tc609_metric_info(
+ "0208",
+ "Rule_TC609_0208_ContentCleanliness",
+ "Combines available text cleanliness checks; modality coverage is partial.",
+ "partial",
+ )
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+ if not isinstance(input_data.data_content, list):
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ "data_content: expected list, "
+ f"got {type(input_data.data_content).__name__}"
+ ]
+ return res
+
+ texts = []
+ for index, item in enumerate(input_data.data_content):
+ if not isinstance(item, dict):
+ res.status = True
+ reasons.append(
+ f"data_content[{index}]: expected dict, "
+ f"got {type(item).__name__}"
+ )
+ continue
+ media_type = item.get("media_type")
+ if not isinstance(media_type, str) or not media_type.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].media_type: expected non-empty str"
+ )
+ continue
+ if media_type.strip().lower() != "text":
+ continue
+ content = item.get("content")
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ reasons.append(
+ f"data_content[{index}].content: expected non-empty str"
+ )
+ continue
+ texts.append(content.strip())
+
+ if not texts:
+ res.status = True
+ reasons.append(
+ "data_content: at least one text item is required"
+ )
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ return res
+
+ rule_watermark = cls._resolve_rule(cls.component_rules[-1])
+ rule_watermark.dynamic_config = EvaluatorRuleArgs(
+ key_list=cls.dynamic_config.key_list or [],
+ )
+ text_input = input_data.model_copy(
+ update={"content": "\n".join(texts)}
+ )
+ return super().eval(text_input)
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"])
+class Rule_TC609_02080101_TextPerplexity(BaseRule):
+ """Check whether text perplexity exceeds the configured threshold."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "FLUENCY",
+ "metric_name": "Rule_TC609_02080101_TextPerplexity",
+ "description": (
+ "Calculates text perplexity with a causal language model and "
+ "flags text whose PPL exceeds the configured threshold"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "02080101",
+ "coverage": "covered",
+ }
+
+ _required_fields = [RequiredField.CONTENT]
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=100.0,
+ model="uer/gpt2-chinese-cluecorpussmall",
+ stride=512,
+ )
+
+ _model_name = None
+ _tokenizer = None
+ _model = None
+
+ @classmethod
+ def _check_dependencies(cls):
+ required_packages = ("torch", "transformers")
+ missing_packages = [
+ package
+ for package in required_packages
+ if importlib.util.find_spec(package) is None
+ ]
+ if missing_packages:
+ raise ImportError(
+ "Rule_TC609_02080101_TextPerplexity requires optional packages: "
+ f"{', '.join(missing_packages)}. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ )
+
+ @classmethod
+ def _get_model_components(cls, model_name):
+ if (
+ cls._model_name == model_name
+ and cls._tokenizer is not None
+ and cls._model is not None
+ ):
+ return cls._tokenizer, cls._model
+
+ try:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ except ImportError as exc:
+ raise ImportError(
+ "Rule_TC609_02080101_TextPerplexity requires transformers and torch. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ ) from exc
+
+ cls._tokenizer = AutoTokenizer.from_pretrained(model_name)
+ cls._model = AutoModelForCausalLM.from_pretrained(model_name)
+ cls._model.eval()
+ cls._model_name = model_name
+ return cls._tokenizer, cls._model
+
+ @classmethod
+ def _calculate_perplexity(cls, content, tokenizer, model, stride):
+ try:
+ import torch
+ except ImportError as exc:
+ raise ImportError(
+ "Rule_TC609_02080101_TextPerplexity requires transformers and torch. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ ) from exc
+
+ encodings = tokenizer(content, return_tensors="pt")
+ input_ids = encodings["input_ids"]
+ sequence_length = input_ids.size(1)
+ if sequence_length < 2:
+ raise ValueError(
+ "Rule_TC609_02080101_TextPerplexity requires at least two model tokens"
+ )
+
+ model_config = getattr(model, "config", None)
+ max_length = getattr(model_config, "n_positions", None)
+ if max_length is None:
+ max_length = getattr(model_config, "max_position_embeddings", 1024)
+ max_length = int(max_length)
+ stride = max(1, min(int(stride), max_length))
+
+ try:
+ device = next(model.parameters()).device
+ except StopIteration:
+ device = torch.device("cpu")
+
+ total_negative_log_likelihood = 0.0
+ total_loss_tokens = 0
+ previous_end = 0
+
+ for begin in range(0, sequence_length, stride):
+ end = min(begin + max_length, sequence_length)
+ target_length = end - previous_end
+ window = input_ids[:, begin:end].to(device)
+ targets = window.clone()
+ targets[:, :-target_length] = -100
+
+ with torch.no_grad():
+ output = model(window, labels=targets)
+
+ loss_tokens = int((targets[:, 1:] != -100).sum().item())
+ if loss_tokens > 0:
+ total_negative_log_likelihood += output.loss.item() * loss_tokens
+ total_loss_tokens += loss_tokens
+
+ previous_end = end
+ if end == sequence_length:
+ break
+
+ if total_loss_tokens == 0:
+ raise ValueError(
+ "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input"
+ )
+
+ mean_loss = total_negative_log_likelihood / total_loss_tokens
+ try:
+ return math.exp(mean_loss)
+ except OverflowError:
+ return float("inf")
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ cls._check_dependencies()
+
+ res = EvalDetail(metric=cls.__name__)
+ content = input_data.content
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["Text perplexity cannot be calculated for empty content"]
+ return res
+
+ threshold = cls.dynamic_config.threshold
+ if threshold is None or threshold <= 0:
+ raise ValueError(
+ "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0"
+ )
+
+ model_name = getattr(
+ cls.dynamic_config,
+ "model",
+ "uer/gpt2-chinese-cluecorpussmall",
+ )
+ stride = getattr(cls.dynamic_config, "stride", 512)
+ tokenizer, model = cls._get_model_components(model_name)
+ perplexity = cls._calculate_perplexity(
+ content,
+ tokenizer,
+ model,
+ stride,
+ )
+
+ if perplexity > threshold:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ f"Text perplexity {perplexity:.4f} exceeds threshold "
+ f"{threshold:.4f} (model: {model_name})"
+ ]
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ res.reason = [
+ f"Text perplexity: {perplexity:.4f} "
+ f"(threshold: {threshold:.4f}, model: {model_name})"
+ ]
+ return res
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao_text"])
+class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleAlphaWords",
+ "dingo.model.rule.rule_common.RuleStopWord",
+ "dingo.model.rule.rule_common.RuleUniqueWords",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080102",
+ "Rule_TC609_02080102_KnowledgeInformationDensity",
+ "Combines alphabetic-word, stop-word, and unique-word ratio checks.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao_text"])
+class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleDocRepeat",
+ "dingo.model.rule.rule_common.RuleDocFormulaRepeat",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080103",
+ "Rule_TC609_02080103_RepeatedContent",
+ "Combines document-text and formula repetition checks.",
+ "covered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao_text"])
+class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleContentNull",
+ "dingo.model.rule.rule_common.RuleContentShort",
+ "dingo.model.rule.rule_common.RuleLineEndWithEllipsis",
+ "dingo.model.rule.rule_common.RuleLineEndWithTerminal",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080104",
+ "Rule_TC609_02080104_TextCompleteness",
+ "Combines null, short, ellipsis-ending, and terminal-ending checks.",
+ "covered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao_text"])
+class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleContentNull",
+ "dingo.model.rule.rule_common.RuleContentShort",
+ "dingo.model.rule.rule_common.RuleSentenceNumber",
+ "dingo.model.rule.rule_common.RuleWordNumber",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080105",
+ "Rule_TC609_02080105_InformationMissing",
+ "Uses content length and sentence/word counts as partial missing-information checks.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao_text"])
+class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleAbnormalChar",
+ "dingo.model.rule.rule_common.RuleAbnormalHtml",
+ "dingo.model.rule.rule_common.RuleInvisibleChar",
+ "dingo.model.rule.rule_common.RuleSpecialCharacter",
+ "dingo.model.rule.rule_common.RuleWatermark",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080106",
+ "Rule_TC609_02080106_TextPurity",
+ "Combines abnormal HTML, character, invisible-content, and watermark checks.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao_text"])
+class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_common.RuleNoPunc",
+ "dingo.model.rule.rule_common.RuleWordSplit",
+ "dingo.model.rule.rule_common.RuleWordStuck",
+ "dingo.model.rule.rule_common.RuleEnterAndSpace",
+ "dingo.model.rule.rule_common.RuleEnterMore",
+ )
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080107",
+ "Rule_TC609_02080107_TextCoherence",
+ "Combines punctuation, word-boundary, and line-break fluency checks.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao_image"])
+class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite):
+ component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",)
+ _required_fields = [RequiredField.IMAGE]
+ _metric_info = _tc609_metric_info(
+ "02080201",
+ "Rule_TC609_02080201_ImageResolution",
+ "Uses image aspect-ratio validation as partial resolution coverage.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao_image"])
+class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite):
+ component_rules = ("dingo.model.rule.rule_image.RuleImageRepeat",)
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080202",
+ "Rule_TC609_02080202_ImageDuplication",
+ "Uses PHash and CNN duplicate-image detection.",
+ "covered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao_image"])
+class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite):
+ component_rules = ("dingo.model.rule.rule_image.RuleImageQuality",)
+ _required_fields = [RequiredField.IMAGE]
+ _metric_info = _tc609_metric_info(
+ "02080203",
+ "Rule_TC609_02080203_ImageSignalNoiseRatio",
+ "Uses NIMA image quality as partial evidence; it is not a true SNR metric.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao_image"])
+class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite):
+ component_rules = (
+ "dingo.model.rule.rule_image.RuleImageValid",
+ "dingo.model.rule.rule_image.RuleImageQuality",
+ )
+ _required_fields = [RequiredField.IMAGE]
+ _metric_info = _tc609_metric_info(
+ "02080204",
+ "Rule_TC609_02080204_ImageClarity",
+ "Combines image validity and NIMA quality as partial clarity coverage.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_video"])
+class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080301", "Rule_TC609_02080301_VideoResolution",
+ "Placeholder: video resolution is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_video"])
+class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080302", "Rule_TC609_02080302_VideoDuplication",
+ "Placeholder: duplicate-video detection is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_video"])
+class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080303", "Rule_TC609_02080303_VideoFrameRate",
+ "Placeholder: video FPS validation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_video"])
+class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080304", "Rule_TC609_02080304_VideoDuration",
+ "Placeholder: video duration validation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_video"])
+class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080305", "Rule_TC609_02080305_VideoClarity",
+ "Placeholder: video clarity evaluation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_video"])
+class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080306", "Rule_TC609_02080306_VideoDynamicRange",
+ "Placeholder: video dynamic-range evaluation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao_audio"])
+class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite):
+ # Existing RuleAudioDuration currently contains the SNR implementation.
+ component_rules = ("dingo.model.rule.rule_audio.RuleAudioDuration",)
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080401",
+ "Rule_TC609_02080401_AudioSignalNoiseRatio",
+ "Uses the existing Welch power-spectrum SNR implementation.",
+ "covered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_audio"])
+class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080402", "Rule_TC609_02080402_SignalDistortionRatio",
+ "Placeholder: signal distortion ratio is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_audio"])
+class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080403", "Rule_TC609_02080403_AudioSampleRate",
+ "Placeholder: sample-rate quality validation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_audio"])
+class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080404", "Rule_TC609_02080404_AudioBitDepth",
+ "Placeholder: audio bit-depth validation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_audio"])
+class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase):
+ _metric_info = _tc609_metric_info(
+ "02080405", "Rule_TC609_02080405_AudioBitRate",
+ "Placeholder: audio bit-rate validation is not implemented.", "uncovered"
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao_audio"])
+class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite):
+ # Existing RuleAudioSnrQuality currently contains the duration implementation.
+ component_rules = ("dingo.model.rule.rule_audio.RuleAudioSnrQuality",)
+ _required_fields = [RequiredField.CONTENT]
+ _metric_info = _tc609_metric_info(
+ "02080406",
+ "Rule_TC609_02080406_AudioDuration",
+ "Uses the existing WAV duration implementation.",
+ "covered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_model"])
+class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase):
+ """0301: Placeholder for content diversity."""
+
+ _metric_info = _tc609_metric_info(
+ "0301",
+ "Rule_TC609_0301_ContentDiversity",
+ "Placeholder: target-scenario distribution coverage is not implemented.",
+ "uncovered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_model"])
+class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase):
+ """0302: Placeholder for scale completeness."""
+
+ _metric_info = _tc609_metric_info(
+ "0302",
+ "Rule_TC609_0302_ScaleCompleteness",
+ "Placeholder: dataset scale versus model requirements is not implemented.",
+ "uncovered",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao_model"])
+class Rule_TC609_0303_DataTimeRange(BaseRule):
+ """Check whether creation/update time fields are within configured ranges."""
+
+ _metric_info = {
+ "category": "National Standard Data Quality Metrics",
+ "quality_dimension": "TIMELINESS",
+ "metric_name": "Rule_TC609_0303_DataTimeRange",
+ "description": (
+ "Checks whether created and updated timestamps are within configured "
+ "time ranges"
+ ),
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": "0303",
+ "coverage": "covered",
+ }
+
+ _required_fields = [RequiredField.DT]
+ dynamic_config = EvaluatorRuleArgs(
+ dt_start=None,
+ dt_end=None,
+ )
+
+ @classmethod
+ def _parse_datetime(cls, value, field_name):
+ if isinstance(value, datetime):
+ if value.tzinfo is None:
+ return value
+ return value.astimezone(timezone.utc).replace(tzinfo=None)
+
+ if isinstance(value, str):
+ text = value.strip()
+ if not text:
+ raise ValueError(f"{field_name} is empty")
+
+ iso_text = text.replace("Z", "+00:00")
+ try:
+ parsed = datetime.fromisoformat(iso_text)
+ if parsed.tzinfo is None:
+ return parsed
+ return parsed.astimezone(timezone.utc).replace(tzinfo=None)
+ except ValueError:
+ raise ValueError(
+ f"{field_name} has unsupported datetime format: {value!r}"
+ ) from None
+
+ raise ValueError(
+ f"{field_name} has unsupported datetime format: {value!r}"
+ )
+
+ @classmethod
+ def _validate_time_range(
+ cls,
+ dt_value,
+ start_value,
+ end_value,
+ ):
+ parsed_dt = cls._parse_datetime(dt_value, "time_value")
+ parsed_dt_start = (
+ cls._parse_datetime(start_value, "start_time")
+ if start_value is not None
+ else None
+ )
+ parsed_dt_end = (
+ cls._parse_datetime(end_value, "end_time")
+ if end_value is not None
+ else None
+ )
+
+ if (
+ parsed_dt_start is not None
+ and parsed_dt_end is not None
+ and parsed_dt_start > parsed_dt_end
+ ):
+ raise ValueError("time range is invalid: start is later than end")
+
+ if parsed_dt_start is not None and parsed_dt < parsed_dt_start:
+ return False, parsed_dt, parsed_dt_start, parsed_dt_end
+ if parsed_dt_end is not None and parsed_dt > parsed_dt_end:
+ return False, parsed_dt, parsed_dt_start, parsed_dt_end
+ return True, parsed_dt, parsed_dt_start, parsed_dt_end
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+
+ dt_start = getattr(cls.dynamic_config, "dt_start", None)
+ dt_end = getattr(cls.dynamic_config, "dt_end", None)
+
+ if dt_start is None and dt_end is None:
+ raise ValueError(
+ "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config"
+ )
+
+ dt_value = getattr(input_data, "dt", None)
+ if dt_value is None:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["dt is missing"]
+ return res
+
+ try:
+ is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range(
+ dt_value=dt_value,
+ start_value=dt_start,
+ end_value=dt_end,
+ )
+ except ValueError as exc:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [f"dt: {exc}"]
+ return res
+
+ if not is_valid:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ if parsed_dt_start is not None and parsed_dt < parsed_dt_start:
+ res.reason = [
+ f"dt {parsed_dt.isoformat(sep=' ')} is earlier than "
+ f"allowed start {parsed_dt_start.isoformat(sep=' ')}"
+ ]
+ else:
+ res.reason = [
+ f"dt {parsed_dt.isoformat(sep=' ')} is later than "
+ f"allowed end {parsed_dt_end.isoformat(sep=' ')}"
+ ]
+ return res
+
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao_model"])
+class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite):
+ """0304: Annotation accuracy, partially covered by label checks."""
+
+ component_rules = (
+ "dingo.model.rule.rule_image.RuleImageLabelOverlap",
+ "dingo.model.rule.rule_image.RuleImageLabelVisualization",
+ )
+ _required_fields = [RequiredField.IMAGE]
+ _metric_info = _tc609_metric_info(
+ "0304",
+ "Rule_TC609_0304_AnnotationAccuracy",
+ "Uses image annotation checks as partial evidence of annotation accuracy.",
+ "partial",
+ )
+
+
+# @Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_model"])
+class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase):
+ """0305: Placeholder for model adaptability."""
+
+ _metric_info = _tc609_metric_info(
+ "0305",
+ "Rule_TC609_0305_ModelAdaptability",
+ "Placeholder: before/after model performance comparison is not implemented.",
+ "uncovered",
+ )
diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py
new file mode 100644
index 00000000..44c9479d
--- /dev/null
+++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py
@@ -0,0 +1,468 @@
+import importlib
+import importlib.util
+import math
+import re
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io.input import Data, RequiredField
+from dingo.io.output.eval_detail import EvalDetail, QualityLabel
+from dingo.model.rule.base import BaseRule
+
+TC609_DATASET_TYPE_DESCRIPTIONS = {
+ "通识数据集": "面向普通公众，内容属于跨行业普遍适用、无需特定行业背景即可理解的通用知识",
+ "行业通识数据集": "面向特定行业，内容属于该行业从业者普遍需要掌握的基础知识、通用规范或常见实践",
+ "行业专识数据集": "面向特定行业的专业人员，内容包含需要行业专业背景才能理解或应用的专业概念、方法、技术或经验",
+}
+
+
+def _tc609_metric_info(code, name, description, coverage):
+ """Build consistent documentation metadata for TC609 rules."""
+ return {
+ "category": "SAC/TC609 High-quality Dataset Metrics",
+ "quality_dimension": code,
+ "metric_name": name,
+ "description": description,
+ "paper_title": "High-quality dataset quality evaluation specification",
+ "paper_url": "",
+ "paper_authors": "SAC/TC609",
+ "evaluation_results": "",
+ "standard_code": code,
+ "coverage": coverage,
+ }
+
+
+_text_embedding_components = {}
+
+
+def _get_text_embedding_components(model_name, device):
+ """Load and cache a transformer model used for text embeddings."""
+ missing_packages = [
+ package
+ for package in ("torch", "transformers")
+ if importlib.util.find_spec(package) is None
+ ]
+ if missing_packages:
+ raise ImportError(
+ "Text consistency evaluation requires optional packages: "
+ f"{', '.join(missing_packages)}. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ )
+
+ import torch
+ from transformers import AutoModel, AutoTokenizer
+
+ if device == -1:
+ torch_device = "cpu"
+ elif isinstance(device, int):
+ torch_device = f"cuda:{device}"
+ else:
+ torch_device = str(device)
+
+ cache_key = (model_name, torch_device)
+ if cache_key not in _text_embedding_components:
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ model = AutoModel.from_pretrained(model_name)
+ model.to(torch_device)
+ model.eval()
+ _text_embedding_components[cache_key] = (
+ tokenizer,
+ model,
+ torch_device,
+ )
+ return _text_embedding_components[cache_key]
+
+
+def _encode_texts(texts, model_name, device, batch_size, max_length):
+ """Encode texts once in batches and return normalized sentence vectors."""
+ import torch
+ import torch.nn.functional as functional
+
+ tokenizer, model, torch_device = _get_text_embedding_components(
+ model_name,
+ device,
+ )
+ embeddings = []
+ for start in range(0, len(texts), batch_size):
+ batch = texts[start:start + batch_size]
+ encoded = tokenizer(
+ batch,
+ padding=True,
+ truncation=True,
+ max_length=max_length,
+ return_tensors="pt",
+ )
+ encoded = {
+ key: value.to(torch_device)
+ for key, value in encoded.items()
+ }
+ with torch.inference_mode():
+ hidden_state = model(**encoded).last_hidden_state
+ attention_mask = encoded["attention_mask"].unsqueeze(-1)
+ pooled = (
+ (hidden_state * attention_mask).sum(dim=1)
+ / attention_mask.sum(dim=1).clamp(min=1)
+ )
+ embeddings.append(functional.normalize(pooled, p=2, dim=1).cpu())
+ return torch.cat(embeddings, dim=0)
+
+
+def calculate_text_consistency(
+ texts,
+ model_name,
+ device=-1,
+ threshold=0.5,
+ batch_size=16,
+ max_length=512,
+ consensus_keep_ratio=0.8,
+):
+ """Calculate semantic consistency for two or more texts.
+
+ Two texts are compared directly. For three or more texts, every text is
+ compared with a robust semantic center so the calculation remains linear
+ in the number of texts rather than evaluating all text pairs.
+ """
+ if (
+ not isinstance(texts, list)
+ or len(texts) < 2
+ or any(not isinstance(text, str) or not text.strip() for text in texts)
+ ):
+ raise ValueError(
+ "calculate_text_consistency requires at least two non-empty texts"
+ )
+ if (
+ isinstance(threshold, bool)
+ or not isinstance(threshold, (int, float))
+ or not 0 <= threshold <= 1
+ ):
+ raise ValueError("threshold must be in [0, 1]")
+ if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0:
+ raise ValueError("batch_size must be a positive integer")
+ if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length <= 0:
+ raise ValueError("max_length must be a positive integer")
+ if not 0 < consensus_keep_ratio <= 1:
+ raise ValueError("consensus_keep_ratio must be in (0, 1]")
+
+ import torch
+ import torch.nn.functional as functional
+
+ normalized_texts = [text.strip() for text in texts]
+ embeddings = _encode_texts(
+ normalized_texts,
+ model_name,
+ device,
+ batch_size,
+ max_length,
+ )
+
+ if len(normalized_texts) == 2:
+ score = float(torch.sum(embeddings[0] * embeddings[1]).item())
+ item_scores = [score, score]
+ else:
+ initial_center = functional.normalize(
+ embeddings.mean(dim=0),
+ p=2,
+ dim=0,
+ )
+ initial_scores = embeddings @ initial_center
+ keep_count = max(
+ 2,
+ math.ceil(len(normalized_texts) * consensus_keep_ratio),
+ )
+ keep_indexes = torch.topk(initial_scores, keep_count).indices
+ robust_center = functional.normalize(
+ embeddings[keep_indexes].mean(dim=0),
+ p=2,
+ dim=0,
+ )
+ similarities = embeddings @ robust_center
+ score = float(torch.min(similarities).item())
+ item_scores = [float(value) for value in similarities.tolist()]
+
+ score = min(1.0, max(0.0, score))
+ item_scores = [
+ min(1.0, max(0.0, value))
+ for value in item_scores
+ ]
+ return {
+ "score": score,
+ "is_consistent": score >= threshold,
+ "item_scores": item_scores,
+ "outlier_indexes": [
+ index
+ for index, item_score in enumerate(item_scores)
+ if item_score < threshold
+ ],
+ }
+
+
+class Rule_TC609_Composite(BaseRule):
+ """Base class for a TC609 metric composed from existing Dingo rules."""
+
+ component_rules = ()
+ composition_mode = "all"
+
+ @classmethod
+ def _resolve_rule(cls, dotted_path):
+ module_name, class_name = dotted_path.rsplit(".", 1)
+ module = importlib.import_module(module_name)
+ return getattr(module, class_name)
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ reasons = []
+ scores = []
+ passed_count = 0
+ for dotted_path in cls.component_rules:
+ rule = cls._resolve_rule(dotted_path)
+ try:
+ component_res = rule.eval(input_data)
+ except (ImportError, ModuleNotFoundError):
+ raise
+ except ValueError:
+ # Invalid evaluator configuration must stop the composite
+ # instead of being converted into a data-quality finding.
+ raise
+ except Exception as exc:
+ reasons.append(f"{rule.__name__}: {type(exc).__name__}: {exc}")
+ continue
+ if component_res.score is not None:
+ scores.append(component_res.score)
+ if component_res.status:
+ component_reasons = component_res.reason or ["quality check failed"]
+ reasons.extend(
+ f"{rule.__name__}: {reason}" for reason in component_reasons
+ )
+ else:
+ passed_count += 1
+
+ if scores:
+ res.score = sum(scores) / len(scores)
+ if cls.composition_mode == "any":
+ res.status = passed_count == 0
+ else:
+ res.status = passed_count != len(cls.component_rules)
+ if res.status:
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = reasons
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ return res
+
+
+class _TC609PlaceholderBase(BaseRule):
+ """Base class for registered TC609 metrics that are not implemented yet."""
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ raise NotImplementedError(
+ f"{cls.__name__} is a TC609 placeholder and is not implemented yet"
+ )
+
+
+class Rule_TC609_01_DocCompleteness(BaseRule):
+ """Shared logic for dataset documentation completeness checks."""
+
+ _required_fields = [RequiredField.CONTENT]
+ _default_threshold = 0.8
+ _default_semantic_threshold = 0.5
+ _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli"
+ _default_device = -1
+ _default_dimension_name = "Dataset documentation"
+ _default_aspect_keywords = {}
+ dynamic_config = EvaluatorRuleArgs(
+ threshold=_default_threshold,
+ semantic_threshold=_default_semantic_threshold,
+ model=_default_model,
+ device=_default_device,
+ dimension_name=_default_dimension_name,
+ aspect_keywords=_default_aspect_keywords,
+ )
+ _model_name = None
+ _model_device = None
+ _classifier = None
+
+ @classmethod
+ def _get_classifier(cls, model_name, device):
+ if (
+ cls._model_name == model_name
+ and cls._model_device == device
+ and cls._classifier is not None
+ ):
+ return cls._classifier
+
+ required_packages = ("torch", "transformers")
+ missing_packages = [
+ package
+ for package in required_packages
+ if importlib.util.find_spec(package) is None
+ ]
+ if missing_packages:
+ raise ImportError(
+ f"{cls.__name__} requires optional packages: "
+ f"{', '.join(missing_packages)}. "
+ 'Install them with: pip install "dingo-python[hhem]"'
+ )
+
+ from transformers import pipeline
+
+ cls._classifier = pipeline(
+ "zero-shot-classification",
+ model=model_name,
+ device=device,
+ )
+ cls._model_name = model_name
+ cls._model_device = device
+ return cls._classifier
+
+ @classmethod
+ def _calculate_aspect_score(cls, content, aspect_text, model_name, device):
+ classifier = cls._get_classifier(model_name, device)
+ result = classifier(
+ content,
+ candidate_labels=[aspect_text],
+ hypothesis_template="这段文本包含{}相关说明。",
+ multi_label=True,
+ truncation=True,
+ )
+ labels = result.get("labels", [])
+ scores = result.get("scores", [])
+ if not labels or not scores or labels[0] != aspect_text:
+ raise RuntimeError("Zero-shot classifier returned an invalid result")
+ score = float(scores[0])
+ if not math.isfinite(score) or not 0.0 <= score <= 1.0:
+ raise RuntimeError(
+ f"Zero-shot classifier returned an invalid score: {score}"
+ )
+ return score
+
+ @classmethod
+ def _match_aspects(
+ cls,
+ content,
+ normalized_content,
+ aspect_keywords,
+ model_name,
+ device,
+ semantic_threshold,
+ ):
+ matched = {}
+ missing = []
+ for aspect_name, keywords in aspect_keywords.items():
+ aspect_text = (
+ f"{aspect_name}（关键词示例：{'、'.join(keywords)}）"
+ if keywords
+ else aspect_name
+ )
+ score = cls._calculate_aspect_score(
+ content, aspect_text, model_name, device
+ )
+ evidence_keyword = next(
+ (
+ keyword
+ for keyword in keywords
+ if keyword.lower() in normalized_content
+ ),
+ None,
+ )
+ if score >= semantic_threshold:
+ matched[aspect_name] = {
+ "score": round(score, 4),
+ "keyword": evidence_keyword,
+ }
+ else:
+ missing.append(aspect_name)
+ return matched, missing
+
+ @classmethod
+ def eval(cls, input_data: Data) -> EvalDetail:
+ res = EvalDetail(metric=cls.__name__)
+ content = getattr(input_data, "content", None)
+ if not isinstance(content, str) or not content.strip():
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = ["Content is missing or empty"]
+ res.score = 0.0
+ return res
+
+ threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold)
+ if threshold is None or not 0 < threshold <= 1:
+ raise ValueError(
+ f"{cls.__name__} dynamic_config.threshold must be in (0, 1]"
+ )
+
+ semantic_threshold = getattr(
+ cls.dynamic_config,
+ "semantic_threshold",
+ cls._default_semantic_threshold,
+ )
+ if semantic_threshold is None or not 0 < semantic_threshold <= 1:
+ raise ValueError(
+ f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]"
+ )
+
+ aspect_keywords = getattr(
+ cls.dynamic_config,
+ "aspect_keywords",
+ cls._default_aspect_keywords,
+ )
+ if not isinstance(aspect_keywords, dict) or not aspect_keywords:
+ raise ValueError(
+ f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict"
+ )
+
+ model_name = getattr(cls.dynamic_config, "model", cls._default_model)
+ if not isinstance(model_name, str) or not model_name.strip():
+ raise ValueError(
+ f"{cls.__name__} dynamic_config.model must be a non-empty string"
+ )
+
+ device = getattr(cls.dynamic_config, "device", cls._default_device)
+
+ dimension_name = getattr(
+ cls.dynamic_config, "dimension_name", cls._default_dimension_name
+ )
+
+ normalized_content = re.sub(r"\s+", "", content).lower()
+ matched, missing = cls._match_aspects(
+ content=content,
+ normalized_content=normalized_content,
+ aspect_keywords=aspect_keywords,
+ model_name=model_name,
+ device=device,
+ semantic_threshold=semantic_threshold,
+ )
+
+ total = len(aspect_keywords)
+ matched_count = len(matched)
+ score = matched_count / total if total else 0.0
+ res.score = round(score, 4)
+ matched_desc = (
+ ", ".join(
+ (
+ f"{aspect}(semantic_score={detail['score']:.4f}, "
+ f"keyword_hit={detail['keyword'] or 'None'})"
+ )
+ for aspect, detail in matched.items()
+ )
+ if matched
+ else "None"
+ )
+ missing_desc = ", ".join(missing) if missing else "None"
+
+ if score < threshold:
+ res.status = True
+ res.label = [f"{cls.metric_type}.{cls.__name__}"]
+ res.reason = [
+ f"{dimension_name} completeness score {score:.4f} is below "
+ f"threshold {threshold:.4f} (A/B={matched_count}/{total}); "
+ f"matched: {matched_desc}; missing: {missing_desc}"
+ ]
+ else:
+ res.label = [QualityLabel.QUALITY_GOOD]
+ res.reason = [
+ f"{dimension_name} completeness score {score:.4f} meets "
+ f"threshold {threshold:.4f} (A/B={matched_count}/{total}); "
+ f"matched: {matched_desc}; missing: {missing_desc}"
+ ]
+ return res
diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py
index 4ab70635..75d3d0db 100644
--- a/dingo/model/rule/rule_common.py
+++ b/dingo/model/rule/rule_common.py
@@ -2108,7 +2108,12 @@ class RuleUnsafeWords(BaseRule):
"paper_authors": "Together Computer, 2023",
"evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md"
}
- dynamic_config = EvaluatorRuleArgs(refer_path=[])
+ dynamic_config = EvaluatorRuleArgs(
+ key_list=[],
+ refer_path=[],
+ )
+ _unsafe_words_list = None
+ _unsafe_words_automaton = None
_required_fields = [RequiredField.CONTENT]
@@ -2121,17 +2126,32 @@ def eval(cls, input_data: Data) -> EvalDetail:
res = EvalDetail(metric=cls.__name__)
content = input_data.content
- key_list = cls.dynamic_config.key_list
- if key_list is None:
- key_list = get_unsafe_words(cls.dynamic_config.refer_path)
-
- A = ahocorasick.Automaton()
- for index, key in enumerate(key_list):
- A.add_word(key, (index, key))
- A.make_automaton()
+ if cls._unsafe_words_list is None:
+ unsafe_words_list = []
+ unsafe_words_list.extend(cls.dynamic_config.key_list or [])
+ unsafe_words_list.extend(get_unsafe_words(cls.dynamic_config.refer_path or []))
+ unsafe_words_list = list(dict.fromkeys(
+ word for word in unsafe_words_list
+ if isinstance(word, str) and word
+ ))
+ if not unsafe_words_list:
+ raise ValueError(
+ "RuleUnsafeWords requires unsafe words from dynamic_config."
+ "key_list or files configured in dynamic_config.refer_path"
+ )
+ cls._unsafe_words_list = unsafe_words_list
+
+ if cls._unsafe_words_automaton is None:
+ automaton = ahocorasick.Automaton()
+ for index, key in enumerate(cls._unsafe_words_list):
+ automaton.add_word(key, (index, key))
+ automaton.make_automaton()
+ cls._unsafe_words_automaton = automaton
matches = []
- for end_index, (index, keyword) in A.iter(content):
+ for end_index, (index, keyword) in cls._unsafe_words_automaton.iter(
+ content
+ ):
start_index = end_index - len(keyword) + 1
# 检查单词边界
@@ -2149,6 +2169,10 @@ def eval(cls, input_data: Data) -> EvalDetail:
@classmethod
def _is_whole_word(cls, text: str, start: int, end: int) -> bool:
"""检查匹配是否是一个完整的单词"""
+ keyword = text[start:end + 1]
+ if not keyword.isascii():
+ return True
+
# 检查左侧边界
if start > 0 and text[start - 1].isalnum():
return False
@@ -2274,6 +2298,11 @@ class RuleWatermark(BaseRule):
@classmethod
def eval(cls, input_data: Data) -> EvalDetail:
+ if not cls.dynamic_config.key_list:
+ raise ValueError(
+ "RuleWatermark requires non-empty dynamic_config.key_list"
+ )
+
res = EvalDetail(metric=cls.__name__)
matches = re.findall("|".join(cls.dynamic_config.key_list), input_data.content)
if matches:
diff --git a/dingo/model/rule/rule_hallucination_hhem.py b/dingo/model/rule/rule_hallucination_hhem.py
index bf2a759d..229b0abb 100644
--- a/dingo/model/rule/rule_hallucination_hhem.py
+++ b/dingo/model/rule/rule_hallucination_hhem.py
@@ -1,17 +1,34 @@
"""
-HHEM-2.1-Open Hallucination Detection Rule
-
-This module provides integration with Vectara's HHEM-2.1-Open model as a rule-based
-hallucination detection tool for efficient local inference without API costs.
-
-Key advantages of HHEM-2.1-Open:
-- Superior performance compared to GPT-3.5/GPT-4 on benchmarks
-- Local inference with <600MB RAM usage
-- Fast processing (~1.5s for 2k tokens on modern CPU)
-- No API costs or rate limits
+MiniCheck Hallucination Detection Rule
+
+This module provides local, API-free hallucination (ungrounded-claim) detection
+for RAG-style data by checking whether a response is supported by its context.
+
+Model: `lytang/MiniCheck-Flan-T5-Large` (EMNLP 2024, arXiv:2404.10774).
+
+Why MiniCheck instead of Vectara HHEM-2.1-Open (the original backing model):
+- Stronger grounding accuracy: MiniCheck-Flan-T5-Large scores 75.0 vs HHEM's
+ 71.8 on the LLM-AggreFact benchmark (llm-aggrefact.github.io).
+- Robust across transformers versions: MiniCheck is a *standard*
+ `T5ForConditionalGeneration` (config `model_type: t5`, no `auto_map` /
+ `trust_remote_code`). HHEM shipped custom remote code that breaks on
+ transformers >= 4.49 (AttributeError: `all_tied_weights_keys`), which forced
+ a `<4.49` pin. MiniCheck removes that constraint.
+- Still efficient and CPU-friendly (0.8B params), no API costs or rate limits.
+
+The class name is kept as `RuleHallucinationHHEM` for backward compatibility
+(the registered rule id `QUALITY_BAD_HALLUCINATION` and existing configs are
+unaffected).
+
+Inference is replicated faithfully from the official MiniCheck source
+(Liyan06/MiniCheck): input `"predict: " + doc + + claim`, a single-step
+decoder forward, then a 2-way softmax over label token ids [3, 209] where
+index 1 is P(supported). Long documents are split into word chunks and the
+support probability is aggregated by max.
"""
import json
+from threading import Lock
from typing import List
from dingo.config.input_args import EvaluatorRuleArgs
@@ -25,13 +42,17 @@
@Model.rule_register("QUALITY_BAD_HALLUCINATION", ["hallucination", "rag"])
class RuleHallucinationHHEM(BaseRule):
"""
- HHEM-2.1-Open hallucination detection rule.
+ MiniCheck-based hallucination detection rule.
+
+ Detects ungrounded claims by checking whether the response (content) is
+ supported by the provided context, using `lytang/MiniCheck-Flan-T5-Large`:
+ - Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8)
+ - Standard T5 model -> no transformers version pin, no remote code
+ - Local inference, CPU-friendly, no API costs or rate limits
- Provides efficient local hallucination detection with:
- - Superior performance than GPT models on benchmarks
- - Low resource usage (<600MB RAM)
- - Fast inference (~1.5s for 2k tokens on modern CPU)
- - No API costs or rate limits
+ Note: the class is still named `RuleHallucinationHHEM` for backward
+ compatibility; the underlying model was upgraded from Vectara HHEM-2.1-Open
+ to MiniCheck.
"""
# Metadata for documentation generation
@@ -39,42 +60,131 @@ class RuleHallucinationHHEM(BaseRule):
"category": "SFT Data Assessment Metrics",
"quality_dimension": "HALLUCINATION",
"metric_name": "RuleHallucinationHHEM",
- "description": "Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and context",
- "paper_title": "HHEM-2.1-Open",
- "paper_url": "https://huggingface.co/vectara/hallucination_evaluation_model",
- "paper_authors": "Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch"
+ "description": "Uses the MiniCheck-Flan-T5-Large model for local hallucination "
+ "detection by checking whether the response is grounded in the context",
+ "paper_title": "MiniCheck: Efficient Fact-Checking of LLMs on Grounding Documents",
+ "paper_url": "https://arxiv.org/abs/2404.10774",
+ "paper_authors": "Liyan Tang, Philippe Laban, Greg Durrett"
}
+ # CONTENT = the response/claim to verify; CONTEXT = the grounding document.
+ # These are exactly the two inputs MiniCheck needs (claim vs. document), so
+ # they remain the most fitting required fields.
_required_fields = [RequiredField.CONTENT, RequiredField.CONTEXT]
dynamic_config = EvaluatorRuleArgs(threshold=0.5)
model = None
+ tokenizer = None
+ _load_lock = Lock()
+ _model_repo_id = "lytang/MiniCheck-Flan-T5-Large"
+ # MiniCheck flan-t5 inference config (from Liyan06/MiniCheck)
+ _chunk_size = 500 # words per document chunk before max-aggregation
+ _max_input_length = 2048 # tokenizer truncation length
@classmethod
def load_model(cls):
- """Load HHEM-2.1-Open model"""
+ """Load the MiniCheck-Flan-T5-Large model and tokenizer."""
if cls.model is None:
- try:
- from transformers import AutoModelForSequenceClassification
-
- log.info("Loading HHEM-2.1-Open model...")
- cls.model = AutoModelForSequenceClassification.from_pretrained(
- 'vectara/hallucination_evaluation_model',
- trust_remote_code=True
- )
- log.info("✅ HHEM-2.1-Open model loaded successfully")
+ with cls._load_lock:
+ if cls.model is not None:
+ return
+ try:
+ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+ log.info("Loading MiniCheck-Flan-T5-Large model...")
+ # MiniCheck is a standard T5 model: no trust_remote_code
+ # needed, and it loads on any modern transformers version.
+ try:
+ # Prefer offline / cached load first
+ cls.model = AutoModelForSeq2SeqLM.from_pretrained(
+ cls._model_repo_id, local_files_only=True,
+ )
+ cls.tokenizer = AutoTokenizer.from_pretrained(
+ cls._model_repo_id, local_files_only=True,
+ )
+ except Exception:
+ # Fall back to downloading from the Hub
+ cls.model = AutoModelForSeq2SeqLM.from_pretrained(
+ cls._model_repo_id,
+ )
+ cls.tokenizer = AutoTokenizer.from_pretrained(
+ cls._model_repo_id,
+ )
+ cls.model.eval()
+ log.info("✅ MiniCheck-Flan-T5-Large model loaded successfully")
+
+ except ImportError:
+ raise ImportError(
+ "transformers is required for the MiniCheck model. "
+ "Install with: pip install transformers torch sentencepiece"
+ )
+ except Exception as e:
+ raise RuntimeError(
+ "Failed to load MiniCheck model. "
+ "The first run requires network access to download "
+ f"'{cls._model_repo_id}', or a populated Hugging Face cache. "
+ f"Original error: {e}"
+ ) from e
+
+ @staticmethod
+ def _chunk_document(document: str, chunk_size: int) -> List[str]:
+ """Split a document into consecutive chunks of ~chunk_size words.
+
+ Lightweight, dependency-free approximation of MiniCheck's sentence-based
+ chunking (the official code uses nltk sent_tokenize). RAG contexts
+ usually fit in a single chunk, so this rarely changes behavior; long
+ documents are still split so no content is silently truncated.
+ """
+ text = document.strip()
+ if not text:
+ return []
+ words = text.split()
+ if len(words) <= chunk_size:
+ return [text]
+ return [" ".join(words[i:i + chunk_size])
+ for i in range(0, len(words), chunk_size)]
- except ImportError:
- raise ImportError(
- "transformers library is required for HHEM model. "
- "Install with: pip install transformers"
- )
- except Exception as e:
- raise RuntimeError(f"Failed to load HHEM model: {e}")
+ @classmethod
+ def _support_prob(cls, document: str, claim: str) -> float:
+ """Probability that `claim` is supported by `document` (0=unsupported, 1=supported).
+
+ Faithful replication of the official MiniCheck flan-t5 inference:
+ input = "predict: " + doc_chunk + tokenizer.eos_token + claim
+ forward = model(input_ids, attention_mask, decoder_input_ids=zeros(B,1))
+ logits = outputs.logits.squeeze(1)
+ probs = softmax(logits[:, [3, 209]]) # 3=no support, 209=support
+ support = probs[:, 1]
+ Aggregated by max over document chunks.
+ """
+ import torch
+
+ chunks = cls._chunk_document(document, cls._chunk_size) or [""]
+ texts = ["predict: " + cls.tokenizer.eos_token.join([chunk, claim])
+ for chunk in chunks]
+ enc = cls.tokenizer(
+ texts,
+ max_length=cls._max_input_length,
+ truncation=True,
+ padding=True,
+ return_tensors="pt",
+ )
+ decoder_input_ids = torch.zeros(
+ (enc["input_ids"].size(0), 1), dtype=torch.long)
+ with torch.no_grad():
+ logits = cls.model(
+ input_ids=enc["input_ids"],
+ attention_mask=enc["attention_mask"],
+ decoder_input_ids=decoder_input_ids,
+ ).logits.squeeze(1)
+ # Label token ids from the official MiniCheck code: 3=no support, 209=support
+ label_probs = torch.nn.functional.softmax(
+ logits[:, torch.tensor([3, 209])], dim=-1)
+ support_probs = label_probs[:, 1]
+ return float(support_probs.max().item())
@classmethod
def eval(cls, input_data: Data) -> EvalDetail:
"""
- Evaluate hallucination using HHEM-2.1-Open model.
+ Evaluate hallucination using the MiniCheck-Flan-T5-Large model.
Args:
input_data: Data object containing content and context
@@ -93,9 +203,9 @@ def eval(cls, input_data: Data) -> EvalDetail:
result.status = True
# result.type = cls.metric_type
# result.name = "MISSING_CONTEXT"
- # result.reason = ["Context is required for HHEM hallucination detection but was not provided"]
+ # result.reason = ["Context is required for hallucination detection but was not provided"]
result.label = [f"{cls.metric_type}.MISSING_CONTEXT"]
- result.reason = ["Context is required for HHEM hallucination detection but was not provided"]
+ result.reason = ["Context is required for hallucination detection but was not provided"]
return result
else:
contexts = input_data.context
@@ -117,20 +227,15 @@ def eval(cls, input_data: Data) -> EvalDetail:
response = input_data.content
- # Create premise-hypothesis pairs for HHEM evaluation
- # Format: (premise, hypothesis) where premise=context, hypothesis=response
- pairs = [(context, response) for context in context_list]
-
try:
- # Use HHEM model's official predict() method
- # This returns consistency scores (0=hallucinated, 1=consistent)
- scores = cls.model.predict(pairs)
-
- # Convert to list if tensor
- consistency_scores = scores.tolist() if hasattr(scores, 'tolist') else list(scores)
-
- # HHEM returns consistency scores (0=hallucinated, 1=consistent)
- # We convert to hallucination scores (1=hallucinated, 0=consistent)
+ # Score each context with MiniCheck: P(response supported by context).
+ # support prob in [0,1], 1 = fully grounded / consistent.
+ consistency_scores = [
+ cls._support_prob(context, response) for context in context_list
+ ]
+
+ # Convert support probabilities to hallucination scores
+ # (1 = hallucinated / ungrounded, 0 = consistent)
hallucination_scores = [1.0 - score for score in consistency_scores]
# Average hallucination score across all contexts
@@ -138,7 +243,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
# Create result
result = EvalDetail(metric=cls.__name__)
- # result.score = avg_hallucination_score
+ result.score = avg_hallucination_score
# Determine if hallucination detected based on threshold
if avg_hallucination_score > cls.dynamic_config.threshold:
@@ -149,7 +254,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
# Generate detailed analysis
analysis_parts = [
- f"🔍 HHEM-2.1-Open 幻觉检测分析",
+ "🔍 MiniCheck 幻觉检测分析",
f"📊 平均幻觉分数: {avg_hallucination_score:.3f} (阈值: {cls.dynamic_config.threshold})",
f"📝 评估上下文数量: {len(context_list)}"
]
@@ -184,7 +289,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
f"🚨 结论: 检测到幻觉 (分数 {avg_hallucination_score:.3f} > 阈值 {cls.dynamic_config.threshold})",
" 回答与提供的上下文存在显著矛盾",
"",
- "💡 模型信息: 使用 Vectara HHEM-2.1-Open (本地推理)"
+ "💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理)"
])
# result.reason = ["\n".join(analysis_parts)]
@@ -197,11 +302,11 @@ def eval(cls, input_data: Data) -> EvalDetail:
# Generate analysis for non-hallucination case
analysis = (
- f"✅ HHEM-2.1-Open 幻觉检测分析\n"
+ f"✅ MiniCheck 幻觉检测分析\n"
f"📊 平均幻觉分数: {avg_hallucination_score:.3f} (阈值: {cls.dynamic_config.threshold})\n"
f"📝 评估上下文数量: {len(context_list)}\n"
f"🎉 结论: 未检测到幻觉，回答与上下文基本一致\n"
- f"💡 模型信息: 使用 Vectara HHEM-2.1-Open (本地推理)"
+ f"💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理)"
)
# result.reason = [analysis]
result.reason = [analysis]
@@ -213,10 +318,10 @@ def eval(cls, input_data: Data) -> EvalDetail:
result = EvalDetail(metric=cls.__name__)
result.status = True
# result.type = cls.metric_type
- # result.name = "HHEM_ERROR"
- # result.reason = [f"HHEM model inference failed: {str(e)}"]
- result.label = [f"{cls.metric_type}.HHEM_ERROR"]
- result.reason = [f"HHEM model inference failed: {str(e)}"]
+ # result.name = "MINICHECK_ERROR"
+ # result.reason = [f"MiniCheck model inference failed: {str(e)}"]
+ result.label = [f"{cls.metric_type}.MINICHECK_ERROR"]
+ result.reason = [f"MiniCheck model inference failed: {str(e)}"]
return result
@classmethod
@@ -236,7 +341,7 @@ def evaluate_with_detailed_output(cls, input_data: Data) -> dict:
# "assessment_type": result.type,
# "assessment_name": result.name,
"analysis": result.reason[0] if result.reason else "",
- "model_info": "HHEM-2.1-Open (Vectara)"
+ "model_info": "MiniCheck-Flan-T5-Large"
}
@classmethod
diff --git a/dingo/retrieval/backends/agentic.py b/dingo/retrieval/backends/agentic.py
index c110bddf..515eda00 100644
--- a/dingo/retrieval/backends/agentic.py
+++ b/dingo/retrieval/backends/agentic.py
@@ -7,8 +7,8 @@
POST {api_url}/v1/search -- direct connection to the Go service, no auth.
Public (when api_token is set):
- POST {api_url}/agentic-search -- SciVerse public gateway with Bearer auth.
- POST {api_url}/meta-search -- SciVerse metadata search gateway.
+ POST {api_url}/agentic-search -- Sciverse public gateway with Bearer auth.
+ POST {api_url}/meta-search -- Sciverse metadata search gateway.
Rate limit defaults to 1 RPS.
Examples:
@@ -291,16 +291,33 @@ class MetaSearchClient(AgenticSearchClient):
def __init__(
self,
*args,
- search_type: str = "paper",
+ search_type: str = "all",
sort_by: str | None = None,
freshness_boost: str | None = None,
filters: list[dict[str, Any]] | dict[str, Any] | None = None,
**kwargs: Any,
) -> None:
- self.search_type = search_type
+ api_url = kwargs.get("api_url")
+ if api_url:
+ normalized_url = str(api_url).rstrip("/")
+ if normalized_url.endswith("/meta-search"):
+ kwargs["api_url"] = normalized_url[: -len("/meta-search")]
+ self.search_type = (search_type or "all").strip().lower()
+ if self.search_type not in {"all", "paper", "ebook"}:
+ raise ValueError("meta_search search_type must be 'all', 'paper', or 'ebook'")
self.sort_by = sort_by
self.freshness_boost = freshness_boost
self.filters = self._normalize_filters(filters)
+ if self.search_type != "all" and not any(
+ item.get("field") == "metadata_type" for item in self.filters
+ ):
+ self.filters.append(
+ {
+ "field": "metadata_type",
+ "operator": "FILTER_OP_EQ",
+ "value": self.search_type,
+ }
+ )
super().__init__(*args, **kwargs)
if self._public_mode:
self.name = "sciverse-meta-search-api"
diff --git a/dingo/retrieval/backends/openalex.py b/dingo/retrieval/backends/openalex.py
index b32b7907..52f21627 100644
--- a/dingo/retrieval/backends/openalex.py
+++ b/dingo/retrieval/backends/openalex.py
@@ -50,7 +50,8 @@
_DEFAULT_SELECT = (
"id,doi,display_name,title,abstract_inverted_index,publication_year,"
- "relevance_score,cited_by_count"
+ "relevance_score,cited_by_count,type,language,authorships,keywords,"
+ "primary_location,open_access"
)
@@ -70,6 +71,8 @@ def __init__(
**_kwargs: Any,
) -> None:
self.base_url = api_url.rstrip("/")
+ if self.base_url.endswith("/works"):
+ self.base_url = self.base_url[: -len("/works")]
self.api_key = api_token or os.environ.get("OPENALEX_API_KEY")
self.timeout = timeout
self.search_type = (search_type or "search").strip().lower()
diff --git a/docs/config.md b/docs/config.md
index 2e6104f2..ba7ba129 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -30,7 +30,7 @@
| Parameter | Type | Default | Required | Description |
|-----------|------|---------|----------|-------------|
| source | str | "hugging_face" | Yes | 数据源类型，可选值：['hugging_face', 'local'] |
-| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'plaintext', 'listjson', 'csv', 'parquet', 'mineru', 'mineru_v2'] |
+| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'plaintext', 'listjson', 'csv', 'parquet', 'md', 'mineru', 'mineru_v2'] |
| field | object | - | Yes | 字段映射配置 |
| hf_config | object | - | No | HuggingFace 特定配置 |
| mineru_config | object | - | No | MinerU 格式特定配置（仅 mineru / mineru_v2 格式使用） |
@@ -46,6 +46,7 @@ MinerU 格式特定配置，用于过滤 block 类型：
MinerU 支持的 block 类型包括：`text`, `title`, `image`, `table`, `equation`, `code`, `list`, `header`, `page_footer`, `page_footnote`, `chart` 等。
**格式说明：**
+- `md`：读取单个 `.md` 文件或目录下全部 `.md` 文件；每个文件构造成一条 `Data`，包含 `id`（文件名）和 `content`（全文字符串）
- `mineru`：对应 MinerU 的 `content_list.json`，顶层为 block 数组
- `mineru_v2`：对应 MinerU 的 `content_list_v2.json`，顶层为页面数组，每页包含 block 数组
@@ -99,6 +100,55 @@ HuggingFace 特定配置：
| all_labels | bool | false | No | 是否保存所有标签 |
| raw | bool | false | No | 是否保存原始数据 |
+### LLM Token 使用量输出
+
+当 LLM 服务返回 token usage 时，Dingo 会在对应的 `EvalDetail` 中写入 `usage` 字段，并在 `summary.json` 中按字段组合和 evaluator 汇总到 `token_usage`。该统计来自模型服务商返回的 `usage`，不会本地估算；如果兼容 API 不返回 usage，则对应字段为空。
+
+单条结果示例：
+
+```json
+{
+ "metric": "LLMTextQualityV5",
+ "status": false,
+ "label": ["QUALITY_GOOD"],
+ "reason": ["pass"],
+ "usage": {
+ "prompt_tokens": 812,
+ "completion_tokens": 96,
+ "total_tokens": 908,
+ "reasoning_tokens": null,
+ "cached_tokens": null,
+ "model": "gpt-4o-mini",
+ "provider": "openai",
+ "calls": 1,
+ "source": "provider"
+ }
+}
+```
+
+汇总结果示例：
+
+```json
+{
+ "token_usage": {
+ "content": {
+ "LLMTextQualityV5": {
+ "prompt_tokens": 81200,
+ "completion_tokens": 9600,
+ "total_tokens": 90800,
+ "reasoning_tokens": 0,
+ "cached_tokens": 12000,
+ "calls": 100,
+ "records": 100,
+ "models": {"gpt-4o-mini": 100},
+ "providers": {"openai": 100},
+ "sources": {"provider": 100}
+ }
+ }
+ }
+}
+```
+
### Evaluator 配置 (evaluator)
评估器相关配置：
diff --git a/docs/dataset/sql.md b/docs/dataset/sql.md
index e03c8e8a..09a81193 100644
--- a/docs/dataset/sql.md
+++ b/docs/dataset/sql.md
@@ -10,6 +10,8 @@
- ✅ **多数据库支持**: 支持 PostgreSQL, MySQL, SQLite 等主流数据库
- ✅ **内存友好**: 逐行处理数据，适合处理大规模数据集
- ✅ **灵活查询**: 支持任意 SQL 查询语句（SELECT、JOIN、WHERE 等）
+- ✅ **连接更稳健**: 默认启用 `pool_pre_ping=True`，MySQL/MariaDB 默认 `pool_recycle=1800`
+- ✅ **URL 更安全**: 使用 SQLAlchemy `URL.create(...)` 构建连接，避免特殊字符导致的 URL 解析问题
## 依赖安装
@@ -168,6 +170,7 @@ for data in dataset.get_data():
| `port` | str | 否 | 数据库端口 |
| `database` | str | 是 | 数据库名称或文件路径（SQLite） |
| `connect_args` | str | 否 | 连接参数，如 `?charset=utf8mb4`、`?sslmode=require` 等 |
+| `engine_args` | str | 否 | SQLAlchemy 引擎参数，格式同 `connect_args`，如 `pool_recycle=3600&pool_pre_ping=true` |
*注：对于 SQLite，`username` 和 `host` 不是必填项；对于其他数据库，这些是必填项。
@@ -260,9 +263,53 @@ sql_config = DatasetSqlArgs(
)
```
+### 4. 使用引擎参数（连接池/探活）
+
+对于需要控制 SQLAlchemy 引擎行为的场景，可以使用 `engine_args`（字符串格式，支持 `k=v&k2=v2`）：
+
+```python
+# 覆盖默认连接池回收时间，显式开启连接探活
+sql_config = DatasetSqlArgs(
+ dialect="mysql",
+ driver="pymysql",
+ username="root",
+ password="password",
+ host="localhost",
+ port="3306",
+ database="test_db",
+ connect_args="charset=utf8mb4",
+ engine_args="pool_recycle=3600&pool_pre_ping=true"
+)
+
+# 支持 ? 前缀写法
+sql_config = DatasetSqlArgs(
+ dialect="postgresql",
+ driver="psycopg2",
+ username="myuser",
+ password="mypassword",
+ host="localhost",
+ port="5432",
+ database="mydb",
+ engine_args="?pool_size=8&max_overflow=16&pool_timeout=30"
+)
+```
+
+当前 `engine_args` 支持参数：
+
+- `pool_pre_ping`（bool，默认 `true`，仅支持 `true/false`）
+- `pool_recycle`（int，MySQL/MariaDB 默认 `1800`）
+- `pool_size`（int）
+- `max_overflow`（int）
+- `pool_timeout`（int）
+
+> ⚠️ Warning: 若在 `engine_args` 里显式设置同名参数，会覆盖默认值。
+
## 工作原理
1. **连接创建**: `SqlDataSource` 使用 SQLAlchemy 创建数据库引擎
+ - 使用 `URL.create(...)` 安全构建 URL（兼容密码中的 `@/#/:` 等特殊字符）
+ - 默认开启 `pool_pre_ping=True`；MySQL/MariaDB 默认 `pool_recycle=1800`
+ - 可通过 `engine_args` 覆盖默认引擎参数
2. **流式查询**: 使用 `connection.execution_options(stream_results=True)` 启用服务器端游标
3. **逐行迭代**: SQLAlchemy 自动处理数据分页，逐行返回结果
4. **数据转换**: 每行数据通过 `jsonl` 转换器转换为 `Data` 对象
@@ -326,8 +373,20 @@ pip install pymysql # MySQL
- 检查数据库服务是否运行
- 检查网络连接和防火墙设置
- 验证主机地址和端口号
+- 在 `connect_args` 中显式增加数据库驱动超时参数（如 `read_timeout`、`write_timeout`）
+
+### 问题4: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query')
+
+**现象**: 查询执行中连接被重置（例如 `Connection reset by peer`）。
+
+**解决**:
+- 检查数据库和网络链路（代理/LB/NAT）是否会中断长连接
+- 将大查询拆分为分页或分片查询，减少单次长事务时长
+- 检查服务端超时参数（如 `wait_timeout`、`net_read_timeout`）
+- 确认已使用默认连接稳健配置（`pool_pre_ping=True`、MySQL 默认 `pool_recycle=1800`）
+- 按需在 `engine_args` 中调整连接池策略（如 `pool_recycle`、`pool_timeout`）
-### 问题4: TypeError: Data() argument after ** must be a mapping
+### 问题5: TypeError: Data() argument after ** must be a mapping
**解决**: 确保使用 `format="jsonl"` 而不是 `format="json"`
diff --git a/docs/hallucination_detection_guide.md b/docs/hallucination_detection_guide.md
index 4da27e0d..8e44fb3a 100644
--- a/docs/hallucination_detection_guide.md
+++ b/docs/hallucination_detection_guide.md
@@ -1,6 +1,8 @@
# Dingo Hallucination Detection - Complete Guide
-This guide introduces how to use integrated hallucination detection features in Dingo, supporting two detection methods: **HHEM-2.1-Open local model** (recommended) and **GPT-based cloud detection**.
+This guide introduces how to use integrated hallucination detection features in Dingo, supporting two detection methods: **MiniCheck local model** (recommended) and **GPT-based cloud detection**.
+
+> Note: the local rule is still named `RuleHallucinationHHEM` for backward compatibility, but its underlying model was upgraded from Vectara HHEM-2.1-Open to `lytang/MiniCheck-Flan-T5-Large` (a standard T5 grounding checker, arXiv:2404.10774).
## 🎯 Feature Overview
@@ -57,7 +59,7 @@ context = {"passages": ["Context 1", "Context 2"]}
## 🚀 Quick Start
-### Method 1: HHEM-2.1-Open Local Model (Recommended ⭐)
+### Method 1: MiniCheck Local Model (Recommended ⭐)
**Advantages**:
- ✅ Fast speed
@@ -72,7 +74,7 @@ context = {"passages": ["Context 1", "Context 2"]}
pip install dingo-python[hhem]
# Or install dependencies manually
-pip install sentence-transformers torch
+pip install transformers torch sentencepiece
```
**Usage**:
@@ -82,7 +84,7 @@ from dingo.config.input_args import EvaluatorRuleArgs
from dingo.io.input import Data
from dingo.model.rule.rule_hallucination_hhem import RuleHallucinationHHEM
-# Configure (first run will auto-download model ~400MB)
+# Configure (first run will auto-download model ~3GB, flan-t5-large)
RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs(
threshold=0.5 # Hallucination threshold, higher = stricter
)
@@ -208,7 +210,7 @@ print(f"Pass rate: {summary.score}%")
### Threshold Adjustment
```python
-# Method 1: Rule-based (HHEM)
+# Method 1: Rule-based (MiniCheck)
RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs(
threshold=0.5 # Range: 0.0-1.0
)
@@ -227,38 +229,33 @@ LLMHallucination.dynamic_config = EvaluatorLLMArgs(
- **General scenarios** (Q&A systems): 0.5-0.6
- **Loose scenarios** (creative content): 0.7-0.8
-### Device Selection (HHEM Only)
-
-```python
-# Auto-select (default: uses GPU if available)
-RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs()
+### Device (MiniCheck)
-# Force CPU
-import torch
-RuleHallucinationHHEM.device = "cpu"
+By default the MiniCheck model is loaded and runs on **CPU**, which is
+sufficient for the 0.8B flan-t5-large checkpoint. If you want to run inference
+on GPU, move the loaded model to your device after the first load:
-# Force GPU
-RuleHallucinationHHEM.device = "cuda"
-
-# Specific GPU
-RuleHallucinationHHEM.device = "cuda:0"
+```python
+# Trigger a load, then move to GPU
+RuleHallucinationHHEM.load_model()
+RuleHallucinationHHEM.model = RuleHallucinationHHEM.model.to("cuda")
```
## 📈 Performance Comparison
-| Feature | HHEM-2.1-Open | GPT-based |
+| Feature | MiniCheck (local) | GPT-based |
|---------|---------------|-----------|
-| **Speed** | Fast (~50ms/sample) | Slower (~1-2s/sample) |
+| **Speed** | Fast (~200-500ms/sample on CPU) | Slower (~1-2s/sample) |
| **Cost** | Free | API costs |
-| **Accuracy** | High (F1: 0.84) | Very High |
+| **Accuracy** | High (75.0 balanced acc. on LLM-AggreFact) | Very High |
| **Privacy** | Local, secure | Data sent to API |
-| **Deployment** | Needs model download (~400MB) | Needs API key |
+| **Deployment** | Needs model download (~3GB) | Needs API key |
| **Offline** | ✅ Supported | ❌ Requires network |
**Recommendations**:
-- **Production environment**: HHEM-2.1-Open (fast, free, private)
+- **Production environment**: MiniCheck (fast, free, private)
- **High-precision scenarios**: GPT-based (highest accuracy)
-- **Offline scenarios**: HHEM-2.1-Open (can run completely offline)
+- **Offline scenarios**: MiniCheck (can run completely offline)
## 🌟 Best Practices
@@ -321,21 +318,23 @@ data = Data(
## ❓ FAQ
-### Q1: HHEM vs GPT-based, which to choose?
+### Q1: MiniCheck vs GPT-based, which to choose?
-- **Production/large-scale**: HHEM (fast, free, private)
+- **Production/large-scale**: MiniCheck (fast, free, private)
- **High-precision evaluation**: GPT-based (highest accuracy, but has costs)
-- **Offline scenarios**: HHEM (can run completely offline)
+- **Offline scenarios**: MiniCheck (can run completely offline)
-### Q2: Why does HHEM download model on first run?
+### Q2: Why does the local rule download a model on first run?
-HHEM uses Sentence-Transformers model (~400MB), auto-downloads and caches on first run. Subsequent runs load directly from cache, no re-download needed.
+The local rule uses the `lytang/MiniCheck-Flan-T5-Large` model (~3GB),
+auto-downloads and caches on first run. Subsequent runs load directly from
+cache, no re-download needed.
### Q3: What if model download fails?
```bash
# Manually download
-huggingface-cli download vectara/hallucination_evaluation_model --local-dir ~/.cache/huggingface/hub/models--vectara--hallucination_evaluation_model
+hf download lytang/MiniCheck-Flan-T5-Large
# Or use mirror
export HF_ENDPOINT=https://hf-mirror.com
@@ -352,7 +351,7 @@ export HF_ENDPOINT=https://hf-mirror.com
- [RAG Evaluation Metrics Guide](rag_evaluation_metrics.md)
- [Factuality Assessment Guide](factuality_assessment_guide.md)
-- [HHEM Paper](https://arxiv.org/abs/2406.09053)
+- [MiniCheck Paper](https://arxiv.org/abs/2404.10774)
## 📝 Example Scenarios
diff --git a/docs/hallucination_guide.md b/docs/hallucination_guide.md
index 2ca58899..d546f1bc 100644
--- a/docs/hallucination_guide.md
+++ b/docs/hallucination_guide.md
@@ -1,6 +1,8 @@
# Dingo 幻觉检测功能完整指南
-本指南介绍如何在 Dingo 中使用集成的幻觉检测功能，支持两种检测方案：**HHEM-2.1-Open 本地模型**（推荐）和 **GPT-based 云端检测**。
+本指南介绍如何在 Dingo 中使用集成的幻觉检测功能，支持两种检测方案：**MiniCheck 本地模型**（推荐）和 **GPT-based 云端检测**。
+
+> 说明：本地规则仍沿用类名 `RuleHallucinationHHEM`（保持向后兼容），但底层模型已从 Vectara HHEM-2.1-Open 升级为 `lytang/MiniCheck-Flan-T5-Large`（标准 T5 事实核查模型，arXiv:2404.10774）。相比 HHEM，MiniCheck 在 LLM-AggreFact 基准上更强（75.0 vs 71.8），且为标准 T5，不再受 transformers 版本限制。
## 🎯 功能概述
@@ -58,15 +60,28 @@ context = "单个参考上下文"
## 🚀 快速开始
-### 方法一：HHEM-2.1-Open 本地模型（推荐）
+### 方法一：MiniCheck 本地模型（推荐）
#### 安装依赖
```bash
-pip install transformers torch
+pip install transformers torch sentencepiece
# 或使用专门的依赖文件
pip install -r requirements/hhem_integration.txt
```
+#### 模型下载与镜像
+
+首次运行会自动从 Hugging Face 下载 `lytang/MiniCheck-Flan-T5-Large` 模型（约 3GB，flan-t5-large），之后从本地缓存加载，无需重复下载。
+
+如果无法访问 `huggingface.co`（如国内网络），可在运行前设置镜像环境变量，让下载走 [hf-mirror.com](https://hf-mirror.com)：
+
+```bash
+# 设置镜像后再运行 dingo（对 huggingface_hub / transformers / datasets 全部生效）
+export HF_ENDPOINT=https://hf-mirror.com
+```
+
+> 说明：Dingo 不会强制修改该变量，以免影响能直连官网的环境（如 CI）；是否使用镜像由你自行控制。也可用 `hf download lytang/MiniCheck-Flan-T5-Large` 提前手动下载。
+
#### 基本使用
```python
@@ -127,7 +142,7 @@ print(f"详细原因: {result.reason[0]}") # 包含幻觉分数等详细信息
## 📊 批量数据集评估
-### 使用 HHEM-2.1-Open（本地，免费）
+### 使用 MiniCheck 本地模型（本地，免费）
```python
from dingo.config import InputArgs
@@ -146,7 +161,7 @@ input_data = {
}
},
"executor": {
- "rule_list": ["RuleHallucinationHHEM"], # Use HHEM rule instead of LLM
+ "rule_list": ["RuleHallucinationHHEM"], # 使用本地 MiniCheck 规则替代 LLM
"result_save": {
"bad": True,
"good": True # Also save good examples for comparison
@@ -165,7 +180,7 @@ input_args = InputArgs(**input_data)
executor = Executor.exec_map["local"](input_args)
result = executor.execute()
-print(f"HHEM 幻觉检测完成: 发现 {result.bad_count}/{result.total_count} 个问题")
+print(f"MiniCheck 幻觉检测完成: 发现 {result.bad_count}/{result.total_count} 个问题")
```
### 使用 GPT（在线，需要 API）
@@ -218,7 +233,7 @@ print(f"GPT 幻觉检测完成: 发现 {result.bad_count}/{result.total_count}
```python
# 方式1: 直接设置类属性
-RuleHallucinationHHEM.dynamic_config.threshold = 0.3 # HHEM 更严格的检测
+RuleHallucinationHHEM.dynamic_config.threshold = 0.3 # MiniCheck 更严格的检测
LLMHallucination.threshold = 0.3 # GPT 更严格的检测
# 方式2: 通过配置文件
@@ -248,7 +263,7 @@ LLMHallucination.threshold = 0.3 # GPT 更严格的检测
### 性能优化配置
```python
-# HHEM 批量处理优化
+# MiniCheck 批量处理优化
RuleHallucinationHHEM.load_model() # 预加载模型
results = RuleHallucinationHHEM.batch_evaluate(data_list) # 批量更高效
@@ -306,16 +321,19 @@ result.reason # List[str]: 详细分析原因（包含幻觉分数信
### 典型输出示例
-#### HHEM 输出示例
+#### MiniCheck 输出示例
```
-HHEM 幻觉分数: 0.650 (阈值: 0.500)
-处理了 2 个上下文对：
+🔍 MiniCheck 幻觉检测分析
+📊 平均幻觉分数: 0.350 (阈值: 0.500)
+📝 评估上下文数量: 2
+❌ 发现 1 个潜在矛盾:
+ 1. 上下文: "爱因斯坦在1921年获得诺贝尔奖。"
+ 一致性分数: 0.350, 幻觉分数: 0.650
+✅ 1 个上下文与回答一致:
1. 上下文: "爱因斯坦因发现光电效应获得诺贝尔奖。"
- 一致性: 0.95 → 幻觉分数: 0.05
- 2. 上下文: "爱因斯坦在1921年获得诺贝尔奖。"
- 一致性: 0.35 → 幻觉分数: 0.65
-平均幻觉分数: 0.350
-❌ 检测到幻觉: 超过阈值 0.500
+ 一致性分数: 0.950, 幻觉分数: 0.050
+🚨 结论: 检测到幻觉 (分数 0.350 > 阈值 0.500)
+💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理)
```
#### GPT 输出示例
@@ -353,7 +371,7 @@ HHEM 幻觉分数: 0.650 (阈值: 0.500)
### 1. RAG 系统质量监控
```python
-# 实时基于RAG监控回答质量（使用本地HHEM）
+# 实时基于RAG监控回答质量（使用本地 MiniCheck）
def monitor_rag_response(question, generated_answer, retrieved_docs):
data = Data(
data_id=f"rag_{timestamp}",
@@ -363,7 +381,6 @@ def monitor_rag_response(question, generated_answer, retrieved_docs):
)
result = RuleHallucinationHHEM.eval(data) # 本地、快速、免费
-
if result.status:
logger.warning(f"检测到幻觉: {result.reason[0]}")
# 触发人工审核或回答重生成
@@ -372,7 +389,7 @@ def monitor_rag_response(question, generated_answer, retrieved_docs):
### 2. SFT 数据集预处理
```python
-# 训练前检查SFT数据质量（批量处理使用HHEM）
+# 训练前检查SFT数据质量（批量处理使用 MiniCheck）
input_data = {
"input_path": "sft_training_data.jsonl",
"custom_config": {
@@ -391,7 +408,7 @@ def filter_hallucinated_responses(responses_with_context):
for item in responses_with_context:
data = Data(**item)
- # 使用本地HHEM进行快速检测
+ # 使用本地 MiniCheck 进行快速检测
result = RuleHallucinationHHEM.eval(data)
if not result.status: # 无幻觉
@@ -411,7 +428,7 @@ class RAGWithHallucinationDetection:
self.retriever = retriever
self.llm = llm
self.detector = hallucination_detector
- # 预加载HHEM模型以提高性能
+ # 预加载 MiniCheck 模型以提高性能
self.detector.load_model()
def generate_answer(self, question):
@@ -473,13 +490,13 @@ dingo/
│ ├── llm/
│ │ └── llm_hallucination.py # GPT-based 检测（DeepEval风格）
│ ├── rule/
-│ │ └── rule_hallucination_hhem.py # HHEM-2.1-Open 集成
+│ │ └── rule_hallucination_hhem.py # MiniCheck-Flan-T5-Large 集成
│ ├── prompt/prompt_hallucination.py # GPT 提示词模板
│ └── response/response_hallucination.py # 响应数据结构
├── io/input/Data.py # 扩展Data类支持context
├── examples/hallucination/ # 使用示例
-│ ├── sdk_rule_hhem_detection.py # Rule-based HHEM 使用示例
+│ ├── sdk_rule_hhem_detection.py # Rule-based MiniCheck 使用示例
│ ├── sdk_hallucination_detection.py # GPT 使用示例
│ └── dataset_hallucination_evaluation.py # 批量评估示例
-└── requirements/hhem_integration.txt # HHEM 依赖
+└── requirements/hhem_integration.txt # MiniCheck 依赖
```
diff --git a/docs/metrics.md b/docs/metrics.md
index 586d26f0..c717e52d 100644
--- a/docs/metrics.md
+++ b/docs/metrics.md
@@ -30,6 +30,15 @@ This document provides comprehensive information about all quality metrics used
| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
+### National Standard LLM Assessment Metrics
+
+| Type | Metric | Description | Source | Evaluation Results | Examples |
+|------|--------|-------------|--------|-------------------|----------|
+| `LLM_TC609_0101_DocBasicInfoCompleteness` | LLM_TC609_0101_DocBasicInfoCompleteness | Uses an LLM to assess dataset scale, format, file structure, access channel, and technical support in dataset documentation. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) |
+| `LLM_TC609_0102_DocContentFeatureCompleteness` | LLM_TC609_0102_DocContentFeatureCompleteness | Uses an LLM to assess modality, distribution, label statistics, sample examples, and limitations. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) |
+| `LLM_TC609_0103_DocConstructionProcessCompleteness` | LLM_TC609_0103_DocConstructionProcessCompleteness | Uses an LLM to assess source, collection, processing, annotation, and version control. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) |
+| `LLM_TC609_0104_DocApplicationCompleteness` | LLM_TC609_0104_DocApplicationCompleteness | Uses an LLM to assess license, target scenarios, evaluation method, benchmark results, and typical cases. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) |
+
### SFT Data Assessment Metrics
| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
@@ -41,7 +50,7 @@ This document provides comprehensive information about all quality metrics used
| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A |
| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A |
| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A |
-| `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A |
+| `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses the MiniCheck-Flan-T5-Large model for local hallucination detection by checking whether the response is grounded... | [MiniCheck: Efficient Fact-Checking of LLMs on Grounding Documents](https://arxiv.org/abs/2404.10774) (Liyan Tang, Philippe Laban, Greg Durrett) | N/A | N/A |
### Classification Metrics
@@ -86,6 +95,12 @@ This document provides comprehensive information about all quality metrics used
| `QUALITY_BAD_EFFECTIVENESS` | RuleAudioDuration | Check whether the audio duration meets the standard | Internal Implementation | N/A | N/A |
| `QUALITY_BAD_EFFECTIVENESS` | RuleAudioSnrQuality | Check whether the audio signal-to-noise ratio meets the standard | Internal Implementation | N/A | N/A |
+### Document Quality Assessment Metrics
+
+| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
+|------|--------|-------------|--------------|-------------------|----------|
+| `LLMAISmell` | LLMAISmell | Detects AI-generated writing patterns in requirement documents across 5 dimensions: hollow truisms, repetition, rainb... | Internal Implementation | N/A | [📝 View Example](../examples/llm_and_rule/llm_local.py) |
+
### Job Hunting Strategy Metrics
| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
@@ -126,7 +141,7 @@ This document provides comprehensive information about all quality metrics used
| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
|------|--------|-------------|--------------|-------------------|----------|
-| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A |
### Rule-Based RESUME Quality Metrics
@@ -140,6 +155,21 @@ This document provides comprehensive information about all quality metrics used
| `RESUME_QUALITY_BAD_PROFESSIONALISM` | RuleResumeEmoji, RuleResumeInformal | Detects emoji usage in resume which reduces professionalism; Detects informal or colloquial expressions in resume | Internal Implementation | N/A | N/A |
| `RESUME_QUALITY_BAD_STRUCTURE` | RuleResumeNameMissing, RuleResumeSectionMissing | Checks if resume contains a name in the first 200 characters; Checks if resume contains required sections like educat... | Internal Implementation | N/A | N/A |
+### SAC/TC609 High-quality Dataset Metrics
+
+Only the following eight TC609 rule metrics are currently registered. Other rule implementations remain in the source code with their registration decorators commented out.
+
+| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
+|------|--------|-------------|--------------|-------------------|----------|
+| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks TC609 annotation metadata fields, types, and enumerated values. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `source` and `source_details`; validates non-empty traceability information and HTTP/HTTPS URL syntax when applicable. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses multilingual text embeddings to check consistency among text items in `data_content`; multiple texts use robust-center aggregation instead of all-pairs comparison. | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Checks whether text items in `data_content` match the configured dataset type | Internal Implementation | N/A | N/A |
+| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | Combines available text cleanliness checks; modality coverage is partial. | Internal Implementation | N/A | N/A |
+
### SFT Data Assessment Metrics - Agent-Enhanced
| Type | Metric | Description | Paper Source | Evaluation Results | Examples |
diff --git a/docs/rules.md b/docs/rules.md
index e8d5a023..8bd72f6d 100644
--- a/docs/rules.md
+++ b/docs/rules.md
@@ -2,26 +2,94 @@ The specific rules for each quality metric are as follows:
| Function Name | Type | Description | Reference |
|------------------------------|-------------------|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| RuleAlphaWords | EFFECTIVENESS | check whether the ratio of words that contain at least one alphabetic character > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleCapitalWords | UNDERSTANDABILITY | check whether capital words ratio > 0.2 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) |
-| RuleCharNumber | EFFECTIVENESS | check whether the number of char > 100 | [MAP-en](https://arxiv.org/abs/2405.19327) |
-| RuleColonEnd | COMPLETENESS | check whether the last char is ':' | |
-| RuleContentNull | EFFECTIVENESS | check whether content is null | |
-| RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) |
-| RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) |
-| RuleHtmlEntity | RELEVANCE | check whether content has html entity | |
-| RuleIDCard | SECURITY | check if the content contains ID card. | |
-| RuleLineEndWithEllipsis | COMPLETENESS | check whether the ratio of line ends with ellipsis < 0.3 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleLineEndWithTerminal | COMPLETENESS | check whether the ratio of line ends with terminal punctuation mark > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
+| RuleAbnormalChar | EFFECTIVENESS | Check whether content contains abnormal characters. | |
+| RuleAbnormalHtml | EFFECTIVENESS | Check whether content contains abnormal HTML. | |
+| RuleAbnormalNumber | FLUENCY | Check PDF content for abnormal page or index numbers. | |
+| RuleAgentTraceLatencyAnomaly | AGENT_TRACE_QUALITY | Detect abnormally slow agent steps using statistical outlier analysis. | |
+| RuleAgentTraceLoopDetection | AGENT_TRACE_QUALITY | Detect repetitive tool-call patterns that indicate loops. | |
+| RuleAgentTraceTokenBudget | AGENT_TRACE_QUALITY | Check whether agent token usage exceeds the configured budget. | |
+| RuleAlphaWords | EFFECTIVENESS | check whether the ratio of words that contain at least one alphabetic character > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleAudioDataFormat | EFFECTIVENESS | Check whether audio data has the expected format. | |
+| RuleAudioDuration | EFFECTIVENESS | Check whether audio duration meets the configured standard. | |
+| RuleAudioSnrQuality | EFFECTIVENESS | Check whether the audio signal-to-noise ratio meets the configured standard. | |
+| RuleAuthorFieldValidation | EFFECTIVENESS | Validate scientific metadata author fields. | |
+| RuleCapitalWords | UNDERSTANDABILITY | check whether capital words ratio > 0.2 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) |
+| RuleCharNumber | EFFECTIVENESS | check whether the number of char > 100 | [MAP-en](https://arxiv.org/abs/2405.19327) |
+| RuleCharSplit | FLUENCY | Check PDF content for abnormally split characters. | |
+| RuleColonEnd | COMPLETENESS | check whether the last char is ':' | |
+| RuleContentNull | EFFECTIVENESS | check whether content is null | |
+| RuleContentShort | EFFECTIVENESS | Check whether content is too short. | |
+| RuleContentShortMultiLan | EFFECTIVENESS | Check whether multilingual content is too short. | |
+| RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) |
+| RuleDictConsistency | EFFECTIVENESS | Compare two dictionary fields and report mismatched keys. | |
+| RuleDocFormulaRepeat | SIMILARITY | Check whether formulas repeat in a document. | |
+| RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) |
+| RuleDoi | EFFECTIVENESS | Validate DOI metadata. | |
+| RuleEnterAndSpace | EFFECTIVENESS | Check abnormal combinations of line breaks and spaces. | |
+| RuleEnterMore | EFFECTIVENESS | Check whether content has excessive consecutive line breaks. | |
+| RuleEnterRatioMore | EFFECTIVENESS | Check whether the line-break ratio is excessive. | |
+| RuleHallucinationHHEM | HALLUCINATION | Detect ungrounded claims with MiniCheck-Flan-T5-Large (checks whether the response is supported by the context). | |
+| RuleHeadWordAr | RELEVANCE | Check Arabic content for irrelevant source information. | |
+| RuleHeadWordCs | RELEVANCE | Check Czech content for irrelevant source information. | |
+| RuleHeadWordHu | RELEVANCE | Check Hungarian content for irrelevant source information. | |
+| RuleHeadWordKo | RELEVANCE | Check Korean content for irrelevant source information. | |
+| RuleHeadWordRu | RELEVANCE | Check Russian content for irrelevant source information. | |
+| RuleHeadWordSr | RELEVANCE | Check Serbian content for irrelevant source information. | |
+| RuleHeadWordTh | RELEVANCE | Check Thai content for irrelevant source information. | |
+| RuleHeadWordVi | RELEVANCE | Check Vietnamese content for irrelevant source information. | |
+| RuleHtmlEntity | RELEVANCE | check whether content has html entity | |
+| RuleHtmlTag | EFFECTIVENESS | Check whether content contains image links or HTML tags. | |
+| RuleIDCard | SECURITY | check if the content contains ID card. | |
+| RuleImageArtimuse | IMG_ARTIMUSE | Detect inappropriate artificial-image usage. | |
+| RuleImageDataFormat | EFFECTIVENESS | Check whether image data has the expected format. | |
+| RuleImageLabelOverlap | IMG_LABEL_OVERLAP | Check whether image labels overlap. | |
+| RuleImageLabelVisualization | IMG_LABEL_VISUALIZATION | Visualize and validate image labels. | |
+| RuleImageQuality | IMG_EFFECTIVENESS | Check whether image quality meets the configured standard. | |
+| RuleImageRepeat | IMG_SIMILARITY | Detect duplicate images using perceptual hash or CNN features. | |
+| RuleImageSizeValid | IMG_EFFECTIVENESS | Check whether image dimensions and aspect ratio are valid. | |
+| RuleImageTextSimilarity | IMG_RELEVANCE | Check similarity between an image and its text content. | |
+| RuleImageValid | IMG_EFFECTIVENESS | Check whether an image is valid and not uniformly white or black. | |
+| RuleInvisibleChar | EFFECTIVENESS | Check whether content contains invisible characters. | |
+| RuleIsbn | EFFECTIVENESS | Validate ISBN metadata. | |
+| RuleLatexSpecialChar | EFFECTIVENESS | Check PDF content for abnormal LaTeX characters. | |
+| RuleLineEndWithEllipsis | COMPLETENESS | check whether the ratio of line ends with ellipsis < 0.3 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleLineEndWithTerminal | COMPLETENESS | check whether the ratio of line ends with terminal punctuation mark > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
+| RuleLineJavascriptCount | EFFECTIVENESS | check whether line with the word Javascript. | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
| RuleLineStartWithBulletpoint | UNDERSTANDABILITY | check whether the ratio of line starts with bullet points < 0.9 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleLineJavascriptCount | EFFECTIVENESS | check whether line with the word Javascript. | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
-| RuleLoremIpsum | EFFECTIVENESS | check whether the ratio of lorem ipsum < 3e-08 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
-| RuleMeanWordLength | EFFECTIVENESS | check whether the mean length of word in [3, 10] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleNoPunc | FLUENCY | check whether paragraph has no punctuation. | |
-| RuleSentenceNumber | COMPLETENESS | check whether the number of sentence in [3, 7500] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
-| RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | |
-| RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
-| RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) |
-| RuleWatermark | RELEVANCE | check whether content has watermarks. | |
-| RuleWordNumber | EFFECTIVENESS | check whether the number of word in [20, 100000] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleLoremIpsum | EFFECTIVENESS | check whether the ratio of lorem ipsum < 3e-08 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
+| RuleMeanWordLength | EFFECTIVENESS | check whether the mean length of word in [3, 10] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleMetadataSimilarity | EFFECTIVENESS | Compare the similarity of scientific metadata fields. | |
+| RuleNlpDataFormat | EFFECTIVENESS | Check whether NLP data has the expected format. | |
+| RuleNoPunc | FLUENCY | check whether paragraph has no punctuation. | |
+| RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | |
+| RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | |
+| RulePIIDetection | SECURITY | Detect personally identifiable information in text. | |
+| RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | |
+| RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | |
+| RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | |
+| RuleResumeEducationMissing | RESUME_COMPLETENESS | Check whether a resume is missing its education section. | |
+| RuleResumeEmailMissing | RESUME_CONTACT | Check whether a resume is missing an email address. | |
+| RuleResumeEmoji | RESUME_PROFESSIONALISM | Check whether a resume contains emoji. | |
+| RuleResumeExcessiveWhitespace | RESUME_FORMAT | Check whether a resume contains excessive whitespace. | |
+| RuleResumeExperienceMissing | RESUME_COMPLETENESS | Check whether a resume is missing work experience. | |
+| RuleResumeIDCard | RESUME_PRIVACY | Check whether a resume contains a Chinese ID card number. | |
+| RuleResumeInformal | RESUME_PROFESSIONALISM | Check whether a resume contains informal language. | |
+| RuleResumeMarkdown | RESUME_FORMAT | Check whether a resume contains Markdown syntax errors. | |
+| RuleResumeNameMissing | RESUME_STRUCTURE | Check whether a resume is missing a name in its first section. | |
+| RuleResumePhoneFormat | RESUME_CONTACT | Check whether a phone number has an invalid format. | |
+| RuleResumePhoneMissing | RESUME_CONTACT | Check whether a resume is missing a phone number. | |
+| RuleResumeSectionMissing | RESUME_STRUCTURE | Check whether a resume is missing required sections. | |
+| RuleSentenceNumber | COMPLETENESS | check whether the number of sentence in [3, 7500] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) |
+| RuleSftDataFormat | EFFECTIVENESS | Check whether supervised fine-tuning data has the expected format. | |
+| RuleSourceFieldValidation | EFFECTIVENESS | Validate scientific metadata source fields. | |
+| RuleSpaceMore | EFFECTIVENESS | Check whether content contains excessive spaces. | |
+| RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | |
+| RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) |
+| RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | |
+| RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | |
+| RuleWatermark | RELEVANCE | check whether content has watermarks. | |
+| RuleWordNumber | EFFECTIVENESS | check whether the number of word in [20, 100000] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) |
+| RuleWordSplit | FLUENCY | Check PDF content for abnormally split words. | |
+| RuleWordStuck | FLUENCY | Check whether words are abnormally joined together. | |
diff --git a/docs/rules_tc609.md b/docs/rules_tc609.md
new file mode 100644
index 00000000..442233ef
--- /dev/null
+++ b/docs/rules_tc609.md
@@ -0,0 +1,14 @@
+# SAC/TC609 High-quality Dataset Rules
+
+The following eight TC609 rules are currently registered. Rules whose registration decorators are commented out are not included.
+
+| Rule | Metric Type | Description | Standard Source |
+|------|-------------|-------------|-----------------|
+| Rule_TC609_0201_FormatCompliance | QUALITY_BAD_TC609_0201 | Checks required fields and types against the configured field schema. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
+| Rule_TC609_0202_SafetyCompliance | QUALITY_BAD_TC609_0202 | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
+| Rule_TC609_0203_AnnotationCompliance | QUALITY_BAD_TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements |
+| Rule_TC609_0204_StructuralCompleteness | QUALITY_BAD_TC609_0204 | Checks configured fields for missing values. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
+| Rule_TC609_0205_ContentAuthenticity | QUALITY_BAD_TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements |
+| Rule_TC609_0206_ContentConsistency | QUALITY_BAD_TC609_0206 | Checks consistency among text items in `data_content` using multilingual embeddings and robust-center aggregation. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
+| Rule_TC609_0207_DataTypeConsistency | QUALITY_BAD_TC609_0207 | Checks whether text items in `data_content` match the configured dataset type. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
+| Rule_TC609_0208_ContentCleanliness | QUALITY_BAD_TC609_0208 | Combines available text cleanliness checks; modality coverage is partial. | TC609-5-2025-04 High-quality dataset quality evaluation specification |
diff --git a/docs/search_result_authority_executor.md b/docs/search_result_authority_executor.md
new file mode 100644
index 00000000..3b171df4
--- /dev/null
+++ b/docs/search_result_authority_executor.md
@@ -0,0 +1,187 @@
+# Search Result Authority Executor Usage
+
+This document describes the executor-based authority evaluation for meta search results.
+
+## What Changed
+
+`examples/retrieval/sdk_eval_authority.py` evaluates each retrieved result with `LLMSearchResultAuthority` through Dingo `LocalExecutor`.
+
+The standalone authority script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds.
+
+`LLMSearchResultAuthority` is rule-based despite being registered as an LLM evaluator. It does not call an external model. The score is based on citation impact, venue/source signals, and DOI availability.
+
+## Flow
+
+1. Read query-level meta search JSONL.
+2. Flatten each query's top-k results into result-level JSONL rows.
+3. Build `InputArgs`.
+4. Run `Executor.exec_map["local"]`.
+5. `LLMSearchResultAuthority` returns standard `EvalDetail(metric/status/score/label/reason)`.
+6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Authority/Error_*.jsonl`.
+
+Each flattened input row contains:
+
+| Field | Meaning |
+|---|---|
+| `query` | Original query text |
+| `query_index` | Query order in source JSONL |
+| `rank` | Result rank under the query |
+| `title` | Result title or display name |
+| `search_result` | Full result payload passed to evaluator |
+
+## Test Data
+
+The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`). Authority evaluation is rule-based, so this smoke test does not require an LLM API:
+
+```powershell
+python examples/retrieval/sdk_eval_authority.py `
+ --input-jsonl test/data/test_search_result.jsonl `
+ --output-dir outputs/search_result_authority_smoke `
+ --top-k 3 `
+ --threshold 0.15 `
+ --save-good
+```
+
+## Commands
+
+Run authority evaluation:
+
+```powershell
+python examples/retrieval/sdk_eval_authority.py `
+ --input-jsonl outputs/meta_search_97_query_results.jsonl `
+ --output-dir outputs/search_result_authority_97q_executor `
+ --top-k 10 `
+ --threshold 0.15 `
+ --save-good
+```
+
+Smoke test with fewer queries:
+
+```powershell
+python examples/retrieval/sdk_eval_authority.py `
+ --input-jsonl outputs/meta_search_97_query_results.jsonl `
+ --output-dir outputs/search_result_authority_smoke `
+ --top-k 10 `
+ --max-queries 5 `
+ --threshold 0.15 `
+ --save-good
+```
+
+Useful parameters:
+
+| Parameter | Default | Meaning |
+|---|---:|---|
+| `--top-k` | `10` | Number of results evaluated per query |
+| `--max-queries` | `None` | Limit query count for smoke tests |
+| `--threshold` | `0.15` | Result-level bad threshold |
+| `--max-workers` | `4` | LocalExecutor worker count |
+| `--batch-size` | `10` | LocalExecutor batch size |
+| `--save-good` | off | Save passing samples |
+| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows |
+
+## Output
+
+The executor creates a timestamped child directory under `--output-dir`, for example:
+
+```text
+outputs/search_result_authority_97q_executor/20260709_172501_0f73c631/
+```
+
+Main files:
+
+| Path | Meaning |
+|---|---|
+| `summary.json` | Executor summary |
+| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` |
+| `search_result/Authority/Error_*.jsonl` | Bad result-level samples grouped by error type |
+
+`summary.json` uses result-level statistics:
+
+- `total`: number of evaluated retrieved results.
+- `num_good`: result count with no error labels.
+- `num_bad`: result count with at least one error label.
+- `score`: `num_good / total * 100`.
+- `metrics_score.search_result.stats.LLMSearchResultAuthority`: result-level authority score distribution.
+- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio.
+
+Error labels:
+
+| Label path | Trigger |
+|---|---|
+| `search_result/Authority/Error_Authority_Low.jsonl` | Final authority score below threshold |
+| `search_result/Authority/Error_Citation_Miss.jsonl` | Authority is low and citation score is zero |
+| `search_result/Authority/Error_Venue_Low_Signal.jsonl` | Authority is low and venue/source has only low signal |
+| `search_result/Authority/Error_DOI_Miss.jsonl` | Authority is low and no DOI signal is present |
+
+The sub-labels are emitted only when the final authority score is below the threshold. For example, a result without DOI can still be `QUALITY_GOOD` if citation and venue signals are strong enough.
+
+## Scoring
+
+The metric score is:
+
+```text
+authority =
+ 0.45 * citation_score
++ 0.20 * influential_citation_score
++ 0.25 * venue_score
++ 0.10 * doi_score
+```
+
+Citation scores use log normalization and are clamped to `[0, 1]`:
+
+```text
+citation_score = log1p(citation_count) / log1p(500)
+influential_citation_score = log1p(influential_citation_count) / log1p(50)
+```
+
+Venue is read from the first available field:
+
+```text
+publication_venue_name_unified
+publication_venue_name
+venue
+source
+```
+
+Venue scoring:
+
+| Condition | `venue_score` | Reason |
+|---|---:|---|
+| Known repository or preprint source | `0.45` | `repository_or_preprint` |
+| Explicit academic book series or ebook platform | `0.55` | `academic_book_series` |
+| Venue matches a prestigious journal/conference family | `0.85` | `prestigious_venue_family` |
+| Venue or publisher matches a recognized scholarly organization | `0.75` | `recognized_scholarly_publisher_or_venue` |
+| Journal/conference type or valid ISSN is present | `0.65` | `structured_journal_or_conference` |
+| A venue name is present without stronger structured signals | `0.40` | `named_venue` |
+| Unknown or low-signal source | `0.25` | `unknown_or_low_signal_venue` |
+
+Explicit source types run first. Repository detection prevents names such as `Open Science Framework` from being promoted merely because they contain a prestigious-looking word, while book series and ebook platforms remain at the book-source tier even when their publisher is recognized.
+
+Prestigious venue families include:
+
+```text
+Nature and Nature subject journals, Nature Communications,
+npj journals, Communications journals, Scientific Reports/Data,
+the official Science journal family, selected Cell Press flagships,
+NEJM, Lancet, JAMA, The BMJ, PNAS, JACS, PRL, and major AI conferences
+```
+
+The matcher uses anchored family patterns instead of unrestricted substrings. For example, `Science Translational Medicine` matches the Science family, while `Chemical Engineering Science` does not. HTML highlight tags are removed before matching.
+
+Recognized publisher metadata includes established scholarly publishers and societies such as Springer Nature, Elsevier, Wiley, Oxford University Press, Cambridge University Press, IEEE, ACM, ACS, RSC, IOP, BMJ, PLOS, the Royal Society, De Gruyter, CRC Press, and World Scientific. Publisher recognition is deliberately scored below an explicitly prestigious venue because publisher reputation alone does not make every title a flagship journal.
+
+The ISSN and venue-type fallback is the main protection against an incomplete whitelist: a journal does not need to appear in a hard-coded title list to receive a structured scholarly venue score.
+
+DOI scoring:
+
+```text
+doi_score = 1.0
+```
+
+when the result has a `doi` field or `locations` contains `doi.org`; otherwise:
+
+```text
+doi_score = 0.0
+```
+
+Authority does not judge query relevance or content completeness. A low authority score means the result lacks academic trust signals such as citations, venue, or DOI. It does not necessarily mean the result is irrelevant or unusable.
diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md
new file mode 100644
index 00000000..234def47
--- /dev/null
+++ b/docs/search_result_effectiveness_executor.md
@@ -0,0 +1,154 @@
+# Search Result Effectiveness Executor Usage
+
+This document describes the executor-based effectiveness evaluation for meta search results.
+
+## What Changed
+
+`examples/retrieval/sdk_eval_effectiveness.py` reuses Dingo `LocalExecutor`. It no longer hand-builds chunk-style `summary.json` or bad/good folders.
+
+The standalone effectiveness script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds.
+
+## Flow
+
+1. Read query-level meta search JSONL.
+2. Flatten each query's top-k results into result-level JSONL rows.
+3. Build `InputArgs`.
+4. Run `Executor.exec_map["local"]`.
+5. `LLMSearchResultEffectiveness` returns standard `EvalDetail(metric/status/score/label/reason)`.
+6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Effectiveness/Error_*.jsonl`.
+
+Each flattened input row contains:
+
+| Field | Meaning |
+|---|---|
+| `query` | Original query text |
+| `query_index` | Query order in source JSONL |
+| `rank` | Result rank under the query |
+| `title` | Result title or display name |
+| `search_result` | Full result payload passed to evaluator |
+
+## Test Data
+
+The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`).
+
+Smoke test with LLM second judgment enabled:
+
+```powershell
+python examples/retrieval/sdk_eval_effectiveness.py `
+ --input-jsonl test/data/test_search_result.jsonl `
+ --output-dir outputs/search_result_effectiveness_smoke `
+ --top-k 3 `
+ --llm-max-tokens 1024 `
+ --threshold 0.15 `
+ --save-good
+```
+
+## Commands
+
+Fast rule-only run:
+
+```powershell
+python examples/retrieval/sdk_eval_effectiveness.py `
+ --input-jsonl outputs/meta_search_97_query_results.jsonl `
+ --output-dir outputs/search_result_effectiveness_97q_executor `
+ --top-k 10 `
+ --threshold 0.15 `
+ --disable-llm-quality `
+ --save-good
+```
+
+Run with LLM second judgment for abnormal-character candidates:
+
+For full runs, use a low-latency Flash model such as `deepseek-v4-flash`. The LLM is only used to review rule-selected suspicious fields, but a slow Pro model can still substantially increase runtime. Reserve Pro models for small-sample diagnosis, and use temperature `0` for reproducible comparisons.
+
+```powershell
+$env:OPENAI_API_KEY="..."
+$env:OPENAI_BASE_URL="http://35.220.164.252:3888/v1/"
+$env:OPENAI_MODEL="deepseek-v4-flash"
+$env:OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_effectiveness.py `
+ --input-jsonl outputs/meta_search_97_query_results.jsonl `
+ --output-dir outputs/search_result_effectiveness_97q_executor_llm `
+ --top-k 10 `
+ --threshold 0.15 `
+ --llm-max-tokens 512 `
+ --llm-workers 4 `
+ --save-good
+```
+
+Useful parameters:
+
+| Parameter | Default | Meaning |
+|---|---:|---|
+| `--top-k` | `10` | Number of results evaluated per query |
+| `--max-queries` | `None` | Limit query count for smoke tests |
+| `--threshold` | `0.15` | Result-level bad threshold |
+| `--save-good` | off | Save passing samples |
+| `--disable-llm-quality` | off | Skip LLM second judgment and use deterministic rules only |
+| `--llm-max-tokens` | `512` | Max tokens for LLM second judgment |
+| `--llm-workers` | `4` | LocalExecutor worker count |
+| `--batch-size` | `10` | LocalExecutor batch size |
+| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows |
+
+## Output
+
+The executor creates a timestamped child directory under `--output-dir`, for example:
+
+```text
+outputs/search_result_effectiveness_97q_executor/20260709_172501_0f73c631/
+```
+
+Main files:
+
+| Path | Meaning |
+|---|---|
+| `summary.json` | Executor summary |
+| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` |
+| `search_result/Effectiveness/Error_*.jsonl` | Bad result-level samples grouped by error type |
+
+`summary.json` uses result-level statistics:
+
+- `total`: number of evaluated retrieved results.
+- `num_good`: result count with no error labels.
+- `num_bad`: result count with at least one error label.
+- `score`: `num_good / total * 100`.
+- `metrics_score.search_result.stats.LLMSearchResultEffectiveness`: result-level effectiveness score distribution.
+- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio.
+
+Error labels:
+
+| Label path | Trigger |
+|---|---|
+| `search_result/Effectiveness/Error_Title_Miss.jsonl` | Missing title |
+| `search_result/Effectiveness/Error_Abstract_Miss.jsonl` | Missing abstract |
+| `search_result/Effectiveness/Error_Keywords_Miss.jsonl` | Missing keywords |
+| `search_result/Effectiveness/Error_Author_Miss.jsonl` | Missing author metadata |
+| `search_result/Effectiveness/Error_HTML_Tag.jsonl` | LLM-confirmed HTML tag pollution |
+| `search_result/Effectiveness/Error_Mojibake.jsonl` | LLM-confirmed mojibake |
+| `search_result/Effectiveness/Error_Invisible_Char.jsonl` | LLM-confirmed invisible characters |
+| `search_result/Effectiveness/Error_Unreadable_Text.jsonl` | LLM-confirmed unreadable text |
+| `search_result/Effectiveness/Error_Special_Char_Noise.jsonl` | LLM-confirmed special-character noise |
+| `search_result/Effectiveness/Error_Effectiveness_Low.jsonl` | Final effectiveness score below threshold |
+
+`RuleSpecialCharacter` and `RuleInvisibleChar` are treated as candidate triggers. If LLM confirms a concrete issue, for example `title:html_tag`, the final label keeps only the concrete business label such as `Effectiveness.Error_HTML_Tag` and suppresses the intermediate rule label. The original rule issue is still retained in `EvalDetail.reason` for traceability.
+
+## Scoring
+
+The metric score is:
+
+```text
+Effectiveness =
+ title_score * 0.30
++ abstract_score * 0.50
++ keywords_score * 0.10
++ author_score * 0.10
+```
+
+Each non-empty title, abstract, keywords list, or author list receives full presence credit before corruption checks. Character length, token count, keyword count, and author count do not affect the base score. A confirmed HTML, mojibake, invisible-character, unreadable-text, or special-character issue can still reduce the affected field score.
+
+`author_score` checks whether at least one non-empty author value is present. A single author receives full presence credit; the metric does not reward a larger author count or penalize a short name.
+
+Title, abstract, keywords, and author use presence checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation.
+
+The HTML candidate pattern requires a valid tag name immediately after `<` or `` is not treated as markup, while real academic formatting tags such as ``, ``, ``, and `` remain detectable.
diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md
new file mode 100644
index 00000000..a9d8c85e
--- /dev/null
+++ b/docs/search_result_quality_metrics.md
@@ -0,0 +1,714 @@
+# Search Result Quality 三指标评测说明
+
+本文档说明检索结果的三类评测指标：相关性、内容有效性、权威性，以及对应的单项评测脚本和端到端综合评测脚本。该方案面向无人工 GT 的检索结果质量检查，既可读取预计算的 query+results，也可从 query 文件直接请求 SciVerse Meta Search 或 OpenAlex，再通过 Dingo Executor 完成评测和分类。
+
+## 1. 适用场景
+
+该评测用于回答三个业务问题：
+
+| 指标 | 业务问题 | 评测方式 |
+|---|---|---|
+| 相关性 `relevance` | 检索结果是否回答了用户 query 的真实检索意图 | LLM 逐条判断 query-result 匹配程度 |
+| 内容有效性 `effectiveness` | 结果记录本身是否完整、可读、可用于判断论文价值 | 规则检查字段缺失，RuleSpecialCharacter/RuleInvisibleChar 初筛异常候选，LLM 二次确认；不使用字段长度打分 |
+| 权威性 `authority` | 结果是否具备学术可信度和来源影响力信号 | 规则检查 citation、influential citation、venue、DOI |
+
+三个指标关注点不同：
+
+- 相关性判断“是不是用户要找的内容”。
+- 内容有效性判断“这条结果记录是否有足够信息可读可用”。
+- 权威性判断“这条结果是否有论文影响力、来源、DOI 等可信信号”。
+
+例如，用户搜索 `Wallace Chafe`，rank1 返回标题也是 `Wallace Chafe`，相关性可能较好；但如果该结果没有 abstract、keywords、author，则内容有效性会较低。Publication venue 的缺失由权威性指标负责。
+
+## 2. 输入格式
+
+综合脚本支持两种输入模式。
+
+### 2.1 预计算结果
+
+JSONL 每行一个 query 及其检索结果：
+
+```json
+{"query": "PBPK Review", "results": [{"title": "...", "abstract": "..."}]}
+```
+
+脚本支持的 query 字段名：
+
+- `query`
+- `query_text`
+- `q`
+
+脚本支持的结果列表字段名：
+
+- `results`
+- `top_results`
+- `top_api_results`
+- `search_results`
+
+常用输入路径示例：
+
+```bash
+outputs/meta_search_97_query_results.jsonl
+```
+
+### 2.2 端到端检索
+
+设置 `--retrieval-backend meta_search` 或 `openalex` 后，输入只需要包含 query。支持：
+
+- TXT：每行一个 query；
+- CSV：读取 `query`、`query_text` 或 `q` 列；
+- JSON/JSONL：字符串或包含上述 query 字段的对象。
+
+```json
+{"query": "PBPK相关综述"}
+```
+
+端到端模式使用后端默认相关度检索，不添加时间排序或业务过滤；`--top-k` 同时控制请求数量和评测数量。
+
+仓库提供了三条 query 的端到端测试输入：`test/data/test_search_queries.jsonl`。
+
+## 3. 输出文件
+
+综合评测脚本 `sdk_eval_search_result.py` 会在 `--output-dir` 下生成一个时间戳子目录，核心结果都放在该子目录中，例如：
+
+```text
+outputs/search_result_eval_97q/20260710_162652_1ac3f3be/
+```
+
+默认核心文件如下：
+
+| 文件 | 粒度 | 说明 |
+|---|---|---|
+| `summary.json` | 全局 | 指标均值、中位数、最小值、最大值、bad/good 数量、阈值、LLM 配置等 |
+| `query_scores.csv` | query 级 | 每个 query 的 top-k 排名加权平均分、label、eval_status |
+| `result_scores.csv` | result 级 | 每个 query 的每条 top-k 结果分数和诊断信息 |
+| `all_results.jsonl` | result 级原始明细 | executor 输出的逐条评测结果，保留三个指标的完整 `eval_details` |
+| `retrieval_results.jsonl` | query 级检索结果 | 仅端到端模式生成，保存可复跑的 query+results |
+| `retrieval_request_log.jsonl` | query 级请求日志 | 仅端到端模式生成，记录状态码、耗时、结果数和错误，不含 token |
+| `bad/` | query 级分类 | 只包含三种 query 聚合指标低分记录，每行一个 query 及其完整 results |
+| `good/` | query 级分类 | 使用 `--save-good` 时保存通过的 query 记录 |
+
+`detailed_results.json` 默认不生成；需要完整嵌套诊断时加 `--save-detailed`。
+
+单独运行 `sdk_eval_effectiveness.py`、`sdk_eval_relevancy.py`、`sdk_eval_authority.py` 时，会保留 Dingo executor 的单指标 result 级分类目录，例如 `search_result/Effectiveness/Error_*.jsonl`。
+
+分类 label 示例：
+
+```text
+QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW
+QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW
+QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW
+QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS
+```
+
+单独运行 `sdk_eval_effectiveness.py` 时，`summary.json` 采用类似 `sdk_chunk_eval.py` 的 result 级结构：每个 query 的每条 top-k 检索文献都是一个测试对象，`total`、`num_good`、`num_bad` 和 `score` 都按 result 级计算。`type_ratio.search_result` 中会统计 `Effectiveness.Error_*` 和 `QUALITY_GOOD` 的比例，`metrics_score.search_result` 中会统计 `LLMSearchResultEffectiveness` 的 result 级分数分布。
+
+内容有效性还会额外输出 result 级错误类型文件，例如：
+
+```text
+Effectiveness/Error_Title_Miss.jsonl
+Effectiveness/Error_Abstract_Miss.jsonl
+Effectiveness/Error_Keywords_Miss.jsonl
+Effectiveness/Error_Author_Miss.jsonl
+Effectiveness/Error_HTML_Tag.jsonl
+Effectiveness/Error_Mojibake.jsonl
+Effectiveness/Error_Invisible_Char.jsonl
+Effectiveness/Error_Unreadable_Text.jsonl
+Effectiveness/Error_Special_Char_Noise.jsonl
+Effectiveness/Error_LLM_Quality_Parse.jsonl
+QUALITY_GOOD.jsonl
+```
+
+只有实际出现的错误类型会生成对应文件。
+
+## 4. Query 级汇总逻辑
+
+三个指标都先对 top-k 中每条 result 打分，然后使用排名折扣加权平均汇总到 query 级。第 `rank` 条结果的权重为：
+
+```text
+weight(rank) = 1 / log2(rank + 1)
+query_score = sum(result_score_i * weight_i) / sum(weight_i)
+```
+
+rank1 权重最高，越靠后的结果权重越低。综合脚本的 bad/good 判定只比较这三个 query 级加权平均分和统一阈值；result 级分数仍保留在 `result_scores.csv` 与 `all_results.jsonl` 中用于定位问题，但不生成 result 级 bad/good 分类文件。空结果 query 的三个聚合分均为 `0`。
+
+## 5. 相关性 Relevance
+
+### 5.1 业务逻辑
+
+相关性判断每条检索结果与 query 是否匹配，重点看：
+
+- result 是否直接回答或覆盖 query 意图。
+- 标题和摘要是否围绕 query 主题。
+- 对短词、人名、论文题名、中文长 query 等非 DOI query，LLM 根据语义进行判断。
+- DOI query 使用结构化精确匹配：规范化 query DOI 与结果 `doi`、`unique_id` 或 location DOI，完全一致才算命中。
+- DOI 前缀相似、同出版社或主题相似但 DOI 不一致时，result 相关性为 0，不允许 LLM 猜测。
+
+### 5.2 输入字段
+
+| 输入 | 来源 |
+|---|---|
+| `query` | query 字段 |
+| `title` | result 的 `title` 或 `display_name` |
+| `abstract` | result 的 `abstract`、`summary` 或 `content` |
+| `doi` | DOI query 的精确标识符匹配 |
+| `unique_id`、`locations` | result 缺少顶层 DOI 时的标识符补充来源 |
+
+### 5.3 Result 级输出
+
+| 字段 | 说明 |
+|---|---|
+| `relevance` | result 总相关性分数 |
+| `query_relevance` | query 与 result 的语义匹配程度 |
+| `result_quality` | result 内容质量辅助判断 |
+| `content_issues` | LLM 判断是否存在内容问题 |
+| `confidence` | LLM 对评分的置信度 |
+| `error` | LLM 输出解析失败等错误 |
+| `reasoning` | 简短原因 |
+
+DOI query 不调用 LLM。Result 级完全匹配为 `1.0`，否则为 `0.0`；reason 中记录 expected DOI 和 result DOI。
+
+DOI query 的 result 级相关性仍按精确匹配得到 `1.0` 或 `0.0`，query 级相关性按精确命中的排名折扣：
+
+```text
+doi_relevance = max(exact_match_i / log2(rank_i + 1))
+```
+
+因此 rank1 精确命中为 `1.0`，rank2 命中为 `0.63093`，没有精确命中为 `0.0`。
+
+### 5.4 LLM 解析异常
+
+单独运行相关性脚本时，如果某个 query 的任意 rank 出现 LLM JSON 解析失败，会增加诊断 label：
+
+```text
+QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR
+```
+
+这类 label 表示运行/解析质量告警，不一定代表业务相关性低。综合端到端脚本仍会记录 `relevance_error_count` 和错误文本，但 bad 目录只使用三个指标低分 label；解析失败导致相关性分数低于阈值时，统一归入 `SEARCH_RESULT_RELEVANCE_LOW`。
+
+使用单项评测结果分析低相关时建议区分：
+
+- `SEARCH_RESULT_RELEVANCE_LOW`：业务低相关。
+- `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`：LLM 输出格式或解析异常。
+
+### 5.5 可调参数
+
+| 参数 | 默认值 | 说明 |
+|---|---:|---|
+| `--top-k` | 10 | 每个 query 评测前 k 条结果 |
+| `--threshold` | 0.15 | query 级 hard bad 阈值 |
+| `--llm-max-tokens` | 1024 | LLM 输出最大 token 数 |
+| `--llm-workers` | 4 | 并发 LLM 调用数 |
+| `--llm-timeout` | 60 | 单次 LLM 请求超时秒数 |
+| `--prompt-mode` | `detailed` | prompt 模式 |
+| `OPENAI_MODEL` | `gpt-5.4-mini` | LLM 模型名，可通过环境变量覆盖 |
+| `OPENAI_BASE_URL` | 空 | OpenAI compatible endpoint |
+| `OPENAI_TEMPERATURE` | 0.0 | LLM temperature |
+
+### 5.6 LLM 模型选择建议
+
+全量检索评测需要逐条判断 query-result 对，LLM 调用量通常接近 `query 数 × top-k`。建议优先使用低延迟的 Flash 类模型，例如：
+
+```text
+OPENAI_MODEL=gpt-5.4-mini
+```
+
+- **全量评测和日常回归**：推荐 Flash 类模型，可显著缩短相关性判断时间，并降低长时间批处理中的超时风险。
+- **疑难样本复核**：Pro 类模型更适合抽取少量低分、边界或争议样本进行人工辅助复核，不建议直接用于数百至数千条 result 的常规全量评测。
+- **新旧版本对比**：两次评测必须固定相同模型、prompt、`max_tokens` 和 temperature；建议设置 `OPENAI_TEMPERATURE=0`，减少 LLM 随机波动。
+- **并发设置**：建议从 `--llm-workers 2` 至 `4` 开始，根据模型服务的限流和稳定性逐步调整。并发过高可能增加 5xx、超时或空响应。
+
+模型名称由实际 OpenAI-compatible 服务决定，当前推荐使用 `gpt-5.4-mini` 兼顾判断质量与运行速度，但它不是 Dingo 的强制依赖。
+
+## 6. 内容有效性 Effectiveness
+
+### 6.1 业务逻辑
+
+内容有效性判断的是一条检索结果记录是否“可读、完整、可用于用户判断”，不判断它是否与 query 相关。
+
+当前不按 `metadata_type` 做差异化处理。也就是说，`paper`、`ebook`、未来新增类型都使用相同的 title、abstract、keywords、author 完整性标准。Venue 是否存在及其来源可信度统一交给权威性指标，避免对 ebook 等类型重复惩罚。
+
+### 6.2 字段权重
+
+单条 result 的分数为：
+
+```text
+Effectiveness =
+ title_score * 0.30
++ abstract_score * 0.50
++ keywords_score * 0.10
++ author_score * 0.10
+```
+
+| 子项 | 权重 | 业务含义 |
+|---|---:|---|
+| `title_score` | 0.30 | 标题是否存在、是否可读 |
+| `abstract_score` | 0.50 | 摘要是否存在、是否可读 |
+| `keywords_score` | 0.10 | 是否至少存在一个非空关键词 |
+| `author_score` | 0.10 | 是否至少存在一个非空作者值；不按作者数量额外加分 |
+
+字段为空时，该字段直接得 0 分。
+
+示例：如果 result 只有非空且无异常的标题，其他字段为空，则 `title_score=1.0`：
+
+```text
+1.0 * 0.30 + 0 + 0 + 0 = 0.30
+```
+
+### 6.3 字段评分逻辑
+
+每个字段先做基础质量判断：
+
+- 为空：0 分。
+- title、abstract、keywords、author 四个字段参与有效性评分并进行缺失检查。venue 不参与有效性加权，也不会因为缺失产生问题；其来源可信度由权威性指标负责。
+- title、abstract、keywords、venue、author 五个字段都会进行异常检查。venue 有值时仍检查 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声。HTML 标签和 Unicode 替换字符 `�` 出现即进入 LLM 二次确认；其他异常字符按规则阈值筛选。乱码筛选还包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。
+- HTML 初筛要求 `<` 后直接出现合法标签名；`< Previous page | Next page >` 等翻页或比较文本不视为 HTML，``、``、``、`` 等真实标签仍进入 LLM 二次确认。
+- 字段非空：基础分为 1 分，不因字符长度、token 数、关键词数量或作者数量增减。
+- 字段存在 HTML 泄漏、乱码、不可见字符、不可读文本或严重特殊字符噪声：规则先召回候选，LLM 确认后降低相应字段分数。
+- 长文本不会获得额外加分，短文本也不会仅因长度被扣分。
+
+`keywords` 只判断是否至少有一个非空值；空列表计为缺失，关键词数量不影响分数。
+
+`venue` 读取优先级：
+
+```text
+publication_venue_name_unified
+publication_venue_name
+venue
+source
+```
+
+`author` 兼容 `author`、`authors` 字段以及字符串、对象、对象列表等常见结构。存在至少一个非空作者值时，作者基础分为 1；作者名称长度和作者数量不会影响基础分。乱码、HTML 或特殊字符噪声仍会进入异常检查。
+
+对比不同检索后端时，需要把后端原始作者信息统一映射到 `author` 或 `authors`。未映射作者字段会被视为缺失并使单条 result 的有效性总分降低 `0.10`。
+
+### 6.4 Issues 类型
+
+| issue | 含义 |
+|---|---|
+| `missing_title` | 标题为空 |
+| `missing_abstract` | 摘要为空 |
+| `missing_keywords` | 关键词为空 |
+| `missing_author` | 作者为空 |
+| `title:html_tag` / `abstract:html_tag` / `venue:html_tag` / `author:html_tag` | LLM 判断字段中有 HTML/XML 标签泄漏 |
+| `*:mojibake` | LLM 判断字段存在乱码或编码错误 |
+| `*:invisible_char` | LLM 判断字段存在不可见/控制字符 |
+| `*:unreadable_text` | LLM 判断字段整体不可读 |
+| `*:special_char_noise` | LLM 判断字段中特殊字符噪声已经影响阅读 |
+| `llm_quality_parse_error` | LLM 字段质量判断调用或解析失败 |
+
+注意：
+
+- `RuleSpecialCharacter` 和 `RuleInvisibleChar` 只用于快速召回疑似异常字段，不直接作为最终扣分依据。
+- LaTeX、化学符号、单位、希腊字母、`|` 分隔符等正常学术表达不应被 LLM 判为问题。
+- HTML 高亮标签、明显 mojibake、不可见字符、严重乱码会由 LLM 输出字段级 issue，并降低对应字段分数。
+- 分析 bad 样本时建议结合原始 title、abstract、keywords、venue、author 和 `llm_quality_reason` 进行人工抽查。
+
+### 6.5 可调参数
+
+| 参数 | 默认值 | 说明 |
+|---|---:|---|
+| `--top-k` | 10 | 每个 query 评测前 k 条结果 |
+| `--threshold` | 0.15 | query 级 hard bad 阈值 |
+| `--llm-max-tokens` | 1024 | 内容有效性 LLM 字段质量判断的最大 token 数 |
+| `--llm-workers` | 4 | 内容有效性 LLM 二次复核并发数 |
+| `--llm-timeout` | 60 | LLM 请求超时秒数 |
+| `--disable-llm-quality` | false | 关闭 LLM 字段质量判断；保留字段缺失检查，并由规则候选直接触发异常扣分 |
+
+阈值建议：
+
+- `0.15` 适合作 hard bad，只筛严重不可用结果。
+- 如果要发现 metadata 缺失、摘要不足等一般质量问题，可额外关注 `< 0.45` 的 warning 区间。
+
+## 7. 权威性 Authority
+
+### 7.1 业务逻辑
+
+权威性判断检索结果是否具备学术可信度和影响力信号，主要来自：
+
+- 引用数。
+- 高影响引用数。
+- 期刊/会议/来源类型。
+- DOI。
+
+该指标不判断 query 相关性，也不判断内容字段是否完整。它是独立的学术权威信号，不建议单独用于硬判“结果错误”。
+
+### 7.2 字段权重
+
+单条 result 的权威性分数为：
+
+```text
+authority =
+ 0.45 * citation_score
++ 0.20 * influential_citation_score
++ 0.25 * venue_score
++ 0.10 * doi_score
+```
+
+| 子项 | 权重 | 业务含义 |
+|---|---:|---|
+| `citation_score` | 0.45 | 普通引用影响力 |
+| `influential_citation_score` | 0.20 | 高影响引用 |
+| `venue_score` | 0.25 | 来源/期刊/会议可信度 |
+| `doi_score` | 0.10 | 是否具备 DOI 标识 |
+
+### 7.3 Citation 归一化
+
+引用数使用 log 归一化，避免高引用老论文过度碾压。
+
+```text
+citation_score = log(1 + citation_count) / log(1 + 500)
+
+influential_citation_score =
+ log(1 + influential_citation_count) / log(1 + 50)
+```
+
+分数会被限制在 `[0, 1]`。
+
+### 7.4 Venue 评分
+
+`venue` 读取优先级：
+
+```text
+publication_venue_name_unified
+publication_venue_name
+venue
+source
+```
+
+当前规则：
+
+| 条件 | `venue_score` | reason |
+|---|---:|---|
+| 已知 repository 或 preprint（优先判断） | 0.45 | `repository_or_preprint` |
+| 明确的学术 book series 或 ebook platform（优先判断） | 0.55 | `academic_book_series` |
+| 明确命中权威期刊/会议家族 | 0.85 | `prestigious_venue_family` |
+| 命中正规学术出版机构或来源组织 | 0.75 | `recognized_scholarly_publisher_or_venue` |
+| `publication_venue_type` 是 journal/conference，或存在有效 ISSN | 0.65 | `structured_journal_or_conference` |
+| 只有非空来源名称 | 0.40 | `named_venue` |
+| 未知或低信号来源 | 0.25 | `unknown_or_low_signal_venue` |
+
+权威期刊家族使用带边界的规则匹配，包括：
+
+```text
+Nature 及 Nature 学科子刊、Nature Communications、npj 系列、
+Communications 系列、Scientific Reports/Data、Science 官方期刊家族、
+部分 Cell Press 旗舰刊、NEJM、Lancet、JAMA、The BMJ、PNAS、
+JACS、PRL 以及主要 AI 会议
+```
+
+匹配前会移除 HTML 高亮标签，并按 `|` 拆分中英文来源别名。规则不再使用无限制的普通子串：`Science Translational Medicine` 会命中 Science 家族，而 `Chemical Engineering Science` 不会。
+
+正规出版机构包括 Springer Nature、Elsevier、Wiley、Oxford University Press、Cambridge University Press、IEEE、ACM、ACS、RSC、IOP、BMJ、PLOS、Royal Society、De Gruyter、CRC Press、World Scientific 等。出版商只能提供正规学术来源信号，因此分数低于明确命中的旗舰期刊。
+
+ISSN 和 journal/conference 类型是防止白名单漏判的主要兜底：专业期刊即使不在硬编码刊名列表中，只要具有结构化来源信息，也能获得 0.65，而不是被直接判成低信号来源。
+
+### 7.5 DOI 评分
+
+```text
+doi_score = 1.0
+```
+
+条件：
+
+- result 有 `doi` 字段；或
+- `locations` 中包含 `doi.org`。
+
+否则：
+
+```text
+doi_score = 0.0
+```
+
+### 7.6 使用注意
+
+权威性对以下 query 类型可能偏保守：
+
+- 人名，例如 `Michael Pecht`、`张文宏`。
+- 泛词，例如 `Jerry`、`pam`。
+- 书籍、访谈、百科式结果。
+- 缺少 citation、DOI、venue 元数据但实际有用的结果。
+
+因此，权威性低不一定表示检索结果不相关，只表示该结果缺少学术权威信号。
+
+## 8. 统一阈值与分类
+
+综合脚本不计算 overall，也不为三个指标设置权重。默认统一阈值为 `0.15`：
+
+```text
+relevance < 0.15 → SEARCH_RESULT_RELEVANCE_LOW
+effectiveness < 0.15 → SEARCH_RESULT_EFFECTIVENESS_LOW
+authority < 0.15 → SEARCH_RESULT_AUTHORITY_LOW
+```
+
+三个 query 级聚合指标均不低于阈值时才判为 good。同一 query 可以同时命中多个低分标签。空结果的三个指标均记为 `0`，因此会同时进入三类低分文件，不再产生额外的 `EMPTY` bad 类型。LLM 解析错误保留在诊断字段中，但不会形成第四种 bad 分类。
+
+## 9. 使用命令
+
+以下命令均在项目根目录执行。
+
+综合脚本启动时会自动读取项目根目录 `.env`，且不会覆盖已经存在的系统环境变量。可复制 `.env.example` 的字段结构创建本地 `.env`；`.env` 已被 `.gitignore` 忽略，禁止提交真实 token 或 key。
+
+### 内置测试数据
+
+仓库提供了 `test/data/test_search_result.jsonl`，用于在提交代码前快速验证单指标和综合评测流程。该文件包含3条 query、9条 result：
+
+| Query | 场景 |
+|---|---|
+| `BiMLP` | 正常学术论文结果 |
+| `海带` | 元数据中包含检索高亮 HTML |
+| `pam` | 字段稀疏的 ebook 结果 |
+
+综合 smoke test：
+
+```powershell
+python examples/retrieval/sdk_eval_search_result.py `
+ --input-jsonl test/data/test_search_result.jsonl `
+ --output-dir outputs/search_result_quality_smoke `
+ --top-k 3 `
+ --llm-max-tokens 1024 `
+ --effectiveness-llm-max-tokens 1024 `
+ --threshold 0.15 `
+ --save-good
+```
+
+相关性、有效性和综合 smoke test 需要预先设置 OpenAI-compatible 环境变量；权威性是纯规则评测，不需要 LLM API。
+
+综合脚本只生成 query 级分类目录：
+
+```text
+/
+├── bad/
+│ └── QUALITY_BAD/
+│ ├── SEARCH_RESULT_RELEVANCE_LOW.jsonl
+│ ├── SEARCH_RESULT_EFFECTIVENESS_LOW.jsonl
+│ └── SEARCH_RESULT_AUTHORITY_LOW.jsonl
+└── good/ # 仅使用 --save-good 时生成
+ └── QUALITY_GOOD/
+ └── SEARCH_RESULT_METRICS_PASS.jsonl
+```
+
+- 每行是一个唯一 query，包含三个排名加权平均分、分类 label 和完整 `results`。
+- 每个 result 的 `_evaluation` 字段记录 rank 和三个单条分数。
+- 同一 query 的多个聚合指标低于阈值时，会分别写入对应的低分文件。
+- result 级细分问题仍保留在 `all_results.jsonl` 的 `eval_details` 中。
+
+### 9.1 单独跑相关性
+
+```bash
+export OPENAI_API_KEY=""
+export OPENAI_BASE_URL=""
+export OPENAI_MODEL="gpt-5.4-mini"
+export OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_relevancy.py \
+ --input-jsonl outputs/meta_search_97_query_results.jsonl \
+ --output-dir outputs/search_result_relevancy_97q \
+ --top-k 10 \
+ --threshold 0.15 \
+ --llm-max-tokens 1024 \
+ --llm-workers 3 \
+ --llm-timeout 60 \
+ --save-good
+```
+
+Windows PowerShell 示例：
+
+```powershell
+$env:OPENAI_API_KEY=""
+$env:OPENAI_BASE_URL=""
+$env:OPENAI_MODEL="gpt-5.4-mini"
+$env:OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_relevancy.py `
+ --input-jsonl outputs/meta_search_97_query_results.jsonl `
+ --output-dir outputs/search_result_relevancy_97q `
+ --top-k 10 `
+ --threshold 0.15 `
+ --llm-max-tokens 1024 `
+ --llm-workers 3 `
+ --llm-timeout 60 `
+ --save-good
+```
+
+### 9.2 单独跑内容有效性
+
+```bash
+export OPENAI_API_KEY=""
+export OPENAI_BASE_URL=""
+export OPENAI_MODEL="gpt-5.4-mini"
+export OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_effectiveness.py \
+ --input-jsonl outputs/meta_search_97_query_results.jsonl \
+ --output-dir outputs/search_result_effectiveness_97q \
+ --top-k 10 \
+ --threshold 0.15 \
+ --llm-max-tokens 1024 \
+ --llm-workers 8 \
+ --llm-timeout 60 \
+ --save-good
+```
+
+### 9.3 单独跑权威性
+
+```bash
+python examples/retrieval/sdk_eval_authority.py \
+ --input-jsonl outputs/meta_search_97_query_results.jsonl \
+ --output-dir outputs/search_result_authority_97q \
+ --top-k 10 \
+ --threshold 0.15 \
+ --save-good
+```
+
+### 9.4 评测预计算结果
+
+```bash
+export OPENAI_API_KEY=""
+export OPENAI_BASE_URL=""
+export OPENAI_MODEL="gpt-5.4-mini"
+export OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_search_result.py \
+ --input-jsonl outputs/meta_search_97_query_results.jsonl \
+ --output-dir outputs/search_result_quality_97q \
+ --top-k 10 \
+ --threshold 0.15 \
+ --llm-max-tokens 1024 \
+ --effectiveness-llm-max-tokens 512 \
+ --llm-timeout 60 \
+ --save-good
+```
+
+### 9.5 端到端评测 SciVerse Meta Search
+
+```powershell
+$env:SCIVERSE_API_TOKEN=""
+$env:SEARCH_API_URL="https://api.sciverse.space/meta-search"
+$env:OPENAI_API_KEY=""
+$env:OPENAI_BASE_URL=""
+$env:OPENAI_MODEL="gpt-5.4-mini"
+$env:OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_search_result.py `
+ --input-queries outputs/query.txt `
+ --retrieval-backend meta_search `
+ --output-dir outputs/meta_search_end_to_end_97q `
+ --top-k 10 `
+ --threshold 0.15 `
+ --search-workers 4 `
+ --llm-workers 4 `
+ --batch-size 10
+```
+
+### 9.6 端到端评测 OpenAlex
+
+OpenAlex 默认使用普通 `search`，不需要 API key；如有 key，可设置 `OPENALEX_API_KEY`。
+
+```powershell
+$env:OPENALEX_API_KEY=""
+$env:SEARCH_API_URL="https://api.openalex.org/works"
+
+python examples/retrieval/sdk_eval_search_result.py `
+ --input-queries outputs/query.txt `
+ --retrieval-backend openalex `
+ --output-dir outputs/openalex_end_to_end_97q `
+ --top-k 10 `
+ --threshold 0.15 `
+ --search-workers 4 `
+ --llm-workers 4 `
+ --batch-size 10
+```
+
+## 10. 阈值解释
+
+当前默认统一阈值为：
+
+```text
+0.15
+```
+
+该阈值的含义是 hard bad，即只标记严重问题：
+
+- 相关性几乎不匹配。
+- 内容字段严重缺失或不可读。
+- 权威信号极弱。
+
+分析建议：
+
+| 分数段 | 建议解释 |
+|---|---|
+| `< 0.15` | hard bad，优先人工排查 |
+| `0.15 - 0.30` | 低分边缘，适合抽样检查 |
+| `0.30 - 0.45` | 一般质量问题，尤其适合看 metadata 缺失 |
+| `>= 0.45` | 通常可接受，但仍需结合业务 query 类型 |
+
+不同指标的阈值敏感性不同：
+
+- 相关性 `0.15` 适合筛严重错召回；如果要分析一般低相关，可关注 `< 0.30`。
+- 内容有效性 `0.15` 很宽松；如果要推动元数据补全，可关注 `< 0.45`。
+- 权威性 `0.15` 不宜轻易提高太多，因为很多人名、书籍、访谈、非标准论文结果天然 citation/DOI/venue 信号弱。
+
+## 11. 常见分析方法
+
+### 11.1 查看 query 级低分
+
+```powershell
+Import-Csv outputs/search_result_relevancy_97q/query_scores.csv |
+ Sort-Object {[double]$_.relevance} |
+ Select-Object -First 20 query,relevance,error_count,label
+```
+
+### 11.2 查看 result 级有效性 issues
+
+```powershell
+Import-Csv outputs/search_result_effectiveness_97q/result_scores.csv |
+ Where-Object {$_.issues -ne ""} |
+ Select-Object query,rank,title,Effectiveness,issues
+```
+
+### 11.3 查看权威性低分原因
+
+```powershell
+Import-Csv outputs/search_result_authority_97q/result_scores.csv |
+ Sort-Object {[double]$_.authority} |
+ Select-Object -First 20 query,rank,title,authority,citation_score,influential_citation_score,venue_score,doi_score,reason
+```
+
+### 11.4 区分相关性低分和解析错误
+
+```powershell
+Import-Csv outputs/search_result_relevancy_97q/query_scores.csv |
+ Where-Object {[double]$_.relevance -lt 0.15} |
+ Select-Object query,relevance,error_count,label
+```
+
+```powershell
+Import-Csv outputs/search_result_relevancy_97q/query_scores.csv |
+ Where-Object {[int]$_.error_count -gt 0} |
+ Select-Object query,relevance,error_count,label
+```
+
+## 12. 相关代码位置
+
+评测器：
+
+- `dingo/model/llm/llm_search_result_relevance.py`
+- `dingo/model/llm/llm_search_result_effectiveness.py`
+- `dingo/model/llm/llm_search_result_authority.py`
+
+脚本：
+
+- `examples/retrieval/sdk_eval_relevancy.py`
+- `examples/retrieval/sdk_eval_effectiveness.py`
+- `examples/retrieval/sdk_eval_authority.py`
+- `examples/retrieval/sdk_eval_search_result.py`
+- `examples/retrieval/search_result_eval_utils.py`
+
+## 13. 已知注意事项
+
+1. 相关性使用 LLM，temperature 大于 0 时，同一批数据重跑可能有轻微分数波动。
+2. LLM 相关性结果可能出现 JSON 解析失败，脚本会保留解析错误诊断；该条分数为低分时统一归入 `SEARCH_RESULT_RELEVANCE_LOW`。
+3. 内容有效性当前不按 `metadata_type` 放宽 title、abstract、keywords、author 的字段要求；venue 缺失不再降低有效性分数，由权威性指标统一判断。
+4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。
+5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。
diff --git a/docs/search_result_relevance_executor.md b/docs/search_result_relevance_executor.md
new file mode 100644
index 00000000..12cef9b7
--- /dev/null
+++ b/docs/search_result_relevance_executor.md
@@ -0,0 +1,59 @@
+# Search Result Relevance Executor Notes
+
+`examples/retrieval/sdk_eval_relevancy.py` evaluates each query-result pair with `LLMSearchResultRelevance` through Dingo `LocalExecutor`.
+
+## Content Issues
+
+`Relevance.Error_Content_Issues` now uses a strict standard. It is intended for severe visible content corruption, not for ordinary search-result incompleteness.
+
+The LLM prompt asks `content_issues=true` only when the visible title/content has severe problems such as:
+
+- mojibake or garbled text
+- raw HTML/XML tag residue
+- parser residue that materially hurts readability
+- invisible or control characters
+- unreadable text
+
+The evaluator also applies a deterministic evidence filter after the LLM response. Even if the LLM returns `content_issues=true`, the final executor label `Relevance.Error_Content_Issues` is emitted only when the result text contains supporting evidence.
+
+The following should not by itself trigger `Relevance.Error_Content_Issues`:
+
+- missing abstract
+- short snippet
+- truncated preview
+- title-only result, if the title is still readable enough to judge relevance
+
+In `EvalDetail.reason`, the output keeps both:
+
+- `raw_content_issues`: the original LLM boolean
+- `content_issues`: the post-filtered boolean used for final labels
+- `content_issue_evidence`: the matched evidence list
+
+This keeps the LLM signal available for analysis while preventing ordinary truncation or sparse metadata from making an otherwise relevant result bad.
+
+## Test Data
+
+The repository includes `test/data/test_search_result.jsonl` for local smoke tests. It contains three queries and nine results:
+
+| Query | Covered scenario |
+|---|---|
+| `BiMLP` | Normal academic paper results |
+| `海带` | Search-highlight HTML in result metadata |
+| `pam` | Sparse ebook results |
+
+With the OpenAI-compatible environment variables configured, run:
+
+For full-dataset evaluation, prefer a low-latency Flash model such as `deepseek-v4-flash`. A Pro model is better reserved for reviewing a small number of ambiguous samples because pointwise relevance evaluation makes approximately `query count x top-k` LLM calls. Keep the model and prompt fixed and use temperature `0` when comparing search versions.
+
+```powershell
+$env:OPENAI_MODEL="deepseek-v4-flash"
+$env:OPENAI_TEMPERATURE="0"
+
+python examples/retrieval/sdk_eval_relevancy.py `
+ --input-jsonl test/data/test_search_result.jsonl `
+ --output-dir outputs/search_result_relevancy_smoke `
+ --top-k 3 `
+ --llm-max-tokens 1024 `
+ --threshold 0.15 `
+ --save-good
+```
diff --git a/examples/dataset/example_md.py b/examples/dataset/example_md.py
new file mode 100644
index 00000000..3279e6e5
--- /dev/null
+++ b/examples/dataset/example_md.py
@@ -0,0 +1,69 @@
+import tempfile
+from pathlib import Path
+
+from dingo.config import InputArgs
+from dingo.exec import Executor
+
+
+def run_md_single_file_demo():
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = Path(tmp_dir)
+ md_file = tmp_path / "single.md"
+ md_file.write_text("# Single File Demo\n\nThis is markdown content.:", encoding="utf-8")
+
+ input_data = {
+ "input_path": str(md_file),
+ "dataset": {
+ "source": "local",
+ "format": "md",
+ },
+ "evaluator": [
+ {
+ "fields": {"id": "id", "content": "content"},
+ "evals": [
+ {"name": "RuleColonEnd"},
+ ],
+ }
+ ],
+ }
+
+ input_args = InputArgs(**input_data)
+ executor = Executor.exec_map["local"](input_args)
+ result = executor.execute()
+ print("single file demo:")
+ print(result)
+
+
+def run_md_directory_demo():
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_path = Path(tmp_dir)
+ (tmp_path / "a.md").write_text("## A\n\nalpha content:", encoding="utf-8")
+ (tmp_path / "b.md").write_text("## B\n\nbeta content:", encoding="utf-8")
+ (tmp_path / "ignore.txt").write_text("this file will be ignored in md format", encoding="utf-8")
+
+ input_data = {
+ "input_path": str(tmp_path),
+ "dataset": {
+ "source": "local",
+ "format": "md",
+ },
+ "evaluator": [
+ {
+ "fields": {"id": "id", "content": "content"},
+ "evals": [
+ {"name": "RuleColonEnd"},
+ ],
+ }
+ ],
+ }
+
+ input_args = InputArgs(**input_data)
+ executor = Executor.exec_map["local"](input_args)
+ result = executor.execute()
+ print("directory demo:")
+ print(result)
+
+
+if __name__ == "__main__":
+ run_md_single_file_demo()
+ run_md_directory_demo()
diff --git a/examples/guobiao/rule_content_consistency.py b/examples/guobiao/rule_content_consistency.py
new file mode 100644
index 00000000..e5edb4c1
--- /dev/null
+++ b/examples/guobiao/rule_content_consistency.py
@@ -0,0 +1,50 @@
+"""Evaluate text items using the national-standard content-consistency rule.
+
+Optional dependencies:
+ conda run -n dingo pip install "dingo-python[hhem]"
+
+The first run downloads the configured Hugging Face model unless it is already
+available locally.
+"""
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0206_ContentConsistency
+
+
+def main():
+ data = Data(
+ data_id="guobiao-content-consistency-example",
+ data_content=[
+ {
+ "media_type": "text",
+ "content": "高血压患者的日常健康管理",
+ },
+ {
+ "media_type": "text",
+ "content": (
+ "高血压患者应遵医嘱规律用药，并定期监测血压。"
+ "日常生活中还应注意低盐饮食和适量运动。"
+ ),
+ },
+ {
+ "media_type": "image",
+ "content": "../data/images/blood-pressure.jpg",
+ },
+ ],
+ )
+
+ Rule_TC609_0206_ContentConsistency.dynamic_config = EvaluatorRuleArgs(
+ threshold=0.5,
+ model=(
+ "sentence-transformers/"
+ "paraphrase-multilingual-MiniLM-L12-v2"
+ ),
+ device=-1,
+ )
+ result = Rule_TC609_0206_ContentConsistency.eval(data)
+ print(result)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/guobiao/rule_doc_completeness.py b/examples/guobiao/rule_doc_completeness.py
new file mode 100644
index 00000000..bb24f4f5
--- /dev/null
+++ b/examples/guobiao/rule_doc_completeness.py
@@ -0,0 +1,29 @@
+"""Evaluate one dataset document using a guobiao completeness rule.
+
+Optional dependencies:
+ conda run -n dingo pip install "dingo-python[hhem]"
+
+The first run downloads the configured Hugging Face model unless it is already
+available locally.
+"""
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0101_DocBasicInfoCompleteness
+
+
+def main():
+ data = Data(
+ data_id="guobiao-doc-basic-info-example",
+ content="本数据集说明文档包含数据集规模与样本数量说明，提供格式规范、文件结构、访问渠道和技术支持方式。"
+ )
+
+ Rule_TC609_0101_DocBasicInfoCompleteness.dynamic_config = EvaluatorRuleArgs(
+ threshold=0.8,
+ )
+ result = Rule_TC609_0101_DocBasicInfoCompleteness.eval(data)
+ print(result)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/guobiao/rule_text_perplexity.py b/examples/guobiao/rule_text_perplexity.py
new file mode 100644
index 00000000..75501930
--- /dev/null
+++ b/examples/guobiao/rule_text_perplexity.py
@@ -0,0 +1,32 @@
+"""Evaluate one Chinese text using the national-standard perplexity rule.
+
+Optional dependencies:
+ conda run -n dingo pip install "dingo-python[hhem]"
+
+The first run downloads the configured Hugging Face model unless it is already
+available locally. Set ``MODEL_NAME`` to a local model directory to avoid a
+download.
+"""
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_02080101_TextPerplexity
+
+
+def main():
+ data = Data(
+ data_id="guobiao-ppl-example",
+ content="人工智能正在推动科学研究和产业应用快速发展。高质量数据集能够为模型训练提供准确、完整且具有代表性的样本，从而提高模型在真实应用场景中的稳定性和可靠性。",
+ )
+
+ Rule_TC609_02080101_TextPerplexity.dynamic_config = EvaluatorRuleArgs(
+ threshold=100.0,
+ model="uer/gpt2-chinese-cluecorpussmall",
+ stride=512,
+ )
+ result = Rule_TC609_02080101_TextPerplexity.eval(data)
+ print(result)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/guobiao/rule_time_range.py b/examples/guobiao/rule_time_range.py
new file mode 100644
index 00000000..875a61db
--- /dev/null
+++ b/examples/guobiao/rule_time_range.py
@@ -0,0 +1,24 @@
+"""Evaluate one sample using the guobiao data-time-range rule."""
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0303_DataTimeRange
+
+
+def main():
+ data = Data(
+ data_id="guobiao-time-range-example",
+ dt="2025-03-01 10:30:00",
+ content="示例数据",
+ )
+
+ Rule_TC609_0303_DataTimeRange.dynamic_config = EvaluatorRuleArgs(
+ dt_start="2025-01-01",
+ dt_end="2025-12-31 23:59:59",
+ )
+ result = Rule_TC609_0303_DataTimeRange.eval(data)
+ print(result)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/guobiao/rule_type_consistency.py b/examples/guobiao/rule_type_consistency.py
new file mode 100644
index 00000000..b07aa07b
--- /dev/null
+++ b/examples/guobiao/rule_type_consistency.py
@@ -0,0 +1,37 @@
+"""Evaluate text content using the national-standard type-consistency rule.
+
+Optional dependencies:
+ conda run -n dingo pip install "dingo-python[hhem]"
+
+The first run downloads the configured Hugging Face model unless it is already
+available locally.
+"""
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0207_DataTypeConsistency
+
+
+def main():
+ data = Data(
+ data_id="guobiao-type-example",
+ data_content=[
+ {
+ "media_type": "text",
+ "content": "高血压患者应在医生指导下规律用药，并定期监测血压变化。",
+ }
+ ],
+ )
+
+ Rule_TC609_0207_DataTypeConsistency.dynamic_config = EvaluatorRuleArgs(
+ dataset_type="行业通识数据集",
+ threshold=0.5,
+ model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli",
+ device=-1,
+ )
+ result = Rule_TC609_0207_DataTypeConsistency.eval(data)
+ print(result)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/hallucination/sdk_rule_hhem_detection.py b/examples/hallucination/sdk_rule_hhem_detection.py
index 576fbdc6..1d2832dd 100644
--- a/examples/hallucination/sdk_rule_hhem_detection.py
+++ b/examples/hallucination/sdk_rule_hhem_detection.py
@@ -1,15 +1,19 @@
"""
-Dingo Rule-based HHEM-2.1-Open Hallucination Detection Example
+Dingo Rule-based MiniCheck Hallucination Detection Example
-This example demonstrates how to use the HHEM-2.1-Open model as a rule-based
-hallucination detection tool for efficient local inference without API costs.
+This example demonstrates how to use the RuleHallucinationHHEM rule (backed by
+the MiniCheck-Flan-T5-Large model) as a local, API-free tool for detecting
+ungrounded claims — i.e. whether a response is supported by its context.
-Rule-based HHEM offers:
+Rule-based MiniCheck offers:
- Better architecture fit (rules vs LLM for deterministic local models)
-- Superior performance than GPT-3.5 and GPT-4 on benchmarks
-- Local inference with <600MB RAM usage
-- Fast processing (~1.5s for 2k tokens on modern CPU)
-- No API costs or rate limits
+- Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8)
+- Standard T5 model — no transformers version pin, no remote code
+- Local inference, no API costs or rate limits
+
+Note: the rule/class is still named RuleHallucinationHHEM for backward
+compatibility; the underlying model was upgraded from Vectara HHEM-2.1-Open
+to MiniCheck.
"""
from dingo.io.input import Data
@@ -35,7 +39,7 @@ def example_1_basic_rule_hhem_detection():
print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination
print(f"Label: {result.label}")
- print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}")
+ print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}")
print(f"Threshold: {RuleHallucinationHHEM.dynamic_config.threshold}")
print("\nDetailed Analysis:")
print(result.reason[0] if result.reason else "N/A")
@@ -61,7 +65,7 @@ def example_2_no_hallucination_rule():
print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination
print(f"Label: {result.label}")
- print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}")
+ print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}")
print("\nDetailed Analysis:")
print(result.reason[0] if result.reason else "N/A")
print()
@@ -89,7 +93,7 @@ def example_3_complex_scenario_rule():
print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination
print(f"Label: {result.label}")
- print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}")
+ print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}")
print("\nDetailed Analysis:")
print(result.reason[0] if result.reason else "N/A")
print()
@@ -150,7 +154,7 @@ def example_5_batch_evaluation_rule():
print("Batch Rule-based Evaluation Results:")
for i, result in enumerate(results):
- print(f" Item {i + 1}: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}")
+ print(f" Item {i + 1}: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}")
print()
@@ -176,7 +180,7 @@ def example_6_threshold_comparison_rule():
RuleHallucinationHHEM.dynamic_config.threshold = threshold
result = RuleHallucinationHHEM.eval(data)
- print(f"Threshold {threshold}: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}")
+ print(f"Threshold {threshold}: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}")
# Restore original threshold
RuleHallucinationHHEM.dynamic_config.threshold = original_threshold
@@ -204,27 +208,26 @@ def example_7_performance_benchmark_rule():
result = RuleHallucinationHHEM.eval(data)
end_time = time.time()
- print(f"Rule-based HHEM Inference Time: {end_time - start_time:.3f} seconds")
- print(f"Result: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}")
- print(f"Model Info: Local HHEM-2.1-Open (Rule-based)")
+ print(f"Rule-based MiniCheck Inference Time: {end_time - start_time:.3f} seconds")
+ print(f"Result: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}")
+ print(f"Model Info: Local MiniCheck-Flan-T5-Large (Rule-based)")
print()
if __name__ == "__main__":
- print("🔍 Dingo Rule-based HHEM-2.1-Open Hallucination Detection Examples")
+ print("🔍 Dingo Rule-based MiniCheck Hallucination Detection Examples")
print("=" * 70)
print()
- print("💡 Rule-based HHEM-2.1-Open Advantages:")
+ print("💡 Rule-based MiniCheck Advantages:")
print("- Better architecture: Rules for deterministic local models")
print("- Local inference (no API costs)")
- print("- High performance (better than GPT-3.5/GPT-4 on benchmarks)")
- print("- Low resource usage (<600MB RAM)")
- print("- Fast inference (~1.5s for 2k tokens)")
+ print("- Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8)")
+ print("- Standard T5 model (no transformers version pin, no remote code)")
print()
- print("⚠️ First run will download the model (~400MB)")
- print("⚠️ Requires: pip install transformers")
+ print("⚠️ First run will download the model (~3GB, flan-t5-large)")
+ print("⚠️ Requires: pip install transformers torch sentencepiece")
print()
try:
@@ -236,11 +239,11 @@ def example_7_performance_benchmark_rule():
example_6_threshold_comparison_rule()
example_7_performance_benchmark_rule()
- print("🎉 All Rule-based HHEM examples completed successfully!")
+ print("🎉 All Rule-based MiniCheck examples completed successfully!")
except ImportError as e:
print(f"❌ Import Error: {e}")
- print("Please install required dependencies: pip install transformers")
+ print("Please install required dependencies: pip install transformers torch sentencepiece")
except Exception as e:
print(f"❌ Error: {e}")
print("Please check your setup and try again")
@@ -248,8 +251,7 @@ def example_7_performance_benchmark_rule():
print()
print("📈 Rule-based vs LLM-based Comparison:")
print("- Architecture: Rule-based (✓) vs LLM-based (for API models)")
- print("- Performance: HHEM-2.1 > GPT-4 > GPT-3.5")
+ print("- Accuracy: MiniCheck competitive with much larger LLM fact-checkers")
print("- Cost: Rule-based (Free) vs LLM-based (API costs)")
- print("- Speed: Rule-based (~1.5s) vs LLM-based (3-10s)")
print("- Privacy: Rule-based (Local) vs LLM-based (Cloud)")
- print("- Resource: Rule-based (<600MB) vs LLM-based (API dependency)")
+ print("- Dependency: Rule-based (Local model) vs LLM-based (API dependency)")
diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py
new file mode 100644
index 00000000..2c4a7495
--- /dev/null
+++ b/examples/retrieval/sdk_eval_authority.py
@@ -0,0 +1,103 @@
+"""Run search result authority evaluation through Dingo LocalExecutor."""
+
+from __future__ import annotations
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402
+
+from dingo.config import InputArgs # noqa: E402
+from dingo.exec import Executor # noqa: E402
+
+EVALUATOR_NAME = "LLMSearchResultAuthority"
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Evaluate search result authority with Dingo executor.")
+ add_common_args(parser)
+ parser.add_argument("--threshold", type=float, default=0.15)
+ parser.add_argument("--max-workers", type=int, default=4)
+ parser.add_argument("--batch-size", type=int, default=10)
+ parser.add_argument(
+ "--raw-output",
+ action="store_true",
+ help="Write executor records with raw data merged into top-level JSONL rows.",
+ )
+ return parser.parse_args()
+
+
+def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int:
+ items = load_query_result_jsonl(input_path, max_queries)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ count = 0
+ with output_path.open("w", encoding="utf-8") as f:
+ for query_index, item in enumerate(items, start=1):
+ query = item["query"]
+ for rank, result in enumerate(item["results"][:top_k], start=1):
+ row: dict[str, Any] = {
+ "query": query,
+ "query_index": query_index,
+ "rank": rank,
+ "title": get_title(result),
+ "search_result": result,
+ }
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
+ count += 1
+ return count
+
+
+def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]:
+ return {
+ "task_name": "search_result_authority",
+ "input_path": str(flattened_path),
+ "output_path": str(args.output_dir),
+ "dataset": {"source": "local", "format": "jsonl"},
+ "executor": {
+ "max_workers": args.max_workers,
+ "batch_size": args.batch_size,
+ "result_save": {
+ "bad": True,
+ "good": args.save_good,
+ "all_labels": True,
+ "raw": args.raw_output,
+ },
+ },
+ "evaluator": [
+ {
+ "fields": {"search_result": "search_result"},
+ "evals": [{"name": EVALUATOR_NAME, "config": {"threshold": args.threshold}}],
+ }
+ ],
+ }
+
+
+def main() -> None:
+ args = parse_args()
+ os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true")
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ flattened_path = args.output_dir / "meta_search_flattened_authority_input.jsonl"
+ total = flatten_query_results(
+ args.input_jsonl,
+ flattened_path,
+ top_k=args.top_k,
+ max_queries=args.max_queries,
+ )
+ if total == 0:
+ raise ValueError("No search results found to evaluate.")
+
+ summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute()
+ print(summary)
+ print(f"[Done] flattened_input={flattened_path.resolve()}")
+ print(f"[Done] executor_output={summary.output_path}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py
new file mode 100644
index 00000000..8ce3f0d7
--- /dev/null
+++ b/examples/retrieval/sdk_eval_effectiveness.py
@@ -0,0 +1,125 @@
+"""Run search result effectiveness evaluation through Dingo LocalExecutor."""
+
+from __future__ import annotations
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402
+
+from dingo.config import InputArgs # noqa: E402
+from dingo.exec import Executor # noqa: E402
+
+EVALUATOR_NAME = "LLMSearchResultEffectiveness"
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Evaluate search result effectiveness with Dingo executor.")
+ add_common_args(parser)
+ parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY"))
+ parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL"))
+ parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o"))
+ parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0")))
+ parser.add_argument("--llm-max-tokens", type=int, default=512)
+ parser.add_argument("--llm-workers", type=int, default=4)
+ parser.add_argument("--llm-timeout", type=float, default=60.0)
+ parser.add_argument("--batch-size", type=int, default=10)
+ parser.add_argument("--threshold", type=float, default=0.15)
+ parser.add_argument(
+ "--disable-llm-quality",
+ action="store_true",
+ help="Disable LLM second judgment for abnormal-character candidates.",
+ )
+ parser.add_argument(
+ "--raw-output",
+ action="store_true",
+ help="Write executor records with raw data merged into top-level JSONL rows.",
+ )
+ return parser.parse_args()
+
+
+def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int:
+ items = load_query_result_jsonl(input_path, max_queries)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ count = 0
+ with output_path.open("w", encoding="utf-8") as f:
+ for query_index, item in enumerate(items, start=1):
+ query = item["query"]
+ for rank, result in enumerate(item["results"][:top_k], start=1):
+ row: dict[str, Any] = {
+ "query": query,
+ "query_index": query_index,
+ "rank": rank,
+ "title": get_title(result),
+ "search_result": result,
+ }
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
+ count += 1
+ return count
+
+
+def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]:
+ llm_config: dict[str, Any] = {
+ "model": args.openai_model,
+ "key": args.openai_api_key,
+ "api_url": args.openai_base_url,
+ "temperature": args.openai_temperature,
+ "max_tokens": args.llm_max_tokens,
+ "timeout": args.llm_timeout,
+ "threshold": args.threshold,
+ "enable_llm_quality": not args.disable_llm_quality,
+ }
+ return {
+ "task_name": "search_result_effectiveness",
+ "input_path": str(flattened_path),
+ "output_path": str(args.output_dir),
+ "dataset": {"source": "local", "format": "jsonl"},
+ "executor": {
+ "max_workers": args.llm_workers,
+ "batch_size": args.batch_size,
+ "result_save": {
+ "bad": True,
+ "good": args.save_good,
+ "all_labels": True,
+ "raw": args.raw_output,
+ },
+ },
+ "evaluator": [
+ {
+ "fields": {"search_result": "search_result"},
+ "evals": [{"name": EVALUATOR_NAME, "config": llm_config}],
+ }
+ ],
+ }
+
+
+def main() -> None:
+ args = parse_args()
+ os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true")
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ flattened_path = args.output_dir / "meta_search_flattened_effectiveness_input.jsonl"
+ total = flatten_query_results(
+ args.input_jsonl,
+ flattened_path,
+ top_k=args.top_k,
+ max_queries=args.max_queries,
+ )
+ if total == 0:
+ raise ValueError("No search results found to evaluate.")
+
+ input_data = build_input_data(args, flattened_path)
+ summary = Executor.exec_map["local"](InputArgs(**input_data)).execute()
+ print(summary)
+ print(f"[Done] flattened_input={flattened_path.resolve()}")
+ print(f"[Done] executor_output={summary.output_path}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py
new file mode 100644
index 00000000..6f37306d
--- /dev/null
+++ b/examples/retrieval/sdk_eval_relevancy.py
@@ -0,0 +1,124 @@
+"""Run search result relevancy evaluation through Dingo LocalExecutor."""
+
+from __future__ import annotations
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402
+
+from dingo.config import InputArgs # noqa: E402
+from dingo.exec import Executor # noqa: E402
+
+EVALUATOR_NAME = "LLMSearchResultRelevance"
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Evaluate search result relevancy with Dingo executor.")
+ add_common_args(parser)
+ parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY"))
+ parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL"))
+ parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o"))
+ parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0")))
+ parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed")
+ parser.add_argument("--expected-criteria", default=None)
+ parser.add_argument("--llm-max-tokens", type=int, default=1024)
+ parser.add_argument("--llm-workers", type=int, default=4)
+ parser.add_argument("--llm-timeout", type=float, default=60.0)
+ parser.add_argument("--batch-size", type=int, default=10)
+ parser.add_argument("--threshold", type=float, default=0.15)
+ parser.add_argument(
+ "--raw-output",
+ action="store_true",
+ help="Write executor records with raw data merged into top-level JSONL rows.",
+ )
+ return parser.parse_args()
+
+
+def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int:
+ items = load_query_result_jsonl(input_path, max_queries)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ count = 0
+ with output_path.open("w", encoding="utf-8") as f:
+ for query_index, item in enumerate(items, start=1):
+ query = item["query"]
+ for rank, result in enumerate(item["results"][:top_k], start=1):
+ result_payload = dict(result)
+ result_payload["_eval_query"] = query
+ row: dict[str, Any] = {
+ "query": query,
+ "query_index": query_index,
+ "rank": rank,
+ "title": get_title(result),
+ "search_result": result_payload,
+ }
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
+ count += 1
+ return count
+
+
+def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]:
+ llm_config: dict[str, Any] = {
+ "model": args.openai_model,
+ "key": args.openai_api_key,
+ "api_url": args.openai_base_url,
+ "temperature": args.openai_temperature,
+ "prompt_mode": args.prompt_mode,
+ "expected_criteria": args.expected_criteria,
+ "max_tokens": args.llm_max_tokens,
+ "timeout": args.llm_timeout,
+ "threshold": args.threshold,
+ }
+ return {
+ "task_name": "search_result_relevancy",
+ "input_path": str(flattened_path),
+ "output_path": str(args.output_dir),
+ "dataset": {"source": "local", "format": "jsonl"},
+ "executor": {
+ "max_workers": args.llm_workers,
+ "batch_size": args.batch_size,
+ "result_save": {
+ "bad": True,
+ "good": args.save_good,
+ "all_labels": True,
+ "raw": args.raw_output,
+ },
+ },
+ "evaluator": [
+ {
+ "fields": {"search_result": "search_result"},
+ "evals": [{"name": EVALUATOR_NAME, "config": llm_config}],
+ }
+ ],
+ }
+
+
+def main() -> None:
+ args = parse_args()
+ os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true")
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ flattened_path = args.output_dir / "meta_search_flattened_relevancy_input.jsonl"
+ total = flatten_query_results(
+ args.input_jsonl,
+ flattened_path,
+ top_k=args.top_k,
+ max_queries=args.max_queries,
+ )
+ if total == 0:
+ raise ValueError("No search results found to evaluate.")
+
+ summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute()
+ print(summary)
+ print(f"[Done] flattened_input={flattened_path.resolve()}")
+ print(f"[Done] executor_output={summary.output_path}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py
new file mode 100644
index 00000000..9a28be92
--- /dev/null
+++ b/examples/retrieval/sdk_eval_search_result.py
@@ -0,0 +1,703 @@
+"""Evaluate search results with relevance, effectiveness, and authority metrics.
+
+This script uses Dingo LocalExecutor for result-level metric execution, then
+aggregates executor outputs back to query-level CSV/JSON reports.
+"""
+
+from __future__ import annotations
+import argparse
+import concurrent.futures
+import json
+import math
+import os
+import shutil
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_ENV_PATH = PROJECT_ROOT / ".env"
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from search_result_eval_utils import (add_common_args, get_title, load_queries, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, # noqa: E402,E501
+ write_json)
+
+from dingo.config import InputArgs # noqa: E402
+from dingo.exec import Executor # noqa: E402
+from dingo.model.llm.llm_search_result_relevance import is_doi_query # noqa: E402
+from dingo.retrieval.search_client import PaperResult, create_client # noqa: E402
+
+EFFECTIVENESS_LABEL_TO_ISSUE = {
+ "Effectiveness.Error_Title_Miss": "missing_title",
+ "Effectiveness.Error_Abstract_Miss": "missing_abstract",
+ "Effectiveness.Error_Keywords_Miss": "missing_keywords",
+ "Effectiveness.Error_Author_Miss": "missing_author",
+ "Effectiveness.Error_HTML_Tag": "html_tag",
+ "Effectiveness.Error_Mojibake": "mojibake",
+ "Effectiveness.Error_Invisible_Char": "invisible_char",
+ "Effectiveness.Error_Unreadable_Text": "unreadable_text",
+ "Effectiveness.Error_Special_Char_Noise": "special_char_noise",
+ "Effectiveness.Error_LLM_Quality_Parse": "llm_quality_parse_error",
+ "Effectiveness.Error_Effectiveness_Low": "effectiveness_low",
+}
+
+
+def load_env_file(env_path: Path = DEFAULT_ENV_PATH) -> None:
+ """Load an optional local .env without overriding process environment."""
+ if not env_path.exists():
+ return
+ for raw_line in env_path.read_text(encoding="utf-8-sig").splitlines():
+ line = raw_line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ key, value = line.split("=", 1)
+ key = key.strip()
+ value = value.strip().strip('"').strip("'")
+ if key and key not in os.environ:
+ os.environ[key] = value
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description="Evaluate search result quality with Dingo executor.")
+ add_common_args(parser)
+ parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY"))
+ parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL"))
+ parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-5.4-mini"))
+ parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0")))
+ parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed")
+ parser.add_argument("--llm-max-tokens", type=int, default=1024)
+ parser.add_argument("--llm-timeout", type=float, default=60.0)
+ parser.add_argument("--llm-workers", type=int, default=4)
+ parser.add_argument("--batch-size", type=int, default=10)
+ parser.add_argument("--effectiveness-llm-max-tokens", type=int, default=512)
+ parser.add_argument(
+ "--disable-effectiveness-llm-quality",
+ action="store_true",
+ help="Disable LLM readability/corruption judgment for effectiveness.",
+ )
+ parser.add_argument(
+ "--threshold",
+ type=float,
+ default=0.15,
+ help="Unified query-level threshold for rank-weighted relevance, effectiveness, and authority.",
+ )
+ parser.add_argument(
+ "--retrieval-backend",
+ choices=("precomputed", "meta_search", "openalex"),
+ default="precomputed",
+ help="Use precomputed results or retrieve queries before evaluation.",
+ )
+ parser.add_argument(
+ "--search-api-url",
+ default=os.environ.get("SEARCH_API_URL"),
+ help="Search API root or full endpoint; defaults to SEARCH_API_URL.",
+ )
+ parser.add_argument(
+ "--search-api-token",
+ default=None,
+ help="Optional token override; prefer SCIVERSE_API_TOKEN or OPENALEX_API_KEY.",
+ )
+ parser.add_argument("--search-type", default=None)
+ parser.add_argument("--search-timeout", type=float, default=60.0)
+ parser.add_argument("--search-workers", type=int, default=4)
+ parser.add_argument("--search-rate-limit", type=float, default=None)
+ parser.add_argument("--search-max-retries", type=int, default=3)
+ parser.add_argument(
+ "--save-detailed",
+ action="store_true",
+ help="Save detailed_results.json with query-level records and embedded result rows.",
+ )
+ return parser.parse_args()
+
+
+def _openalex_authors(raw: dict[str, Any]) -> list[dict[str, str]]:
+ authors: list[dict[str, str]] = []
+ for authorship in raw.get("authorships") or []:
+ author = authorship.get("author") or {}
+ name = author.get("display_name") or authorship.get("raw_author_name") or ""
+ if name:
+ authors.append({"name": str(name), "orcid": str(author.get("orcid") or "")})
+ return authors
+
+
+def _openalex_keywords(raw: dict[str, Any]) -> list[str]:
+ keywords: list[str] = []
+ for item in raw.get("keywords") or []:
+ value = item.get("display_name") if isinstance(item, dict) else item
+ if value:
+ keywords.append(str(value))
+ return keywords
+
+
+def normalize_search_result(paper: PaperResult, backend: str) -> dict[str, Any]:
+ """Map backend-specific output to the fields consumed by all three metrics."""
+ raw = dict(paper.raw or {})
+ if backend == "meta_search":
+ raw.setdefault("title", paper.title)
+ raw.setdefault("abstract", paper.abstract)
+ raw.setdefault("relevance_score", paper.score)
+ return raw
+
+ primary_location = raw.get("primary_location") or {}
+ source = primary_location.get("source") or {}
+ publisher = source.get("host_organization_name") or ""
+ return {
+ "unique_id": raw.get("id") or paper.paper_id,
+ "title": paper.title,
+ "abstract": paper.abstract,
+ "keywords": _openalex_keywords(raw),
+ "author": _openalex_authors(raw),
+ "doi": raw.get("doi") or "",
+ "citation_count": raw.get("cited_by_count") or 0,
+ "influential_citation_count": 0,
+ "publication_venue_name_unified": source.get("display_name") or "",
+ "publication_venue_type": source.get("type") or "",
+ "publication_venue_issn": source.get("issn") or [],
+ "publication_publisher": [publisher] if publisher else [],
+ "publication_published_year": raw.get("publication_year") or paper.year,
+ "metadata_type": raw.get("type") or "paper",
+ "language": raw.get("language") or "",
+ "relevance_score": paper.score,
+ "access_is_oa": str(bool((raw.get("open_access") or {}).get("is_oa"))).lower(),
+ "openalex_raw": raw,
+ }
+
+
+def _build_search_client(args: argparse.Namespace):
+ kwargs: dict[str, Any] = {
+ "timeout": args.search_timeout,
+ "max_retries": args.search_max_retries,
+ }
+ if args.search_api_token:
+ kwargs["api_token"] = args.search_api_token
+ if args.search_api_url:
+ kwargs["api_url"] = args.search_api_url
+ elif args.retrieval_backend == "meta_search":
+ kwargs["api_url"] = os.environ.get(
+ "SCIVERSE_API_URL", "https://api.sciverse.space"
+ )
+ elif args.retrieval_backend == "openalex":
+ kwargs["api_url"] = os.environ.get(
+ "OPENALEX_API_URL", "https://api.openalex.org"
+ )
+ if args.search_type:
+ kwargs["search_type"] = args.search_type
+ if args.search_rate_limit is not None:
+ kwargs["rate_limit"] = args.search_rate_limit
+ return create_client(args.retrieval_backend, **kwargs)
+
+
+def retrieve_queries(
+ args: argparse.Namespace,
+ output_path: Path,
+ request_log_path: Path,
+) -> dict[str, Any]:
+ queries = load_queries(args.input_jsonl, args.max_queries)
+ if not queries:
+ raise ValueError(f"No queries found in {args.input_jsonl}")
+
+ client = _build_search_client(args)
+
+ def search_one(index_query: tuple[int, str]) -> tuple[int, dict[str, Any], dict[str, Any]]:
+ index, query = index_query
+ response = client.search(query, limit=args.top_k)
+ results = [
+ normalize_search_result(paper, args.retrieval_backend)
+ for paper in response.results[: args.top_k]
+ ]
+ item = {"query": query, "results": results}
+ log = {
+ "query_index": index,
+ "query": query,
+ "status_code": response.status_code,
+ "response_time_ms": round(response.response_time_ms, 3),
+ "result_count": len(results),
+ "error": response.error or "",
+ }
+ return index, item, log
+
+ completed: list[tuple[int, dict[str, Any], dict[str, Any]]] = []
+ with concurrent.futures.ThreadPoolExecutor(max_workers=args.search_workers) as pool:
+ futures = [pool.submit(search_one, item) for item in enumerate(queries, start=1)]
+ for future in concurrent.futures.as_completed(futures):
+ completed.append(future.result())
+ completed.sort(key=lambda item: item[0])
+
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with output_path.open("w", encoding="utf-8") as result_file, request_log_path.open(
+ "w", encoding="utf-8"
+ ) as log_file:
+ for _, item, log in completed:
+ result_file.write(json.dumps(item, ensure_ascii=False) + "\n")
+ log_file.write(json.dumps(log, ensure_ascii=False) + "\n")
+
+ logs = [log for _, _, log in completed]
+ latencies = [float(log["response_time_ms"]) for log in logs]
+ return {
+ "backend": args.retrieval_backend,
+ "query_count": len(logs),
+ "result_count": sum(int(log["result_count"]) for log in logs),
+ "success_count": sum(1 for log in logs if not log["error"]),
+ "error_count": sum(1 for log in logs if log["error"]),
+ "empty_count": sum(1 for log in logs if not log["error"] and not log["result_count"]),
+ "mean_response_time_ms": round(sum(latencies) / len(latencies), 3) if latencies else 0.0,
+ "max_response_time_ms": round(max(latencies), 3) if latencies else 0.0,
+ }
+
+
+def flatten_query_results(
+ input_path: Path,
+ output_path: Path,
+ *,
+ top_k: int,
+ max_queries: int | None,
+) -> tuple[int, list[str]]:
+ items = load_query_result_jsonl(input_path, max_queries)
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ count = 0
+ empty_queries: list[str] = []
+ with output_path.open("w", encoding="utf-8") as f:
+ for query_index, item in enumerate(items, start=1):
+ query = item["query"]
+ if not item["results"][:top_k]:
+ empty_queries.append(query)
+ for rank, result in enumerate(item["results"][:top_k], start=1):
+ result_payload = dict(result)
+ result_payload["_eval_query"] = query
+ row = {
+ "query": query,
+ "query_index": query_index,
+ "rank": rank,
+ "title": get_title(result),
+ "search_result": result_payload,
+ }
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
+ count += 1
+ return count, empty_queries
+
+
+def build_executor_input(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]:
+ relevance_config = {
+ "model": args.openai_model,
+ "key": args.openai_api_key,
+ "api_url": args.openai_base_url,
+ "temperature": args.openai_temperature,
+ "prompt_mode": args.prompt_mode,
+ "max_tokens": args.llm_max_tokens,
+ "timeout": args.llm_timeout,
+ "threshold": args.threshold,
+ }
+ effectiveness_config = {
+ "model": args.openai_model,
+ "key": args.openai_api_key,
+ "api_url": args.openai_base_url,
+ "temperature": args.openai_temperature,
+ "max_tokens": args.effectiveness_llm_max_tokens,
+ "timeout": args.llm_timeout,
+ "threshold": args.threshold,
+ "enable_llm_quality": not args.disable_effectiveness_llm_quality,
+ }
+ authority_config = {"threshold": args.threshold}
+ return {
+ "task_name": "search_result_quality",
+ "input_path": str(flattened_path),
+ "output_path": str(args.output_dir),
+ "dataset": {"source": "local", "format": "jsonl"},
+ "executor": {
+ "max_workers": args.llm_workers,
+ "batch_size": args.batch_size,
+ "result_save": {
+ "bad": True,
+ "good": True,
+ "all_labels": True,
+ "merge": True,
+ },
+ },
+ "evaluator": [
+ {
+ "fields": {"search_result": "search_result"},
+ "evals": [
+ {"name": "LLMSearchResultRelevance", "config": relevance_config},
+ {"name": "LLMSearchResultEffectiveness", "config": effectiveness_config},
+ {"name": "LLMSearchResultAuthority", "config": authority_config},
+ ],
+ }
+ ],
+ }
+
+
+def load_executor_records(executor_output_path: str) -> list[dict[str, Any]]:
+ all_results_path = Path(executor_output_path) / "all_results.jsonl"
+ if not all_results_path.exists():
+ raise FileNotFoundError(f"Executor merged result file not found: {all_results_path}")
+ records = []
+ with all_results_path.open("r", encoding="utf-8") as f:
+ for line in f:
+ if line.strip():
+ records.append(json.loads(line))
+ return records
+
+
+def get_metric_detail(record: dict[str, Any], metric: str) -> dict[str, Any]:
+ details = record.get("eval_details", {}).get("search_result", [])
+ for detail in details:
+ if detail.get("metric") == metric:
+ return detail
+ return {}
+
+
+def first_reason(detail: dict[str, Any]) -> dict[str, Any]:
+ reason = detail.get("reason") or []
+ if reason and isinstance(reason[0], dict):
+ return reason[0]
+ return {}
+
+
+def filtered_effectiveness_issues(detail: dict[str, Any]) -> str:
+ labels = detail.get("label") or []
+ issues = []
+ for label in labels:
+ issue = EFFECTIVENESS_LABEL_TO_ISSUE.get(label)
+ if issue and issue not in issues:
+ issues.append(issue)
+ return "|".join(issues)
+
+
+def build_reports(
+ records: list[dict[str, Any]],
+ args: argparse.Namespace,
+ executor_summary,
+ empty_queries: list[str] | None = None,
+ retrieval_summary: dict[str, Any] | None = None,
+) -> tuple[dict, list[dict], list[dict], list[dict], list[dict]]:
+ records = sorted(
+ records,
+ key=lambda r: (
+ int(r.get("raw_data", {}).get("query_index") or 0),
+ int(r.get("raw_data", {}).get("rank") or 0),
+ ),
+ )
+
+ result_rows = []
+ by_query: dict[str, list[dict[str, Any]]] = {}
+ full_results_by_query: dict[str, list[dict[str, Any]]] = {}
+ rank_relevance_error_count = 0
+ rank_effectiveness_llm_quality_error_count = 0
+
+ for record in records:
+ raw = record.get("raw_data", {})
+ query = raw.get("query", "")
+ relevance_detail = get_metric_detail(record, "LLMSearchResultRelevance")
+ effectiveness_detail = get_metric_detail(record, "LLMSearchResultEffectiveness")
+ authority_detail = get_metric_detail(record, "LLMSearchResultAuthority")
+ relevance_reason = first_reason(relevance_detail)
+ effectiveness_reason = first_reason(effectiveness_detail)
+ authority_reason = first_reason(authority_detail)
+
+ relevance = round(float(relevance_detail.get("score") or 0.0), 5)
+ effectiveness = round(float(effectiveness_detail.get("score") or 0.0), 5)
+ authority = round(float(authority_detail.get("score") or 0.0), 5)
+
+ relevance_error = str(relevance_reason.get("error") or "")
+ effectiveness_error = str(effectiveness_reason.get("llm_quality_error") or "")
+ if relevance_error:
+ rank_relevance_error_count += 1
+ if effectiveness_error:
+ rank_effectiveness_llm_quality_error_count += 1
+
+ row = {
+ "query": query,
+ "rank": raw.get("rank"),
+ "title": raw.get("title", ""),
+ "relevance": relevance,
+ "query_relevance": round(float(relevance_reason.get("query_relevance") or 0.0), 5),
+ "result_quality": round(float(relevance_reason.get("result_quality") or 0.0), 5),
+ "relevance_content_issues": bool(relevance_reason.get("content_issues", False)),
+ "relevance_content_issue_evidence": "|".join(relevance_reason.get("content_issue_evidence") or []),
+ "relevance_error": relevance_error,
+ "relevance_reasoning": relevance_reason.get("reasoning", ""),
+ "effectiveness": effectiveness,
+ "effectiveness_issues": filtered_effectiveness_issues(effectiveness_detail),
+ "effectiveness_llm_quality_reason": effectiveness_reason.get("llm_quality_reason", ""),
+ "effectiveness_llm_quality_error": effectiveness_error,
+ "authority": authority,
+ "citation_score": round(float(authority_reason.get("citation_score") or 0.0), 5),
+ "influential_citation_score": round(float(authority_reason.get("influential_citation_score") or 0.0), 5),
+ "venue_score": round(float(authority_reason.get("venue_score") or 0.0), 5),
+ "doi_score": round(float(authority_reason.get("doi_score") or 0.0), 5),
+ "authority_reason": authority_reason.get("reason", ""),
+ }
+ result_rows.append(row)
+ by_query.setdefault(query, []).append(row)
+ full_result = dict(raw.get("search_result") or {})
+ full_result.pop("_eval_query", None)
+ full_result["_evaluation"] = {
+ "rank": raw.get("rank"),
+ "relevance": relevance,
+ "effectiveness": effectiveness,
+ "authority": authority,
+ }
+ full_results_by_query.setdefault(query, []).append(full_result)
+
+ query_rows = []
+ detailed = []
+ classified_records = []
+ for query, rows in by_query.items():
+ relevance_scores = [float(row["relevance"]) for row in rows]
+ effectiveness_scores = [float(row["effectiveness"]) for row in rows]
+ authority_scores = [float(row["authority"]) for row in rows]
+ if is_doi_query(query):
+ query_relevance = round(
+ max(
+ (
+ float(row["relevance"])
+ / math.log2(max(1, int(row.get("rank") or index)) + 1)
+ )
+ for index, row in enumerate(rows, start=1)
+ ),
+ 5,
+ )
+ relevance_aggregation = "doi_exact_match_rank_discount"
+ else:
+ query_relevance = round(rank_discounted_mean(relevance_scores), 5)
+ relevance_aggregation = "rank_discounted_mean"
+ query_effectiveness = round(rank_discounted_mean(effectiveness_scores), 5)
+ query_authority = round(rank_discounted_mean(authority_scores), 5)
+
+ labels = []
+ if query_relevance < args.threshold:
+ labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW")
+ relevance_errors = sum(1 for row in rows if row["relevance_error"])
+ if query_effectiveness < args.threshold:
+ labels.append("QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW")
+ if query_authority < args.threshold:
+ labels.append("QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW")
+
+ eval_status = bool(labels)
+ if not labels:
+ labels = ["QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS"]
+
+ query_row = {
+ "query": query,
+ "result_count": len(rows),
+ "valid_relevance_count": len(rows) - relevance_errors,
+ "relevance_error_count": relevance_errors,
+ "relevance": query_relevance,
+ "relevance_aggregation": relevance_aggregation,
+ "effectiveness_aggregation": "rank_discounted_mean",
+ "authority_aggregation": "rank_discounted_mean",
+ "effectiveness": query_effectiveness,
+ "authority": query_authority,
+ "eval_status": eval_status,
+ "label": "|".join(labels),
+ }
+ query_rows.append(query_row)
+ detail = {**query_row, "results": rows}
+ detailed.append(detail)
+ classified_records.append({
+ "query": query,
+ "metric": "search_result_quality",
+ "threshold": args.threshold,
+ "eval_status": eval_status,
+ "labels": labels,
+ "relevance": query_relevance,
+ "relevance_aggregation": relevance_aggregation,
+ "effectiveness_aggregation": "rank_discounted_mean",
+ "authority_aggregation": "rank_discounted_mean",
+ "effectiveness": query_effectiveness,
+ "authority": query_authority,
+ "relevance_error_count": relevance_errors,
+ "thresholds": {
+ "relevance": args.threshold,
+ "effectiveness": args.threshold,
+ "authority": args.threshold,
+ },
+ "results": full_results_by_query.get(query, []),
+ })
+
+ for query in empty_queries or []:
+ labels = [
+ "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW",
+ ]
+ query_row = {
+ "query": query,
+ "result_count": 0,
+ "valid_relevance_count": 0,
+ "relevance_error_count": 0,
+ "relevance": 0.0,
+ "relevance_aggregation": (
+ "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean"
+ ),
+ "effectiveness_aggregation": "rank_discounted_mean",
+ "authority_aggregation": "rank_discounted_mean",
+ "effectiveness": 0.0,
+ "authority": 0.0,
+ "eval_status": True,
+ "label": "|".join(labels),
+ }
+ query_rows.append(query_row)
+ detailed.append({**query_row, "results": []})
+ classified_records.append({
+ "query": query,
+ "metric": "search_result_quality",
+ "threshold": args.threshold,
+ "eval_status": True,
+ "labels": labels,
+ "relevance": 0.0,
+ "relevance_aggregation": (
+ "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean"
+ ),
+ "effectiveness_aggregation": "rank_discounted_mean",
+ "authority_aggregation": "rank_discounted_mean",
+ "effectiveness": 0.0,
+ "authority": 0.0,
+ "relevance_error_count": 0,
+ "thresholds": {
+ "relevance": args.threshold,
+ "effectiveness": args.threshold,
+ "authority": args.threshold,
+ },
+ "results": [],
+ })
+
+ summary = {
+ "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+ "metric": "search_result_quality",
+ "top_k": args.top_k,
+ "threshold": args.threshold,
+ "query_aggregation": "rank_discounted_mean",
+ "llm": {
+ "model": args.openai_model,
+ "prompt_mode": args.prompt_mode,
+ "max_tokens": args.llm_max_tokens,
+ "temperature": args.openai_temperature,
+ "timeout": args.llm_timeout,
+ "workers": args.llm_workers,
+ "effectiveness_llm_quality_enabled": not args.disable_effectiveness_llm_quality,
+ "effectiveness_llm_max_tokens": args.effectiveness_llm_max_tokens,
+ },
+ "run_output_path": str(executor_summary.output_path),
+ "metrics": {
+ "relevance": summarize([float(row["relevance"]) for row in query_rows]),
+ "effectiveness": summarize([float(row["effectiveness"]) for row in query_rows]),
+ "authority": summarize([float(row["authority"]) for row in query_rows]),
+ },
+ "query_count": len(query_rows),
+ "result_count": len(result_rows),
+ "rank_relevance_error_count": rank_relevance_error_count,
+ "rank_effectiveness_llm_quality_error_count": rank_effectiveness_llm_quality_error_count,
+ "num_bad": sum(1 for row in query_rows if row["eval_status"]),
+ "num_good": sum(1 for row in query_rows if not row["eval_status"]),
+ }
+ ratio_labels = (
+ "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW",
+ "QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS",
+ )
+ if query_rows:
+ label_counts = {label: 0 for label in ratio_labels}
+ for row in query_rows:
+ row_labels = {
+ label.strip()
+ for label in str(row.get("label") or "").split("|")
+ if label.strip()
+ }
+ for label in label_counts:
+ if label in row_labels:
+ label_counts[label] += 1
+ summary["type_ratio"] = {
+ "search_result_quality": {
+ label: count / len(query_rows)
+ for label, count in label_counts.items()
+ }
+ }
+ if retrieval_summary:
+ summary["retrieval"] = retrieval_summary
+ return summary, query_rows, result_rows, detailed, classified_records
+
+
+def clear_executor_classification_dirs(run_dir: Path) -> None:
+ """Remove result-level classifications before writing query-level output."""
+ for name in ("bad", "good"):
+ path = run_dir / name
+ if path.exists():
+ shutil.rmtree(path)
+
+
+def main() -> None:
+ load_env_file()
+ args = parse_args()
+ os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true")
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ temp_suffix = f"{int(time.time())}_{os.getpid()}"
+ flattened_path = args.output_dir / f".search_result_quality_input_{temp_suffix}.jsonl"
+ retrieval_results_path = args.output_dir / f".search_result_retrieval_{temp_suffix}.jsonl"
+ retrieval_log_path = args.output_dir / f".search_result_retrieval_log_{temp_suffix}.jsonl"
+ retrieval_summary: dict[str, Any] = {"backend": args.retrieval_backend}
+ try:
+ evaluation_input_path = args.input_jsonl
+ flatten_max_queries = args.max_queries
+ if args.retrieval_backend != "precomputed":
+ retrieval_summary = retrieve_queries(
+ args,
+ retrieval_results_path,
+ retrieval_log_path,
+ )
+ evaluation_input_path = retrieval_results_path
+ flatten_max_queries = None
+
+ total, empty_queries = flatten_query_results(
+ evaluation_input_path,
+ flattened_path,
+ top_k=args.top_k,
+ max_queries=flatten_max_queries,
+ )
+ if total:
+ input_data = build_executor_input(args, flattened_path)
+ executor_summary = Executor.exec_map["local"](InputArgs(**input_data)).execute()
+ run_dir = Path(executor_summary.output_path)
+ executor_records = load_executor_records(executor_summary.output_path)
+ clear_executor_classification_dirs(run_dir)
+ else:
+ run_dir = args.output_dir / datetime.now().strftime("%Y%m%d_%H%M%S_empty")
+ run_dir.mkdir(parents=True, exist_ok=True)
+ executor_summary = SimpleNamespace(output_path=str(run_dir))
+ executor_records = []
+
+ summary, query_rows, result_rows, detailed, classified_records = build_reports(
+ executor_records,
+ args,
+ executor_summary,
+ empty_queries,
+ retrieval_summary,
+ )
+
+ write_json(run_dir / "summary.json", summary)
+ if args.save_detailed:
+ write_json(run_dir / "detailed_results.json", {"summary": summary, "queries": detailed})
+ write_csv(run_dir / "query_scores.csv", query_rows)
+ write_csv(run_dir / "result_scores.csv", result_rows)
+ write_classified_jsonl(
+ run_dir,
+ classified_records,
+ save_good=args.save_good,
+ )
+ if args.retrieval_backend != "precomputed":
+ shutil.copyfile(retrieval_results_path, run_dir / "retrieval_results.jsonl")
+ shutil.copyfile(retrieval_log_path, run_dir / "retrieval_request_log.jsonl")
+ print(f"Saved to {run_dir.resolve()}")
+ finally:
+ for temp_path in (flattened_path, retrieval_results_path, retrieval_log_path):
+ if temp_path.exists():
+ temp_path.unlink()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/retrieval/search_result_eval_utils.py b/examples/retrieval/search_result_eval_utils.py
new file mode 100644
index 00000000..227c23c8
--- /dev/null
+++ b/examples/retrieval/search_result_eval_utils.py
@@ -0,0 +1,181 @@
+"""Utilities for search result JSONL evaluation examples."""
+
+from __future__ import annotations
+import argparse
+import csv
+import json
+import math
+import statistics
+from pathlib import Path
+from typing import Any
+
+
+def load_query_result_jsonl(path: Path, max_queries: int | None = None) -> list[dict[str, Any]]:
+ items: list[dict[str, Any]] = []
+ with path.open("r", encoding="utf-8-sig") as f:
+ for line_no, line in enumerate(f, start=1):
+ if not line.strip():
+ continue
+ item = json.loads(line)
+ query = item.get("query") or item.get("query_text") or item.get("q") or ""
+ results = (
+ item.get("results")
+ or item.get("top_results")
+ or item.get("top_api_results")
+ or item.get("search_results")
+ or []
+ )
+ if isinstance(results, dict):
+ results = results.get("results") or results.get("items") or []
+ if not isinstance(results, list):
+ raise ValueError(f"Line {line_no}: results must be a list.")
+ if not query:
+ raise ValueError(f"Line {line_no}: missing query/query_text.")
+ items.append({**item, "query": str(query), "results": results})
+ if max_queries and len(items) >= max_queries:
+ break
+ return items
+
+
+def load_queries(path: Path, max_queries: int | None = None) -> list[str]:
+ """Load unique queries from TXT, CSV, JSON, or JSONL input."""
+ suffix = path.suffix.lower()
+ queries: list[str] = []
+ seen: set[str] = set()
+
+ def add(value: Any) -> None:
+ query = str(value or "").strip()
+ if query and query not in seen:
+ seen.add(query)
+ queries.append(query)
+
+ if suffix == ".txt":
+ with path.open("r", encoding="utf-8-sig") as f:
+ for line in f:
+ add(line)
+ if max_queries and len(queries) >= max_queries:
+ break
+ return queries
+
+ if suffix == ".csv":
+ with path.open("r", encoding="utf-8-sig", newline="") as f:
+ for row in csv.DictReader(f):
+ add(row.get("query") or row.get("query_text") or row.get("q"))
+ if max_queries and len(queries) >= max_queries:
+ break
+ return queries
+
+ if suffix == ".json":
+ payload = json.loads(path.read_text(encoding="utf-8-sig"))
+ rows = payload if isinstance(payload, list) else payload.get("queries", [])
+ for row in rows:
+ if isinstance(row, str):
+ add(row)
+ elif isinstance(row, dict):
+ add(row.get("query") or row.get("query_text") or row.get("q"))
+ if max_queries and len(queries) >= max_queries:
+ break
+ return queries
+
+ with path.open("r", encoding="utf-8-sig") as f:
+ for line_no, line in enumerate(f, start=1):
+ if not line.strip():
+ continue
+ row = json.loads(line)
+ if isinstance(row, str):
+ add(row)
+ elif isinstance(row, dict):
+ add(row.get("query") or row.get("query_text") or row.get("q"))
+ else:
+ raise ValueError(f"Line {line_no}: query row must be a string or object.")
+ if max_queries and len(queries) >= max_queries:
+ break
+ return queries
+
+
+def get_title(result: dict[str, Any]) -> str:
+ return str(result.get("title") or result.get("display_name") or "")
+
+
+def rank_discounted_mean(values: list[float]) -> float:
+ """Return a query-level weighted mean that gives higher ranks more weight."""
+ if not values:
+ return 0.0
+ weights = [1.0 / math.log2(rank + 2) for rank in range(len(values))]
+ return sum(value * weight for value, weight in zip(values, weights)) / sum(weights)
+
+
+def summarize(values: list[float]) -> dict[str, float | int]:
+ if not values:
+ return {"count": 0, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0}
+ return {
+ "count": len(values),
+ "mean": round(statistics.mean(values), 5),
+ "median": round(statistics.median(values), 5),
+ "min": round(min(values), 5),
+ "max": round(max(values), 5),
+ }
+
+
+def add_common_args(parser: argparse.ArgumentParser) -> None:
+ parser.add_argument(
+ "--input-jsonl",
+ "--input-queries",
+ dest="input_jsonl",
+ type=Path,
+ required=True,
+ help="Precomputed query-result JSONL, or a TXT/CSV/JSON/JSONL query file in live retrieval mode.",
+ )
+ parser.add_argument("--output-dir", type=Path, default=Path("outputs/search_result_eval"))
+ parser.add_argument("--top-k", type=int, default=10)
+ parser.add_argument("--max-queries", type=int, default=None)
+ parser.add_argument("--save-good", action="store_true", help="Save good classified records under good/.")
+
+
+def write_json(path: Path, payload: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ if not rows:
+ path.write_text("", encoding="utf-8-sig")
+ return
+ fieldnames: list[str] = []
+ for row in rows:
+ for key in row:
+ if key not in fieldnames:
+ fieldnames.append(key)
+ with path.open("w", encoding="utf-8-sig", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=fieldnames)
+ writer.writeheader()
+ writer.writerows(rows)
+
+
+def write_classified_jsonl(
+ output_dir: Path,
+ records: list[dict[str, Any]],
+ *,
+ save_good: bool = False,
+ level: str | None = None,
+) -> None:
+ for record in records:
+ status = "bad" if record.get("eval_status") else "good"
+ if status == "good" and not save_good:
+ continue
+ labels = record.get("labels") or []
+ if not labels:
+ labels = ["QUALITY_GOOD.PASS"] if status == "good" else ["QUALITY_BAD.UNKNOWN"]
+ for label in labels:
+ parts = str(label).split(".")
+ status_dir = output_dir / status
+ if level:
+ status_dir = status_dir / level
+ if len(parts) > 1:
+ path = status_dir / Path(*parts[:-1]) / f"{parts[-1]}.jsonl"
+ else:
+ path = status_dir / f"{parts[0]}.jsonl"
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with path.open("a", encoding="utf-8") as f:
+ f.write(json.dumps(record, ensure_ascii=False) + "\n")
diff --git a/examples/security/sdk_perspective.py b/examples/security/sdk_perspective.py
new file mode 100644
index 00000000..70974ad7
--- /dev/null
+++ b/examples/security/sdk_perspective.py
@@ -0,0 +1,49 @@
+"""Run LLMPerspective against the Google Perspective API.
+
+Before running this example:
+
+ pip install google-api-python-client
+ export PERSPECTIVE_API_KEY="your-google-api-key"
+ python examples/security/sdk_perspective.py
+"""
+
+import os
+
+from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.io.input import Data
+from dingo.model.llm.llm_perspective import LLMPerspective
+
+
+def main() -> None:
+ api_key = os.getenv("PERSPECTIVE_API_KEY") or os.getenv("GOOGLE_API_KEY")
+ if not api_key:
+ raise SystemExit(
+ "Please set PERSPECTIVE_API_KEY (or GOOGLE_API_KEY) before running this example."
+ )
+
+ LLMPerspective.dynamic_config = EvaluatorLLMArgs(
+ key=api_key,
+ api_url=os.getenv(
+ "PERSPECTIVE_API_URL",
+ "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",
+ ),
+ )
+ LLMPerspective.client = None
+
+ samples = [
+ Data(data_id="perspective-good", content="Thank you for your thoughtful answer."),
+ Data(data_id="perspective-toxic", content="You are stupid and I hate you."),
+ ]
+
+ for sample in samples:
+ result = LLMPerspective.eval(sample)
+ print(f"data_id: {sample.data_id}")
+ print(f"content: {sample.content}")
+ print(f"status: {result.status} # True means a quality issue was detected")
+ print(f"label: {result.label}")
+ print(f"reason: {result.reason}")
+ print()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/requirements/hhem_integration.txt b/requirements/hhem_integration.txt
index 8efa88f2..2158f203 100644
--- a/requirements/hhem_integration.txt
+++ b/requirements/hhem_integration.txt
@@ -1,13 +1,19 @@
-# HHEM-2.1-Open Integration Dependencies
-# Required for Vectara HHEM-2.1-Open hallucination detection model
+# Hallucination Detection Dependencies (MiniCheck-Flan-T5-Large)
+# Required for the RuleHallucinationHHEM rule, which detects ungrounded claims
+# by checking whether a response is supported by its context.
+#
+# Model: lytang/MiniCheck-Flan-T5-Large (arXiv:2404.10774).
+# It is a standard T5ForConditionalGeneration checkpoint (no custom remote
+# code), so it loads on any modern transformers version — the previous
+# <4.49 pin (needed by Vectara HHEM's remote code) is no longer required.
-# Core transformers library for HHEM model
+# Core transformers library
transformers>=4.30.0
-# PyTorch (CPU version should be sufficient for HHEM)
+# PyTorch (CPU version is sufficient)
torch>=1.12.0
-# Additional dependencies for model operations
+# Tokenizer dependencies for flan-t5 (SentencePiece)
sentencepiece>=0.1.99
tokenizers>=0.13.0
diff --git a/requirements/optional.txt b/requirements/optional.txt
index 92aad3e6..d0198546 100644
--- a/requirements/optional.txt
+++ b/requirements/optional.txt
@@ -4,7 +4,6 @@
ftfy
imagededup
google-api-python-client
-lmdeploy
opencv-python-headless
pyiqa
pyspark==3.4.1
@@ -14,5 +13,3 @@ tiktoken
torch>=1.7.1
torchvision
tqdm
-
-git+https://github.com/openai/CLIP.git
diff --git a/setup.py b/setup.py
index fafbe638..b5e7b6ea 100644
--- a/setup.py
+++ b/setup.py
@@ -11,24 +11,32 @@ def _read_requirements(path):
requirements = _read_requirements("./requirements/runtime.txt")
+optional_requirements = _read_requirements("./requirements/optional.txt")
agent_requirements = _read_requirements("./requirements/agent.txt")
hhem_requirements = _read_requirements("./requirements/hhem_integration.txt")
litellm_requirements = ["litellm>=1.80.0,<1.87.0"]
retrieval_requirements = _read_requirements("./requirements/retrieval.txt")
+# lmdeploy 单独成组：它硬性要求 transformers>=4.56 并拉入大量重依赖，作为可选推理后端不并入
+# optional/all，保持默认环境轻量。需要时单独 pip install dingo-python[lmdeploy]。
+# （注：幻觉检测模型已从 Vectara HHEM 换为标准 T5 的 MiniCheck，不再有 transformers<4.49 上限，
+# 故 lmdeploy 与幻觉检测不再存在版本冲突；此处隔离仅出于依赖体量考虑。）
+lmdeploy_requirements = ["lmdeploy"]
extras_require = {
+ 'optional': optional_requirements,
'agent': agent_requirements,
'hhem': hhem_requirements,
'litellm': litellm_requirements,
'retrieval': retrieval_requirements,
- 'all': hhem_requirements + agent_requirements + litellm_requirements + retrieval_requirements,
+ 'lmdeploy': lmdeploy_requirements,
+ 'all': optional_requirements + hhem_requirements + agent_requirements + litellm_requirements + retrieval_requirements,
}
setup(
name="dingo-python",
- version="2.4.0",
+ version="2.5.0",
author="Dingo",
description="A Comprehensive AI Data Quality Evaluation Tool for Large Models",
long_description=long_description,
diff --git a/test/data/test_search_queries.jsonl b/test/data/test_search_queries.jsonl
new file mode 100644
index 00000000..9b00f397
--- /dev/null
+++ b/test/data/test_search_queries.jsonl
@@ -0,0 +1,3 @@
+{"query":"BiMLP"}
+{"query":"海带"}
+{"query":"pam"}
diff --git a/test/data/test_search_result.jsonl b/test/data/test_search_result.jsonl
new file mode 100644
index 00000000..c39ad0e3
--- /dev/null
+++ b/test/data/test_search_result.jsonl
@@ -0,0 +1,3 @@
+{"query":"BiMLP","results":[{"type":["preprint","article"],"publication_venue_type":"repository","access_license":"nonexclusive-distrib","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","cited_by_percentile_year":{},"classifications":{"arxiv_category":["cs.CV"]},"citation_count":9.0,"influential_citation_count":0.0,"abstract":"This paper studies the problem of designing compact binary architectures for vision multi-layer perceptrons (MLPs). We provide extensive analysis on the difficulty of binarizing vision MLPs and find that previous binarization methods perform poorly due to limited capacity of binary MLPs. In contrast with the traditional CNNs that utilizing convolutional operations with large kernel size, fully-connected (FC) layers in MLPs can be treated as convolutional layers with kernel size $1\\times1$. Thus, the representation ability of the FC layers will be limited when being binarized, and places restrictions on the capability of spatial mixing and channel mixing on the intermediate features. To this end, we propose to improve the performance of binary MLP (BiMLP) model by enriching the representation ability of binary FC layers. We design a novel binary block that contains multiple branches to merge a series of outputs from the same stage, and also a universal shortcut connection that encourages the information flow from the previous stage. The downsampling layers are also carefully designed to reduce the computational complexity while maintaining the classification performance. Experimental results on benchmark dataset ImageNet-1k demonstrate the effectiveness of the proposed BiMLP models, which achieve state-of-the-art accuracy compared to prior binary CNNs. The MindSpore code is available at \\url{https://gitee.com/mindspore/models/tree/master/research/cv/BiMLP}.","publication_publisher":[],"access_oa_status":"green","relevance_score":26.327469,"is_content_accessible":true,"publication_venue_name_unified":"arXiv (Cornell University)","locations":[{"type":"","is_oa":"true","url":"http://arxiv.org/abs/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://doi.org/10.48550/arxiv.2212.14158","license":""},{"type":"","is_oa":"unknown","url":"https://arxiv.org/abs/2212.14158","license":""},{"type":"download","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158v1","license":"nonexclusive-distrib"},{"type":"download","is_oa":"true","url":"https://arxiv.org/src/2212.14158v1","license":"nonexclusive-distrib"}],"publication_published_year":2022.0,"keywords":["Computer science","Perceptron","Binary number","Benchmark (surveying)","Binary tree","Layer (electronics)","Artificial intelligence","Convolutional neural network","Kernel (algebra)","Merge (version control)","Pattern recognition (psychology)","Algorithm","Parallel computing","Artificial neural network","Mathematics"],"unique_id":"paper:10.48550/arxiv.2212.14158","doi":"10.48550/arxiv.2212.14158","access_is_oa":"true","doc_id":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","language":"en","citation_normalized_percentile":{},"access_oa_url":["https://arxiv.org/pdf/2212.14158","https://arxiv.org/pdf/2212.14158v1"],"publication_venue_issn":[],"reference_count":55.0,"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","author":[{"orcid":"https://orcid.org/0000-0002-9704-4194","name":"Yixing Xu"},{"orcid":"https://orcid.org/0000-0002-2102-8235","name":"Xinghao Chen"},{"orcid":"https://orcid.org/0000-0002-0013-4530","name":"Yunhe Wang"}]},{"type":["article"],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":[],"access_oa_status":"closed","relevance_score":24.286182,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[{"type":"","is_oa":"false","url":"https://doi.org/10.52202/068431-0367","license":""}],"publication_published_year":2022.0,"keywords":["Perceptron","Binary number","Binary data","Pattern recognition (psychology)","Artificial neural network","Feature (linguistics)"],"unique_id":"paper:10.52202/068431-0367","doi":"10.52202/068431-0367","access_is_oa":"false","language":"","citation_normalized_percentile":{"value":0.33077007,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":[],"publication_venue_issn":[],"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Xinghao Chen"},{"orcid":"","name":"Yunhe Wang"},{"orcid":"","name":"Yixing Xu"}]},{"type":["article"],"publication_venue_type":"journal","access_license":"cc-by","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"Abstract. Satellite radar altimetry has significantly enhanced inland water monitoring by providing consistent, long-term lake-level observations. However, these datasets often contain substantial gaps caused by sensor malfunctions, orbital limitations, or retrieval errors, complicating hydrological analyses and downstream applications. This study introduces a robust benchmarking framework designed to systematically evaluate gap-filling techniques in satellite-derived lake water-level records through controlled pseudo-gap injection experiments. We investigated three large lakes with distinct hydrological behaviours, the Caspian Sea, Lake Superior, and Lake Tanganyika each representing unique temporal dynamics. Synthetic seven-year gaps (2002–2008) were artificially introduced into complete altimetry datasets, and three sophisticated gap-filling methods were compared: Singular Spectrum Analysis (SSA), Bidirectional Autoregressive (BIAR) models, and Bidirectional Multi-Layer Perceptrons (BiMLP). Method performance was assessed using standard metrics (RMSE, MAE, Bias, and R2), alongside statistical properties including variance, skewness, autocorrelation, and stationarity. BiMLP consistently delivered the highest accuracy across all study lakes, demonstrating exceptional adaptability to both smooth and highly variable signals. SSA performed effectively for lakes exhibiting quasi-periodic behavior, while BIAR showed sensitivity to lag selection and reduced performance under non-stationary conditions. These results emphasize the critical role of bidirectional modeling approaches and rigorous time-series diagnostics in selecting appropriate gap-filling methods for satellite altimetry-based hydrological studies. These findings provide practical guidance for selecting appropriate gap-filling strategies under long data outages in satellite altimetry workflows.","publication_publisher":["Copernicus Publications"],"access_oa_status":"diamond","relevance_score":8.976683,"is_content_accessible":false,"publication_venue_name_unified":"ISPRS annals of the photogrammetry, remote sensing and spatial information sciences","locations":[{"type":"","is_oa":"true","url":"https://doi.org/10.5194/isprs-annals-x-4-w8-2025-557-2026","license":"cc-by"},{"type":"","is_oa":"true","url":"https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf","license":"cc-by"},{"type":"","is_oa":"true","url":"https://doaj.org/article/72149efbb5e740b4a046e48e9a2acb4d","license":"cc-by-sa"}],"publication_published_year":2026.0,"keywords":["Benchmarking","Satellite","Sensitivity (control systems)","Adaptability","Water level","Altimeter","Autoregressive model","Satellite altimetry"],"unique_id":"paper:10.5194/isprs-annals-x-4-w8-2025-557-2026","doi":"10.5194/isprs-annals-x-4-w8-2025-557-2026","access_is_oa":"true","language":"en","citation_normalized_percentile":{"value":0.77162252,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":["https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf"],"publication_venue_issn":["2194-9042","2194-9050","2196-6346"],"title":"Benchmarking Gap-Filling Techniques in Satellite Altimetry-Based Lake Water-Level Time Series","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Ali MusaHoseini"},{"orcid":"https://orcid.org/0000-0002-0534-0632","name":"Saeed Farzaneh"},{"orcid":"https://orcid.org/0000-0003-3055-041X","name":"Ehsan Forootan"}]}]}
+{"query":"海带","results":[{"type":[],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"每次幼儿园吃海带,孩子们都不喜欢,端回的餐盘里都会剩下很多海带.鉴于老师说过“挑食不是好孩子”,所以他们不吃的借口也千奇百怪“老师,我已经吃饱了!”“老师我实在吃不下了!”“老师,我觉得这里面是青色的虫子,所以我……”哎!怎么做才能改善这种情况呢?","publication_publisher":[],"access_oa_status":"","relevance_score":32.074493,"is_content_accessible":true,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e教育\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e实践\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e与\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e研究\u003c/span\u003e | Educational Practice \u0026 Research","locations":[],"publication_published_year":2015.0,"keywords":["凉拌海带丝","《海带的传说》","海带","挑食"],"unique_id":"paper:10.3969/j.issn.1009-010x.2015.01.028","doi":"10.3969/j.issn.1009-010x.2015.01.028","access_is_oa":"unknown","doc_id":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1009-010X"],"reference_count":0.0,"title":"海带的传说","author":[{"orcid":"","name":"卢瑞云"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"宝宝,今天邀请爸爸妈妈玩\"海带拳\"游戏吧!宝宝当出拳人,妈妈当挑战者,两人一起喊:\"海带,海带!\"同时双手握拳,双臂上下左右挥舞.如果妈妈手臂方向和宝宝不一样,游戏继续;如果一样,挑战失败,换爸爸挑战宝宝.","publication_publisher":[],"access_oa_status":"","relevance_score":29.815878,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e动漫\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e界\u003c/span\u003e | Dong Man Jie","locations":[],"publication_published_year":2020.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1673-8438.2020.43.009","keywords":[],"unique_id":"paper:10.3969/j.issn.1673-8438.2020.43.009","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1673-8438"],"title":"海带拳","reference_count":0.0,"author":[{"orcid":"","name":"深红"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"育种单位:中国科学院海洋研究所、荣成市蜊江水产有限责任公司\r\n品种简介:该品种是以荣成海带栽培群体后代个体的雌配子体和韩国海带自然种群后代个体的雄配子体杂交产生的后代群体为亲本群体,以藻体深褐色、叶片宽大和孢子囊发育良好为选育指标,采用群体选育技术,经连续4代选育而成.\r\n在相同栽培条件下,与普通海带品种相比,在水温6℃左右(4月上旬)可开始收获,收获期可延续至水温达到19℃左右(7月中下旬),产量提高15.0%以上,抗高温、高光能力较强,淡干海带色泽墨绿.","publication_publisher":[],"access_oa_status":"","relevance_score":29.779959,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e中国\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e水产\u003c/span\u003e | China Fisheries","locations":[],"publication_published_year":2015.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1002-6681.2015.10.029","keywords":[],"unique_id":"paper:10.3969/j.issn.1002-6681.2015.10.029","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1002-6681"],"title":"海带\"205\"","reference_count":0.0,"author":[{"orcid":"","name":"刘峰"},{"orcid":"","name":"王嘉琪"},{"orcid":"","name":"逄少军"},{"orcid":"","name":"刘启顺"},{"orcid":"","name":"孙长彬"}]}]}
+{"query":"pam","results":[{"type":["printbook"],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863390","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.211927,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9788362863390","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396","9788362863390"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863396","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.09857,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"","doi":"","keywords":[],"unique_id":"ebook:9788362863396","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9780439397858","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Scholastic, Inc."],"access_oa_status":"","relevance_score":27.048462,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":1995.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9780439397858","language":"en","access_oa_url":[],"publication_venue_issn":[],"title":"Pam!, Pam!, Pam!","reference_count":0.0,"author":[{"orcid":"","name":"Eve Merriam"}],"isbns":["9780439397858"]}]}
diff --git a/test/scripts/dataset/test_markdown_dataset.py b/test/scripts/dataset/test_markdown_dataset.py
new file mode 100644
index 00000000..6552cfcb
--- /dev/null
+++ b/test/scripts/dataset/test_markdown_dataset.py
@@ -0,0 +1,65 @@
+import pytest
+
+from dingo.config import InputArgs
+from dingo.data.dataset.local import LocalDataset
+from dingo.data.datasource.local import LocalDataSource
+
+
+def test_markdown_single_file_to_data(tmp_path):
+ md_path = tmp_path / "article.md"
+ content = "# Title\n\nThis is markdown content.\n"
+ md_path.write_text(content, encoding="utf-8")
+
+ input_args = InputArgs(
+ input_path=str(md_path),
+ dataset={"source": "local", "format": "md"},
+ evaluator=[],
+ )
+
+ dataset = LocalDataset(source=LocalDataSource(input_args=input_args))
+ rows = list(dataset.get_data())
+
+ assert len(rows) == 1
+ assert rows[0].id == "article.md"
+ assert rows[0].content == content
+
+
+def test_markdown_directory_only_reads_md_files(tmp_path):
+ md1 = tmp_path / "a.md"
+ txt = tmp_path / "ignore.txt"
+ csv_file = tmp_path / "table.csv"
+ subdir = tmp_path / "nested"
+ subdir.mkdir()
+ md2 = subdir / "b.md"
+
+ md1.write_text("alpha", encoding="utf-8")
+ txt.write_text("should be ignored", encoding="utf-8")
+ csv_file.write_text("c1,c2\n1,2\n", encoding="utf-8")
+ md2.write_text("beta", encoding="utf-8")
+
+ input_args = InputArgs(
+ input_path=str(tmp_path),
+ dataset={"source": "local", "format": "md"},
+ evaluator=[],
+ )
+
+ dataset = LocalDataset(source=LocalDataSource(input_args=input_args))
+ rows = list(dataset.get_data())
+
+ assert len(rows) == 2
+ assert {row.id for row in rows} == {"a.md", "b.md"}
+ assert {row.content for row in rows} == {"alpha", "beta"}
+
+
+def test_markdown_directory_without_md_files_returns_empty(tmp_path):
+ (tmp_path / "readme.txt").write_text("plain text", encoding="utf-8")
+
+ input_args = InputArgs(
+ input_path=str(tmp_path),
+ dataset={"source": "local", "format": "md"},
+ evaluator=[],
+ )
+
+ dataset = LocalDataset(source=LocalDataSource(input_args=input_args))
+ rows = list(dataset.get_data())
+ assert rows == []
diff --git a/test/scripts/dataset/test_sql_dataset.py b/test/scripts/dataset/test_sql_dataset.py
index 8254ffb7..c44bfad2 100644
--- a/test/scripts/dataset/test_sql_dataset.py
+++ b/test/scripts/dataset/test_sql_dataset.py
@@ -7,6 +7,9 @@
import os
import sqlite3
import tempfile
+import uuid
+
+import pytest
from dingo.config import DatasetArgs, DatasetSqlArgs, InputArgs
from dingo.data.dataset.sql import SqlDataset
@@ -16,7 +19,7 @@
def create_test_database():
"""创建一个测试 SQLite 数据库"""
# 创建临时数据库文件
- db_path = os.path.join(tempfile.gettempdir(), "test_dingo_sql.db")
+ db_path = os.path.join(tempfile.gettempdir(), f"test_dingo_sql_{uuid.uuid4().hex}.db")
# 连接数据库并创建测试表
conn = sqlite3.connect(db_path)
@@ -63,6 +66,7 @@ def test_sql_dataset():
db_path = create_test_database()
print(f"✓ 创建测试数据库: {db_path}")
+ datasource = None
try:
# 配置 SQL 连接参数（SQLite）
sql_config = DatasetSqlArgs(
@@ -132,10 +136,15 @@ def test_sql_dataset():
print("=" * 60)
finally:
+ if datasource is not None:
+ datasource.engine.dispose()
# 清理测试数据库
if os.path.exists(db_path):
- os.remove(db_path)
- print(f"\n✓ 清理测试数据库: {db_path}")
+ try:
+ os.remove(db_path)
+ print(f"\n✓ 清理测试数据库: {db_path}")
+ except PermissionError:
+ print(f"\n! 跳过清理（文件占用）: {db_path}")
def test_stream_results():
@@ -145,7 +154,10 @@ def test_stream_results():
print("=" * 60)
# 创建一个包含更多数据的测试数据库
- db_path = os.path.join(tempfile.gettempdir(), "test_dingo_sql_stream.db")
+ db_path = os.path.join(
+ tempfile.gettempdir(),
+ f"test_dingo_sql_stream_{uuid.uuid4().hex}.db"
+ )
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
@@ -164,6 +176,7 @@ def test_stream_results():
print(f"✓ 创建包含 1000 条数据的测试数据库")
+ datasource = None
try:
sql_config = DatasetSqlArgs(
dialect="sqlite",
@@ -195,19 +208,135 @@ def test_stream_results():
# 只读取前 10 条，验证流式读取（不会加载全部 1000 条到内存）
print("开始流式读取（只读取前 10 条）:")
count = 0
- for idx, data in enumerate(dataset.get_data()):
- if idx < 10:
- print(f" [{idx + 1}] {data}")
- count += 1
- if idx >= 9: # 只读取前 10 条就停止
- break
+ data_iterator = iter(dataset.get_data())
+ try:
+ for idx, data in enumerate(data_iterator):
+ if idx < 10:
+ print(f" [{idx + 1}] {data}")
+ count += 1
+ if idx >= 9: # 只读取前 10 条就停止
+ break
+ finally:
+ close_method = getattr(data_iterator, "close", None)
+ if callable(close_method):
+ close_method()
print(f"\n✓ 流式读取验证通过（处理了 {count} 条数据后停止）")
finally:
+ if datasource is not None:
+ datasource.engine.dispose()
if os.path.exists(db_path):
- os.remove(db_path)
- print(f"✓ 清理测试数据库: {db_path}")
+ try:
+ os.remove(db_path)
+ print(f"✓ 清理测试数据库: {db_path}")
+ except PermissionError:
+ print(f"! 跳过清理（文件占用）: {db_path}")
+
+
+def test_parse_connect_args_supports_prefix_and_multiple_pairs():
+ query_args = SqlDataSource._parse_connect_args(
+ "?charset=utf8mb4&read_timeout=120&write_timeout=120"
+ )
+ assert query_args["charset"] == "utf8mb4"
+ assert query_args["read_timeout"] == "120"
+ assert query_args["write_timeout"] == "120"
+
+
+def test_mysql_engine_has_stability_pool_settings():
+ sql_config = DatasetSqlArgs(
+ dialect="mysql",
+ driver="pymysql",
+ username="user",
+ password="pass",
+ host="localhost",
+ port="3306",
+ database="db",
+ connect_args="charset=utf8mb4"
+ )
+ engine = SqlDataSource._get_engine(sql_config)
+ try:
+ assert engine.pool._pre_ping is True
+ assert engine.pool._recycle == 1800
+ finally:
+ engine.dispose()
+
+
+def test_mysql_does_not_inject_default_timeout_query_args():
+ sql_config = DatasetSqlArgs(
+ dialect="mysql",
+ driver="pymysql",
+ username="user",
+ password="pass",
+ host="localhost",
+ port="3306",
+ database="db",
+ connect_args="charset=utf8mb4"
+ )
+ query_args = SqlDataSource._parse_connect_args(sql_config.connect_args)
+ url = SqlDataSource._build_connection_url(sql_config, query_args)
+ assert url.query == {"charset": "utf8mb4"}
+
+
+def test_parse_engine_args_with_supported_types():
+ engine_args = SqlDataSource._parse_engine_args(
+ "pool_recycle=3600&pool_pre_ping=true&pool_size=8&max_overflow=16&pool_timeout=30"
+ )
+ assert engine_args == {
+ "pool_recycle": 3600,
+ "pool_pre_ping": True,
+ "pool_size": 8,
+ "max_overflow": 16,
+ "pool_timeout": 30,
+ }
+
+
+def test_engine_args_override_default_pool_recycle():
+ sql_config = DatasetSqlArgs(
+ dialect="mysql",
+ driver="pymysql",
+ username="user",
+ password="pass",
+ host="localhost",
+ port="3306",
+ database="db",
+ engine_args="pool_recycle=7200&pool_pre_ping=false"
+ )
+ engine = SqlDataSource._get_engine(sql_config)
+ try:
+ assert engine.pool._recycle == 7200
+ assert engine.pool._pre_ping is False
+ finally:
+ engine.dispose()
+
+
+def test_engine_args_rejects_unsupported_key():
+ with pytest.raises(RuntimeError, match="Unsupported SQL engine arg"):
+ SqlDataSource._parse_engine_args("unsupported_key=1")
+
+
+def test_engine_args_rejects_invalid_bool_value():
+ with pytest.raises(RuntimeError, match="true' or 'false"):
+ SqlDataSource._parse_engine_args("pool_pre_ping=not_bool")
+
+
+def test_engine_args_rejects_numeric_bool_value():
+ with pytest.raises(RuntimeError, match="true' or 'false"):
+ SqlDataSource._parse_engine_args("pool_pre_ping=1")
+
+
+def test_invalid_port_raises_runtime_error():
+ sql_config = DatasetSqlArgs(
+ dialect="mysql",
+ driver="pymysql",
+ username="user",
+ password="pass",
+ host="localhost",
+ port="not_a_number",
+ database="db"
+ )
+ with pytest.raises(RuntimeError, match="port"):
+ SqlDataSource._build_connection_url(sql_config, {})
if __name__ == "__main__":
diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py
index 44ca5014..f96f3525 100644
--- a/test/scripts/exec/test_local.py
+++ b/test/scripts/exec/test_local.py
@@ -3,7 +3,8 @@
from dingo.config import InputArgs
from dingo.exec import Executor, LocalExecutor
from dingo.io import ResultInfo
-from dingo.io.output.eval_detail import EvalDetail
+from dingo.io.output.eval_detail import EvalDetail, TokenUsage
+from dingo.model import Model
class TestLocal:
@@ -192,6 +193,86 @@ def test_merge_result_info(self):
assert "�I am 8 years old. ^I love apple because:" in all_reasons
assert "文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。" in all_reasons
+ def test_merge_result_info_preserves_token_usage_details(self):
+ localexecutor = LocalExecutor({})
+ item1 = ResultInfo(
+ dingo_id="1",
+ token_usage_details={
+ "content": [
+ EvalDetail(
+ metric="LLMMetricA",
+ usage=TokenUsage(total_tokens=3),
+ )
+ ]
+ },
+ )
+ item2 = ResultInfo(
+ dingo_id="1",
+ token_usage_details={
+ "content": [
+ EvalDetail(
+ metric="LLMMetricB",
+ usage=TokenUsage(total_tokens=5),
+ )
+ ]
+ },
+ )
+
+ merged = localexecutor.merge_result_info([], item1)
+ merged = localexecutor.merge_result_info(merged, item2)
+
+ assert len(merged[0].token_usage_details["content"]) == 2
+
+ def test_token_usage_summary_can_include_unsaved_good_eval_details(self):
+ class TokenUsageGoodLLM:
+ def eval(self, input_data):
+ return EvalDetail(
+ metric="TokenUsageGoodLLM",
+ status=False,
+ label=["QUALITY_GOOD"],
+ usage=TokenUsage(
+ prompt_tokens=10,
+ completion_tokens=5,
+ total_tokens=15,
+ ),
+ )
+
+ old_model = Model.llm_name_map.get("TokenUsageGoodLLM")
+ Model.llm_name_map["TokenUsageGoodLLM"] = TokenUsageGoodLLM
+ try:
+ input_args = InputArgs(
+ executor={
+ "result_save": {
+ "bad": True,
+ "good": False,
+ "all_labels": False,
+ }
+ },
+ evaluator=[
+ {
+ "fields": {"content": "content"},
+ "evals": [{"name": "TokenUsageGoodLLM"}],
+ }
+ ],
+ )
+ executor = LocalExecutor(input_args)
+
+ result = executor.evaluate_single_data(
+ dingo_id="1",
+ eval_fields={"content": "content"},
+ eval_type="llm",
+ map_data={"content": "ok"},
+ eval_list=input_args.evaluator[0].evals,
+ )
+
+ assert result.eval_details == {}
+ assert result.token_usage_details["content"][0].usage.total_tokens == 15
+ finally:
+ if old_model is None:
+ Model.llm_name_map.pop("TokenUsageGoodLLM", None)
+ else:
+ Model.llm_name_map["TokenUsageGoodLLM"] = old_model
+
def test_all_labels_config(self):
input_data = {
"input_path": "test/data/test_local_jsonl.jsonl",
diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py
index c9420ffc..9e7fac5f 100644
--- a/test/scripts/io/test_summary_model.py
+++ b/test/scripts/io/test_summary_model.py
@@ -11,6 +11,7 @@
import pytest
+from dingo.io.output.eval_detail import TokenUsage
from dingo.io.output.summary_model import SummaryModel
@@ -221,6 +222,49 @@ def test_to_dict_without_scores(self):
# 验证没有分数统计字段
assert "metrics_score" not in result
+ def test_add_token_usage_and_to_dict(self):
+ """测试 LLM token 使用量统计输出"""
+ summary = SummaryModel(task_name="test_task", task_id="test_token_001")
+
+ summary.add_token_usage(
+ "content",
+ "LLMTextQualityV5",
+ TokenUsage(
+ prompt_tokens=10,
+ completion_tokens=4,
+ total_tokens=14,
+ reasoning_tokens=1,
+ cached_tokens=3,
+ model="gpt-test",
+ provider="openai",
+ ),
+ )
+ summary.add_token_usage(
+ "content",
+ "LLMTextQualityV5",
+ TokenUsage(
+ prompt_tokens=8,
+ completion_tokens=5,
+ total_tokens=13,
+ model="gpt-test",
+ provider="openai",
+ ),
+ )
+
+ result = summary.to_dict()
+
+ stats = result["token_usage"]["content"]["LLMTextQualityV5"]
+ assert stats["prompt_tokens"] == 18
+ assert stats["completion_tokens"] == 9
+ assert stats["total_tokens"] == 27
+ assert stats["reasoning_tokens"] == 1
+ assert stats["cached_tokens"] == 3
+ assert stats["calls"] == 2
+ assert stats["records"] == 2
+ assert stats["models"] == {"gpt-test": 2}
+ assert stats["providers"] == {"openai": 2}
+ assert stats["sources"] == {"provider": 2}
+
def test_multiple_metrics_different_score_counts(self):
"""测试不同指标有不同数量的分数"""
summary = SummaryModel(
diff --git a/test/scripts/model/llm/test_litellm.py b/test/scripts/model/llm/test_litellm.py
index 43d1ed90..124d5ee4 100644
--- a/test/scripts/model/llm/test_litellm.py
+++ b/test/scripts/model/llm/test_litellm.py
@@ -26,12 +26,12 @@ class _Provider(BaseLiteLLM):
return _Provider
-def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop"):
+def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop", usage=None):
choice = SimpleNamespace(
finish_reason=finish_reason,
message=SimpleNamespace(content=content),
)
- return SimpleNamespace(choices=[choice])
+ return SimpleNamespace(choices=[choice], usage=usage)
# ---------------------------------------------------------------------------
@@ -127,7 +127,25 @@ def test_none_content_returns_empty_string(self):
none_resp = _stub_response(content=None)
with mock.patch("litellm.completion", return_value=none_resp):
result = P.send_messages([{"role": "user", "content": "hi"}])
- assert result == ""
+ assert result.content == ""
+
+ def test_returns_token_usage_when_provider_supplies_usage(self):
+ P = _make_provider(model="gpt-4o")
+ provider_resp = _stub_response(
+ usage={
+ "prompt_tokens": 6,
+ "completion_tokens": 3,
+ "total_tokens": 9,
+ }
+ )
+ with mock.patch("litellm.completion", return_value=provider_resp):
+ result = P.send_messages([{"role": "user", "content": "hi"}])
+
+ assert result.content == '{"score": 1, "reason": "ok"}'
+ assert result.usage.prompt_tokens == 6
+ assert result.usage.completion_tokens == 3
+ assert result.usage.total_tokens == 9
+ assert result.usage.provider == "litellm"
# ---------------------------------------------------------------------------
diff --git a/test/scripts/model/llm/test_llm_custom_metric.py b/test/scripts/model/llm/test_llm_custom_metric.py
index 5eeefdf7..5665bf3d 100644
--- a/test/scripts/model/llm/test_llm_custom_metric.py
+++ b/test/scripts/model/llm/test_llm_custom_metric.py
@@ -3,6 +3,8 @@
from dingo.config.input_args import EvaluatorLLMArgs, InputArgs
from dingo.io.input import Data
+from dingo.io.output.eval_detail import TokenUsage
+from dingo.model.llm.base import LLMCallResult
from dingo.model.llm.llm_custom_metric import LLMCustomMetric
from dingo.model.model import Model
@@ -197,6 +199,42 @@ def test_eval_detail_response_uses_llm_returned_fields():
assert result.reason == ["The content contains AI-style phrasing."]
+def test_eval_detail_response_attaches_token_usage():
+ llm = LLMCustomMetric()
+ Model.set_config_llm(
+ llm, EvaluatorLLMArgs(custom_metric=_custom_metric(metric="SourceLabel"))
+ )
+ llm.create_client = Mock()
+ llm.send_messages = Mock(
+ return_value=LLMCallResult(
+ content=json.dumps(
+ {
+ "status": False,
+ "label": ["SOURCE.AI_GENERATED"],
+ "score": 0.82,
+ "reason": ["The content contains AI-style phrasing."],
+ }
+ ),
+ usage=TokenUsage(
+ prompt_tokens=12,
+ completion_tokens=5,
+ total_tokens=17,
+ model="gpt-test",
+ provider="openai",
+ ),
+ )
+ )
+
+ result = llm.eval(
+ Data(prompt="Classify source", content="As an AI language model...")
+ )
+
+ assert result.usage is not None
+ assert result.usage.prompt_tokens == 12
+ assert result.usage.completion_tokens == 5
+ assert result.usage.total_tokens == 17
+
+
def test_eval_detail_response_rejects_missing_fields():
llm = LLMCustomMetric()
Model.set_config_llm(
diff --git a/test/scripts/model/llm/test_llm_search_result_authority.py b/test/scripts/model/llm/test_llm_search_result_authority.py
new file mode 100644
index 00000000..88d5633b
--- /dev/null
+++ b/test/scripts/model/llm/test_llm_search_result_authority.py
@@ -0,0 +1,126 @@
+from dingo.model.llm.llm_search_result_authority import LLMSearchResultAuthority, _has_doi_in_locations
+
+
+class NonSerializableLocation:
+ pass
+
+
+def test_has_doi_in_location_dict_or_string():
+ assert _has_doi_in_locations([{"url": "https://doi.org/10.1000/test"}])
+ assert _has_doi_in_locations(["https://DOI.ORG/10.1000/test"])
+
+
+def test_has_doi_in_locations_ignores_invalid_shapes():
+ assert not _has_doi_in_locations(True)
+ assert not _has_doi_in_locations({"url": "https://doi.org/10.1000/test"})
+ assert not _has_doi_in_locations([NonSerializableLocation()])
+
+
+def test_authority_grade_handles_non_serializable_location():
+ grade = LLMSearchResultAuthority().grade(
+ result={"doi": "", "locations": [NonSerializableLocation()]},
+ )
+
+ assert grade.doi_score == 0.0
+
+
+def test_nature_portfolio_families_are_recognized():
+ grader = LLMSearchResultAuthority()
+
+ for venue in (
+ "Nature Physics",
+ "Nature Communications",
+ "npj Digital Medicine",
+ "Communications Biology",
+ "Scientific Reports",
+ "Scientific Data",
+ ):
+ grade = grader.grade(result={"publication_venue_name_unified": venue})
+ assert grade.venue_score == 0.85
+ assert grade.reason == "prestigious_venue_family"
+
+
+def test_generic_science_names_do_not_match_science_family():
+ grader = LLMSearchResultAuthority()
+
+ named_only = grader.grade(result={"publication_venue_name_unified": "Grand Garden of Science"})
+ structured = grader.grade(
+ result={
+ "publication_venue_name_unified": "Chemical Engineering Science",
+ "publication_venue_type": "journal",
+ }
+ )
+
+ assert named_only.venue_score == 0.4
+ assert named_only.reason == "named_venue"
+ assert structured.venue_score == 0.65
+ assert structured.reason == "structured_journal_or_conference"
+
+
+def test_repository_takes_priority_over_name_patterns():
+ grade = LLMSearchResultAuthority().grade(
+ result={"publication_venue_name_unified": "Open Science Framework"},
+ )
+
+ assert grade.venue_score == 0.45
+ assert grade.reason == "repository_or_preprint"
+
+
+def test_issn_and_recognized_publisher_provide_venue_fallbacks():
+ grader = LLMSearchResultAuthority()
+ issn_grade = grader.grade(
+ result={
+ "publication_venue_name_unified": "Specialist Research Journal",
+ "publication_venue_issn": ["1234-567X"],
+ }
+ )
+ publisher_grade = grader.grade(
+ result={
+ "publication_venue_name_unified": "Specialist Research Journal",
+ "publication_publisher": ["Oxford University Press"],
+ }
+ )
+
+ assert issn_grade.venue_score == 0.65
+ assert issn_grade.reason == "structured_journal_or_conference"
+ assert publisher_grade.venue_score == 0.75
+ assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue"
+
+
+def test_html_highlight_is_removed_before_venue_matching():
+ grade = LLMSearchResultAuthority().grade(
+ result={"publication_venue_name_unified": "Nature Medicine"},
+ )
+
+ assert grade.venue_score == 0.85
+
+
+def test_ieee_is_recognized_inside_full_conference_name():
+ grade = LLMSearchResultAuthority().grade(
+ result={
+ "publication_venue_name_unified": (
+ "13th IEEE International Workshops on Enabling Technologies"
+ )
+ },
+ )
+
+ assert grade.venue_score == 0.75
+ assert grade.reason == "recognized_scholarly_publisher_or_venue"
+
+
+def test_ebook_platform_and_academic_publisher_are_recognized():
+ grader = LLMSearchResultAuthority()
+ ebook_grade = grader.grade(
+ result={
+ "publication_venue_name_unified": "Springer eBooks",
+ "publication_venue_type": "ebook platform",
+ }
+ )
+ publisher_grade = grader.grade(
+ result={"publication_publisher": ["Walter De Gruyter & Co"]},
+ )
+
+ assert ebook_grade.venue_score == 0.55
+ assert ebook_grade.reason == "academic_book_series"
+ assert publisher_grade.venue_score == 0.75
+ assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue"
diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py
new file mode 100644
index 00000000..9ac95e8c
--- /dev/null
+++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py
@@ -0,0 +1,222 @@
+import pytest
+
+from dingo.model.llm.llm_search_result_effectiveness import ( # isort: skip
+ LLMSearchResultEffectiveness,
+ _filter_llm_field_issues,
+ _issues_to_labels,
+ _looks_like_utf8_latin1_mojibake,
+ _rule_abnormal_char_issues,
+ extract_authors,
+)
+
+
+def _mojibake(value: str) -> str:
+ return value.encode("utf-8").decode("latin-1")
+
+
+def test_detects_utf8_cyrillic_decoded_as_latin1():
+ broken = _mojibake("Развитие научных исследований")
+
+ assert _looks_like_utf8_latin1_mojibake(broken)
+ assert "RuleMojibake" in _rule_abnormal_char_issues(broken)
+ assert _filter_llm_field_issues("title", broken, ["title:mojibake"]) == ["title:mojibake"]
+ assert _issues_to_labels(["RuleMojibake", "title:mojibake"]) == ["Effectiveness.Error_Mojibake"]
+
+
+def test_does_not_flag_valid_latin_or_cyrillic_text():
+ assert not _looks_like_utf8_latin1_mojibake("Ð is a valid Icelandic letter")
+ assert not _looks_like_utf8_latin1_mojibake("Развитие научных исследований")
+
+
+def test_detects_mojibake_fragment_in_mixed_language_text():
+ broken = "中文标题 | " + _mojibake("Научные исследования")
+
+ assert _looks_like_utf8_latin1_mojibake(broken)
+
+
+def test_rule_only_grade_penalizes_mojibake_fields():
+ broken_title = _mojibake("Развитие научных исследований")
+ broken_abstract = _mojibake(
+ "В этой статье рассматриваются современные научные исследования и методы анализа данных. " * 4
+ )
+ grader = LLMSearchResultEffectiveness(enable_llm_quality=False)
+
+ grade = grader.grade(
+ title=broken_title,
+ abstract=broken_abstract,
+ keywords=["research", "analysis", "data"],
+ venue="Science Journal",
+ )
+
+ assert "RuleMojibake" in grade.issues
+ assert grade.title_score == 0.1
+ assert grade.abstract_score == 0.1
+ assert grade.score < 0.5
+
+
+def test_extract_authors_supports_common_response_shapes():
+ assert extract_authors({"author": [{"name": "Alice"}, {"display_name": "张三"}]}) == ["Alice", "张三"]
+ assert extract_authors({"authors": "Alice | Bob"}) == ["Alice", "Bob"]
+ assert extract_authors({"author": {"author_name": "Carol"}}) == ["Carol"]
+
+
+def test_author_is_scored_without_rewarding_author_count():
+ grader = LLMSearchResultEffectiveness(enable_llm_quality=False)
+ common = {
+ "title": "A comprehensive evaluation of academic search result metadata",
+ "abstract": "academic search metadata provides useful information for readers " * 15,
+ "keywords": ["search", "metadata", "quality", "evaluation", "academic"],
+ "venue": "International Journal of Search Quality Research",
+ }
+
+ single_author = grader.grade(**common, authors=["Alice"])
+ multiple_authors = grader.grade(**common, authors=["Alice", "Bob", "Carol"])
+ missing_author = grader.grade(**common)
+
+ assert single_author.author_score == 1.0
+ assert multiple_authors.author_score == 1.0
+ assert single_author.score == multiple_authors.score
+ assert missing_author.author_score == 0.0
+ assert missing_author.score == pytest.approx(single_author.score - 0.1)
+ assert "missing_author" in missing_author.issues
+ assert _issues_to_labels(missing_author.issues) == ["Effectiveness.Error_Author_Miss"]
+
+
+def test_nonempty_fields_are_not_penalized_for_length_or_item_count():
+ grader = LLMSearchResultEffectiveness(enable_llm_quality=False)
+
+ grade = grader.grade(
+ title="D",
+ abstract="短",
+ keywords=["AI"],
+ venue="J",
+ authors=["Q"],
+ )
+
+ assert grade.title_score == 1.0
+ assert grade.abstract_score == 1.0
+ assert grade.keywords_score == 1.0
+ assert grade.venue_score == 1.0
+ assert grade.author_score == 1.0
+ assert grade.score == 1.0
+ assert grade.issues == []
+
+
+def test_longer_content_does_not_receive_more_effectiveness_credit():
+ grader = LLMSearchResultEffectiveness(enable_llm_quality=False)
+ short = grader.grade(title="D", abstract="A", keywords=["K"], authors=["Q"])
+ long = grader.grade(
+ title="A comprehensive academic title",
+ abstract="A complete and readable abstract. " * 100,
+ keywords=["one", "two", "three", "four", "five"],
+ authors=["Alice", "Bob"],
+ )
+
+ assert short.score == long.score == 1.0
+
+
+def test_preview_navigation_text_is_not_treated_as_html():
+ abstract = (
+ "Preview this article: Meaning and the Structure of Language, by Wallace Chafe, "
+ "Page 1 of 1 < Previous page | Next page > "
+ "/docserver/preview/fulltext/ce/33/8/collegeenglish18315-1.gif"
+ )
+
+ assert "RuleSpecialCharacter" not in _rule_abnormal_char_issues(abstract)
+
+ # LLM quality is enabled deliberately: this text should bypass the LLM
+ # because it is not an abnormal-character candidate.
+ grade = LLMSearchResultEffectiveness(enable_llm_quality=True).grade(
+ title="Meaning and the Structure of Language, by Wallace Chafe",
+ abstract=abstract,
+ keywords=["Linguistics"],
+ venue="College English",
+ authors=["Frank Heny"],
+ )
+
+ assert grade.score == 1.0
+ assert grade.issues == []
+
+
+@pytest.mark.parametrize(
+ "markup",
+ [
+ "language",
+ "language",
+ "H2O",
+ "AM",
+ ],
+)
+def test_real_academic_html_tags_remain_detectable(markup: str):
+ assert "RuleSpecialCharacter" in _rule_abnormal_char_issues(markup)
+ assert _filter_llm_field_issues("title", markup, ["title:html_tag"]) == [
+ "title:html_tag"
+ ]
+
+
+def test_missing_venue_is_diagnostic_only_and_does_not_reduce_score():
+ grader = LLMSearchResultEffectiveness(enable_llm_quality=False)
+ common = {
+ "title": "A comprehensive evaluation of academic search result metadata",
+ "abstract": "academic search metadata provides useful information for readers " * 15,
+ "keywords": ["search", "metadata", "quality", "evaluation", "academic"],
+ "authors": ["Alice"],
+ }
+
+ with_venue = grader.grade(**common, venue="International Journal of Search Quality Research")
+ without_venue = grader.grade(**common, venue="")
+
+ assert with_venue.score == without_venue.score
+ assert without_venue.venue_score == 0.0
+ assert "missing_venue" not in without_venue.issues
+ assert "Effectiveness.Error_Venue_Miss" not in _issues_to_labels(without_venue.issues)
+
+
+def _complete_result() -> dict:
+ return {
+ "title": "A comprehensive evaluation of academic search result metadata",
+ "abstract": "academic search metadata provides useful information for readers " * 15,
+ "keywords": ["search", "metadata", "quality", "evaluation", "academic"],
+ "publication_venue_name_unified": "International Journal of Search Quality Research",
+ "author": [{"name": "Alice"}],
+ }
+
+
+@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"])
+def test_all_effectiveness_fields_scan_html_residue(field: str):
+ result = _complete_result()
+ contaminated = "clean text leaked markup"
+ if field == "keywords":
+ result["keywords"] = [contaminated]
+ elif field == "venue":
+ result["publication_venue_name_unified"] = contaminated
+ elif field == "author":
+ result["author"] = [{"name": contaminated}]
+ elif field == "abstract":
+ result["abstract"] = "readable abstract text " * 100 + contaminated
+ else:
+ result[field] = contaminated
+
+ grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result)
+
+ assert "RuleSpecialCharacter" in grade.issues
+ assert getattr(grade, f"{field}_score") <= 0.1
+
+
+@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"])
+def test_all_effectiveness_fields_scan_replacement_character(field: str):
+ result = _complete_result()
+ contaminated = "metadata contains \ufffd broken text"
+ if field == "keywords":
+ result["keywords"] = [contaminated]
+ elif field == "venue":
+ result["publication_venue_name_unified"] = contaminated
+ elif field == "author":
+ result["author"] = [{"name": contaminated}]
+ else:
+ result[field] = contaminated
+
+ grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result)
+
+ assert "RuleMojibake" in grade.issues
+ assert getattr(grade, f"{field}_score") <= 0.1
diff --git a/test/scripts/model/llm/test_llm_search_result_relevance.py b/test/scripts/model/llm/test_llm_search_result_relevance.py
new file mode 100644
index 00000000..210d6d67
--- /dev/null
+++ b/test/scripts/model/llm/test_llm_search_result_relevance.py
@@ -0,0 +1,48 @@
+from dingo.model.llm.llm_search_result_relevance import _extract_result_dois, _grade_doi_result, _normalize_doi, is_doi_query
+
+
+def test_normalize_doi_variants():
+ assert _normalize_doi("10.1016/j.ijbiomac.2025.143529") == "10.1016/j.ijbiomac.2025.143529"
+ assert _normalize_doi("https://doi.org/10.1038/NCOMMS7112") == "10.1038/ncomms7112"
+ assert _normalize_doi(":10.1111/jipb.70096") == "10.1111/jipb.70096"
+ assert _normalize_doi("PBPK review") == ""
+ assert is_doi_query("10.1016/j.ijbiomac.2025.143529")
+ assert not is_doi_query("PBPK review")
+
+
+def test_extract_result_dois_from_supported_fields():
+ result = {
+ "doi": "https://doi.org/10.1000/ABC",
+ "unique_id": "paper:10.2000/xyz",
+ "locations": [{"url": "https://doi.org/10.3000/location"}],
+ }
+ assert _extract_result_dois(result) == [
+ "10.1000/abc",
+ "10.2000/xyz",
+ "10.3000/location",
+ ]
+
+
+def test_extract_result_dois_ignores_non_list_locations():
+ assert _extract_result_dois({"locations": True}) == []
+ assert _extract_result_dois({"locations": 1}) == []
+ assert _extract_result_dois({"locations": {"url": "https://doi.org/10.1000/test"}}) == []
+
+
+def test_doi_query_uses_exact_match():
+ matched = _grade_doi_result(
+ "10.1016/j.ijbiomac.2025.143529",
+ {"doi": "https://doi.org/10.1016/j.ijbiomac.2025.143529"},
+ )
+ mismatched = _grade_doi_result(
+ "10.1016/j.ijbiomac.2025.143529",
+ {"doi": "https://doi.org/10.3390/plants14152362"},
+ )
+
+ assert matched is not None and matched.score == 1.0
+ assert mismatched is not None and mismatched.score == 0.0
+ assert "DOI mismatch" in mismatched.reasoning
+
+
+def test_non_doi_query_falls_back_to_llm():
+ assert _grade_doi_result("PBPK相关综述", {"doi": "10.1000/test"}) is None
diff --git a/test/scripts/model/llm/test_tc609_doc_completeness.py b/test/scripts/model/llm/test_tc609_doc_completeness.py
new file mode 100644
index 00000000..2e155d21
--- /dev/null
+++ b/test/scripts/model/llm/test_tc609_doc_completeness.py
@@ -0,0 +1,47 @@
+from dingo.io.input import Data
+from dingo.model.llm.guobiao.llm_tc609_0101_doc_basic_info_completeness import LLM_TC609_0101_DocBasicInfoCompleteness
+from dingo.model.llm.guobiao.llm_tc609_0102_doc_content_feature_completeness import LLM_TC609_0102_DocContentFeatureCompleteness
+from dingo.model.llm.guobiao.llm_tc609_0103_doc_construction_process_completeness import LLM_TC609_0103_DocConstructionProcessCompleteness
+from dingo.model.llm.guobiao.llm_tc609_0104_doc_application_completeness import LLM_TC609_0104_DocApplicationCompleteness
+
+TC609_LLM_CLASSES = [
+ LLM_TC609_0101_DocBasicInfoCompleteness,
+ LLM_TC609_0102_DocContentFeatureCompleteness,
+ LLM_TC609_0103_DocConstructionProcessCompleteness,
+ LLM_TC609_0104_DocApplicationCompleteness,
+]
+
+
+def test_tc609_doc_llm_prompts_define_binary_json_output():
+ for evaluator in TC609_LLM_CLASSES:
+ assert '"score": 0' in evaluator.prompt
+ assert "至少4项" in evaluator.prompt
+ assert "同义词、近义表达" in evaluator.prompt
+ assert evaluator._required_fields
+
+
+def test_tc609_doc_llm_build_messages_contains_document():
+ document = "这是一份数据集说明文档。"
+ for evaluator in TC609_LLM_CLASSES:
+ messages = evaluator.build_messages(Data(content=document))
+ assert messages == [
+ {"role": "user", "content": evaluator.prompt + document}
+ ]
+
+
+def test_tc609_doc_llm_process_response_pass_and_fail():
+ evaluator = LLM_TC609_0101_DocBasicInfoCompleteness
+
+ passed = evaluator.process_response(
+ '{"score": 1, "reason": "covered: 5项; missing: 无"}'
+ )
+ assert passed.status is False
+ assert passed.reason == ["covered: 5项; missing: 无"]
+
+ failed = evaluator.process_response(
+ '{"score": 0, "reason": "covered: 3项; missing: 访问渠道、技术支持"}'
+ )
+ assert failed.status is True
+ assert failed.reason == [
+ "covered: 3项; missing: 访问渠道、技术支持"
+ ]
diff --git a/test/scripts/model/llm/test_token_usage.py b/test/scripts/model/llm/test_token_usage.py
new file mode 100644
index 00000000..aea2779e
--- /dev/null
+++ b/test/scripts/model/llm/test_token_usage.py
@@ -0,0 +1,156 @@
+from types import SimpleNamespace
+
+from dingo.config.input_args import EvaluatorLLMArgs
+from dingo.io import ResultInfo
+from dingo.io.input import Data
+from dingo.io.output.eval_detail import EvalDetail
+from dingo.model.llm.base import LLMCallResult
+from dingo.model.llm.base_openai import BaseOpenAI
+
+
+def _completion(
+ content='{"score": 1, "reason": "ok"}',
+ usage=None,
+ finish_reason="stop",
+):
+ return SimpleNamespace(
+ choices=[
+ SimpleNamespace(
+ finish_reason=finish_reason,
+ message=SimpleNamespace(content=content),
+ )
+ ],
+ usage=usage,
+ )
+
+
+def test_extract_token_usage_from_openai_response_object():
+ usage = SimpleNamespace(
+ prompt_tokens=11,
+ completion_tokens=7,
+ total_tokens=18,
+ prompt_tokens_details=SimpleNamespace(cached_tokens=3),
+ completion_tokens_details=SimpleNamespace(reasoning_tokens=2),
+ )
+
+ result = BaseOpenAI._extract_token_usage(
+ _completion(usage=usage),
+ model_name="gpt-test",
+ provider="openai",
+ )
+
+ assert result.prompt_tokens == 11
+ assert result.completion_tokens == 7
+ assert result.total_tokens == 18
+ assert result.cached_tokens == 3
+ assert result.reasoning_tokens == 2
+ assert result.model == "gpt-test"
+ assert result.provider == "openai"
+ assert result.calls == 1
+
+
+def test_base_openai_eval_attaches_token_usage():
+ class UsageLLM(BaseOpenAI):
+ prompt = ""
+ dynamic_config = EvaluatorLLMArgs()
+ client = True
+
+ @classmethod
+ def send_messages(cls, messages):
+ return LLMCallResult(
+ content='{"score": 1, "reason": "ok"}',
+ usage=BaseOpenAI._extract_token_usage(
+ _completion(
+ usage={
+ "prompt_tokens": 5,
+ "completion_tokens": 2,
+ "total_tokens": 7,
+ }
+ ),
+ model_name="gpt-test",
+ ),
+ )
+
+ result = UsageLLM.eval(Data(content="sample"))
+
+ assert result.status is False
+ assert result.usage is not None
+ assert result.usage.prompt_tokens == 5
+ assert result.usage.completion_tokens == 2
+ assert result.usage.total_tokens == 7
+
+
+def test_base_openai_error_result_keeps_token_usage():
+ class ParseErrorLLM(BaseOpenAI):
+ prompt = ""
+ dynamic_config = EvaluatorLLMArgs()
+ client = True
+
+ @classmethod
+ def send_messages(cls, messages):
+ return LLMCallResult(
+ content="not json",
+ usage=BaseOpenAI._extract_token_usage(
+ _completion(
+ usage={
+ "prompt_tokens": 3,
+ "completion_tokens": 1,
+ "total_tokens": 4,
+ }
+ ),
+ model_name="gpt-test",
+ ),
+ )
+
+ result = ParseErrorLLM.eval(Data(content="sample"))
+
+ assert result.status is True
+ assert result.label == ["QUALITY_BAD.ConvertJsonError"]
+ assert result.usage is not None
+ assert result.usage.total_tokens == 4
+
+
+def test_base_openai_eval_still_accepts_legacy_string_send_messages():
+ class LegacyLLM(BaseOpenAI):
+ prompt = ""
+ dynamic_config = EvaluatorLLMArgs()
+ client = True
+
+ @classmethod
+ def send_messages(cls, messages):
+ return '{"score": 1, "reason": "ok"}'
+
+ result = LegacyLLM.eval(Data(content="sample"))
+
+ assert result.status is False
+ assert result.usage is None
+
+
+def test_result_info_only_serializes_usage_when_present():
+ with_usage = ResultInfo(
+ dingo_id="1",
+ eval_details={
+ "content": [
+ EvalDetail(
+ metric="LLMMetric",
+ usage=BaseOpenAI._extract_token_usage(
+ _completion(
+ usage={
+ "prompt_tokens": 1,
+ "completion_tokens": 2,
+ "total_tokens": 3,
+ }
+ ),
+ model_name="gpt-test",
+ ),
+ )
+ ]
+ },
+ ).to_dict()
+ without_usage = ResultInfo(
+ dingo_id="2",
+ eval_details={"content": [EvalDetail(metric="RuleMetric")]},
+ ).to_dict()
+
+ assert with_usage["eval_details"]["content"][0]["usage"]["total_tokens"] == 3
+ assert "usage" not in without_usage["eval_details"]["content"][0]
diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py
index bae3279d..b22dc7fe 100644
--- a/test/scripts/model/rule/test_rule_common.py
+++ b/test/scripts/model/rule/test_rule_common.py
@@ -1,6 +1,13 @@
+import pytest
+
+from dingo.config.input_args import EvaluatorRuleArgs
from dingo.io import Data
from dingo.io.output.eval_detail import QualityLabel
-from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords
+from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness,
+ Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange,
+ Rule_TC609_02080101_TextPerplexity)
+from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness
+from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords, RuleWatermark
class TestRuleDocFormulaRepeat:
@@ -13,16 +20,358 @@ def test_rule_doc_formula_repeat(self):
assert res.metric == "RuleDocFormulaRepeat"
assert res.reason == ["Formula has too many consecutive repeated characters, total repeat length: 130, found 1 repeat patterns"]
- def test_rule_unsafe_words(self):
+ def test_requires_configured_watermarks(self, monkeypatch):
+ monkeypatch.setattr(
+ RuleWatermark,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=[]),
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="RuleWatermark requires non-empty dynamic_config.key_list",
+ ):
+ RuleWatermark.eval(Data(content="safe text"))
+
+ def test_rule_unsafe_words(self, monkeypatch):
data = Data(data_id="", prompt="", content="java is good\n \n \n \n hello \n \n but python is better")
- r = RuleUnsafeWords
- r.dynamic_config.key_list = ['av', 'b', 'java']
- tmp = r.eval(data)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None)
+ monkeypatch.setattr(
+ RuleUnsafeWords,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=["av", "b", "java"],
+ refer_path=[],
+ ),
+ )
+ tmp = RuleUnsafeWords.eval(data)
assert tmp.status is True
assert 'av' not in tmp.reason
assert 'b' not in tmp.reason
assert 'java' in tmp.reason
+ def test_rule_unsafe_words_requires_configured_words(self, monkeypatch):
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None)
+ monkeypatch.setattr(
+ RuleUnsafeWords,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=[], refer_path=[]),
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="key_list.*refer_path",
+ ):
+ RuleUnsafeWords.eval(Data(content="safe text"))
+
+ def test_rule_unsafe_words_combines_key_list_and_refer_path(
+ self, monkeypatch, tmp_path
+ ):
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None)
+ unsafe_words_file = tmp_path / "unsafe_words.jsonl"
+ unsafe_words_file.write_text(
+ '{"word": "python"}\n',
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(
+ RuleUnsafeWords,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=["java"],
+ refer_path=[str(unsafe_words_file)],
+ ),
+ )
+
+ result = RuleUnsafeWords.eval(
+ Data(content="java and python are programming languages")
+ )
+
+ assert result.status is True
+ assert result.reason == ["java", "python"]
+
+ def test_rule_unsafe_words_reuses_cached_automaton(self, monkeypatch):
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None)
+ monkeypatch.setattr(
+ RuleUnsafeWords,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=["java"], refer_path=[]),
+ )
+
+ RuleUnsafeWords.eval(Data(content="java"))
+ cached_words = RuleUnsafeWords._unsafe_words_list
+ cached_automaton = RuleUnsafeWords._unsafe_words_automaton
+ RuleUnsafeWords.eval(Data(content="java"))
+
+ assert RuleUnsafeWords._unsafe_words_list is cached_words
+ assert RuleUnsafeWords._unsafe_words_automaton is cached_automaton
+
+
+class TestRule_TC609_02080101_TextPerplexity:
+ @staticmethod
+ def _mock_model(monkeypatch, perplexity):
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "_check_dependencies",
+ classmethod(lambda cls: None),
+ )
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "_get_model_components",
+ classmethod(lambda cls, model_name: (object(), object())),
+ )
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "_calculate_perplexity",
+ classmethod(
+ lambda cls, content, tokenizer, model, stride: perplexity
+ ),
+ )
+
+ def test_high_perplexity_is_bad(self, monkeypatch):
+ self._mock_model(monkeypatch, 125.5)
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ threshold=100.0,
+ model="test-model",
+ stride=64,
+ ),
+ )
+
+ res = Rule_TC609_02080101_TextPerplexity.eval(
+ Data(data_id="ppl-high", content="A valid piece of text.")
+ )
+
+ assert res.status is True
+ assert "125.5000" in res.reason[0]
+ assert "test-model" in res.reason[0]
+
+ def test_low_perplexity_is_good(self, monkeypatch):
+ self._mock_model(monkeypatch, 42.25)
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ threshold=100.0,
+ model="test-model",
+ stride=64,
+ ),
+ )
+
+ res = Rule_TC609_02080101_TextPerplexity.eval(
+ Data(data_id="ppl-low", content="A fluent piece of text.")
+ )
+
+ assert res.status is False
+ assert res.label == [QualityLabel.QUALITY_GOOD]
+ assert "42.2500" in res.reason[0]
+
+ def test_empty_content_is_bad_without_loading_model(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "_check_dependencies",
+ classmethod(lambda cls: None),
+ )
+
+ def fail_if_called(cls, model_name):
+ raise AssertionError("model should not be loaded for empty content")
+
+ monkeypatch.setattr(
+ Rule_TC609_02080101_TextPerplexity,
+ "_get_model_components",
+ classmethod(fail_if_called),
+ )
+
+ res = Rule_TC609_02080101_TextPerplexity.eval(Data(data_id="ppl-empty", content=" "))
+
+ assert res.status is True
+ assert "empty content" in res.reason[0]
+
+ def test_missing_dependencies_raise_clear_error(self, monkeypatch):
+ monkeypatch.setattr(
+ "dingo.model.rule.guobiao.rule_tc609_quality.importlib.util.find_spec",
+ lambda package: None if package == "transformers" else object(),
+ )
+
+ try:
+ Rule_TC609_02080101_TextPerplexity.eval(
+ Data(data_id="ppl-dependency", content="A piece of text.")
+ )
+ except ImportError as exc:
+ assert "transformers" in str(exc)
+ assert "dingo-python[hhem]" in str(exc)
+ else:
+ raise AssertionError("expected ImportError for missing transformers")
+
+
+class TestRule_TC609_0207_DataTypeConsistency:
+ @staticmethod
+ def _mock_classification(monkeypatch, predicted_type, scores):
+ monkeypatch.setattr(
+ Rule_TC609_0207_DataTypeConsistency,
+ "_classify_dataset_type",
+ classmethod(lambda cls, *args: (predicted_type, scores)),
+ )
+ monkeypatch.setattr(
+ Rule_TC609_0207_DataTypeConsistency,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dataset_type="通识数据集",
+ threshold=0.6,
+ model="test-model",
+ device=-1,
+ ),
+ )
+
+ def test_text_matching_dataset_type_is_good(self, monkeypatch):
+ self._mock_classification(
+ monkeypatch,
+ "通识数据集",
+ {"通识数据集": 0.85, "行业通识数据集": 0.1, "行业专识数据集": 0.05},
+ )
+ result = Rule_TC609_0207_DataTypeConsistency.eval(
+ Data(
+ data_id="type-match",
+ data_content=[
+ {"media_type": "text", "content": "普通生活常识"},
+ {"media_type": "image", "content": "image.png"},
+ ],
+ )
+ )
+
+ assert result.status is False
+ assert result.score == 0.85
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+ def test_text_not_matching_dataset_type_is_bad(self, monkeypatch):
+ self._mock_classification(
+ monkeypatch,
+ "行业专识数据集",
+ {"通识数据集": 0.25, "行业通识数据集": 0.2, "行业专识数据集": 0.55},
+ )
+ result = Rule_TC609_0207_DataTypeConsistency.eval(
+ Data(
+ data_id="type-mismatch",
+ data_content=[{"media_type": "text", "content": "专业行业知识"}],
+ )
+ )
+
+ assert result.status is True
+ assert result.score == 0.25
+ assert result.label == [
+ "QUALITY_BAD_TC609_0207.Rule_TC609_0207_DataTypeConsistency"
+ ]
+
+ def test_no_text_content_is_bad(self):
+ result = Rule_TC609_0207_DataTypeConsistency.eval(
+ Data(
+ data_id="no-text",
+ data_content=[{"media_type": "image", "content": "image.png"}],
+ )
+ )
+
+ assert result.status is True
+ assert "at least one text item" in result.reason[0]
+
+ def test_invalid_dataset_type_raises_value_error(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0207_DataTypeConsistency,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dataset_type="医疗数据集",
+ threshold=0.6,
+ model="test-model",
+ device=-1,
+ ),
+ )
+ with pytest.raises(ValueError, match="dataset_type must be one of"):
+ Rule_TC609_0207_DataTypeConsistency.eval(
+ Data(
+ data_id="invalid-type",
+ data_content=[
+ {"media_type": "text", "content": "医疗知识"}
+ ],
+ )
+ )
+
+
+class TestRule_TC609_0303_DataTimeRange:
+ def test_dt_in_range_is_good(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0303_DataTimeRange,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dt_start="2025-01-01",
+ dt_end="2025-12-31 23:59:59",
+ ),
+ )
+
+ result = Rule_TC609_0303_DataTimeRange.eval(
+ Data(
+ data_id="time-good",
+ dt="2025-03-01 08:30:00",
+ )
+ )
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+ def test_created_time_out_of_range_is_bad(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0303_DataTimeRange,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dt_start="2025-01-01",
+ dt_end="2025-12-31 23:59:59",
+ ),
+ )
+
+ result = Rule_TC609_0303_DataTimeRange.eval(
+ Data(
+ data_id="time-created-out",
+ dt="2024-12-31 23:59:59",
+ )
+ )
+ assert result.status is True
+ assert "earlier than allowed start" in result.reason[0]
+
+ def test_dt_invalid_format_is_bad(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0303_DataTimeRange,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dt_start="2025-01-01",
+ dt_end="2025-12-31 23:59:59",
+ ),
+ )
+
+ result = Rule_TC609_0303_DataTimeRange.eval(
+ Data(
+ data_id="time-dt-format",
+ dt="2025年03月01日",
+ )
+ )
+ assert result.status is True
+ assert "unsupported datetime format" in result.reason[0]
+
+ def test_missing_time_field_is_bad(self, monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0303_DataTimeRange,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ dt_start="2025-01-01",
+ dt_end="2025-12-31 23:59:59",
+ ),
+ )
+
+ result = Rule_TC609_0303_DataTimeRange.eval(Data(data_id="time-missing"))
+ assert result.status is True
+ assert "dt is missing" in result.reason[0]
+
class TestRulePIIDetection:
"""PII 检测规则测试"""
@@ -199,3 +548,108 @@ def test_severity_levels(self):
data_low = Data(data_id="14", content="IP：192.168.1.1")
res_low = RulePIIDetection.eval(data_low)
assert "Low Risk" in str(res_low.reason)
+
+
+class TestRuleDatasetDocCompleteness:
+ @staticmethod
+ def _mock_aspect_matching(monkeypatch):
+ def mock_match(
+ cls,
+ content,
+ normalized_content,
+ aspect_keywords,
+ model_name,
+ device,
+ semantic_threshold,
+ ):
+ del cls, content, model_name, device, semantic_threshold
+ matched = {}
+ missing = []
+ for aspect_name, keywords in aspect_keywords.items():
+ evidence_keyword = next(
+ (
+ keyword
+ for keyword in keywords
+ if keyword.lower() in normalized_content
+ ),
+ None,
+ )
+ if evidence_keyword:
+ matched[aspect_name] = {
+ "score": 1.0,
+ "keyword": evidence_keyword,
+ }
+ else:
+ missing.append(aspect_name)
+ return matched, missing
+
+ monkeypatch.setattr(
+ Rule_TC609_01_DocCompleteness,
+ "_match_aspects",
+ classmethod(mock_match),
+ )
+
+ def test_basic_info_completeness_good(self, monkeypatch):
+ self._mock_aspect_matching(monkeypatch)
+ content = (
+ "本数据集说明包含数据集规模与样本数量，给出格式规范和文件结构，"
+ "提供访问渠道，并说明技术支持联系方式。"
+ )
+ res = Rule_TC609_0101_DocBasicInfoCompleteness.eval(
+ Data(data_id="doc-basic-good", content=content)
+ )
+ assert res.status is False
+ assert res.label == [QualityLabel.QUALITY_GOOD]
+ assert res.score == 1.0
+
+ def test_basic_info_completeness_bad(self, monkeypatch):
+ self._mock_aspect_matching(monkeypatch)
+ content = "仅提到样本数量和文件结构，未说明访问渠道。"
+ res = Rule_TC609_0101_DocBasicInfoCompleteness.eval(
+ Data(data_id="doc-basic-bad", content=content)
+ )
+ assert res.status is True
+ assert res.score < 0.8
+
+ def test_content_feature_completeness_good(self, monkeypatch):
+ self._mock_aspect_matching(monkeypatch)
+ content = (
+ "文档包含模态类型、数据分布情况、标签类别统计、样本示例以及局限性说明。"
+ )
+ res = Rule_TC609_0102_DocContentFeatureCompleteness.eval(
+ Data(data_id="doc-content-good", content=content)
+ )
+ assert res.status is False
+ assert res.label == [QualityLabel.QUALITY_GOOD]
+ assert res.score == 1.0
+
+ def test_construction_process_completeness_good(self, monkeypatch):
+ self._mock_aspect_matching(monkeypatch)
+ content = (
+ "建设过程包括数据来源、采集方法、加工处理流程、标注规范和版本控制记录。"
+ )
+ res = Rule_TC609_0103_DocConstructionProcessCompleteness.eval(
+ Data(data_id="doc-process-good", content=content)
+ )
+ assert res.status is False
+ assert res.label == [QualityLabel.QUALITY_GOOD]
+ assert res.score == 1.0
+
+ def test_application_completeness_good(self, monkeypatch):
+ self._mock_aspect_matching(monkeypatch)
+ content = (
+ "应用说明提供使用许可、目标应用场景、评估方法、基准测试结果与典型应用案例。"
+ )
+ res = Rule_TC609_0104_DocApplicationCompleteness.eval(
+ Data(data_id="doc-application-good", content=content)
+ )
+ assert res.status is False
+ assert res.label == [QualityLabel.QUALITY_GOOD]
+ assert res.score == 1.0
+
+ def test_empty_content_is_bad(self):
+ res = Rule_TC609_0104_DocApplicationCompleteness.eval(
+ Data(data_id="doc-empty", content=" ")
+ )
+ assert res.status is True
+ assert "missing or empty" in res.reason[0]
diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py
new file mode 100644
index 00000000..06d84c66
--- /dev/null
+++ b/test/scripts/model/rule/test_rule_tc609_quality.py
@@ -0,0 +1,996 @@
+import inspect
+
+import pytest
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.io.input import RequiredField
+from dingo.io.output.eval_detail import EvalDetail, QualityLabel
+from dingo.model.model import Model
+from dingo.model.rule.guobiao import rule_tc609_quality, rule_tc609_quality_base
+from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance,
+ Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0206_ContentConsistency,
+ Rule_TC609_0208_ContentCleanliness, Rule_TC609_0301_ContentDiversity)
+from dingo.model.rule.rule_common import RuleWatermark
+
+
+def test_only_supported_tc609_quality_metrics_are_registered():
+ rule_classes = {
+ name: cls
+ for name, cls in inspect.getmembers(
+ rule_tc609_quality,
+ lambda value: inspect.isclass(value)
+ and value.__module__ == rule_tc609_quality.__name__,
+ )
+ if name.startswith("Rule_TC609_")
+ }
+
+ assert len(rule_classes) == 40
+ expected_registered = {
+ f"Rule_TC609_020{index}_{suffix}"
+ for index, suffix in enumerate(
+ (
+ "FormatCompliance",
+ "SafetyCompliance",
+ "AnnotationCompliance",
+ "StructuralCompleteness",
+ "ContentAuthenticity",
+ "ContentConsistency",
+ "DataTypeConsistency",
+ "ContentCleanliness",
+ ),
+ start=1,
+ )
+ }
+ actual_registered = set(rule_classes) & set(Model.rule_name_map)
+ assert actual_registered == expected_registered
+
+ expected_primary_codes = {
+ "0101", "0102", "0103", "0104",
+ "0201", "0202", "0203", "0204", "0205", "0206", "0207", "0208",
+ "0301", "0302", "0303", "0304", "0305",
+ }
+ actual_codes = {
+ name.split("_")[2]
+ for name in rule_classes
+ if len(name.split("_")[2]) == 4
+ }
+ assert actual_codes == expected_primary_codes
+
+
+def test_only_supported_tc609_rules_are_grouped_as_data_rules():
+ actual_groups = {
+ group_name: {
+ rule.__name__
+ for rule in rules
+ if rule.__name__.startswith("Rule_TC609_")
+ }
+ for group_name, rules in Model.rule_groups.items()
+ }
+ actual_groups = {
+ group_name: rules
+ for group_name, rules in actual_groups.items()
+ if rules
+ }
+ assert actual_groups == {
+ "guobiao_data": {
+ f"Rule_TC609_020{index}_{suffix}"
+ for index, suffix in enumerate(
+ (
+ "FormatCompliance",
+ "SafetyCompliance",
+ "AnnotationCompliance",
+ "StructuralCompleteness",
+ "ContentAuthenticity",
+ "ContentConsistency",
+ "DataTypeConsistency",
+ "ContentCleanliness",
+ ),
+ start=1,
+ )
+ }
+ }
+
+
+def test_format_compliance_accepts_matching_record(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ field_schema={
+ "data_id": "str",
+ "content": "str",
+ "type": "str",
+ "dt": "str",
+ }
+ ),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(
+ data_id="demo-001",
+ content="example",
+ type="medical",
+ dt="2026-07-20 09:00:00",
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_format_compliance_default_schema_matches_tc609_metadata():
+ assert Rule_TC609_0201_FormatCompliance.dynamic_config.field_schema == {
+ "id": "str",
+ "rid": "Optional[list]",
+ "data_content": "list",
+ "annotation": "Optional[dict]",
+ "original_time": "str",
+ "last_modified_time": "str",
+ "version": "str",
+ "license": "str",
+ "source": "str",
+ "source_details": "str",
+ "generated_data_indicator": "int",
+ }
+
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(
+ id="d6c9a4d5e57597df8fe30f09ae44c985",
+ rid=None,
+ data_content=[
+ {
+ "media_type": "image",
+ "content": "../data/images/streetscape.jpg",
+ }
+ ],
+ annotation=None,
+ original_time="2025-1-1",
+ last_modified_time="2025-1-1",
+ version="1.0.0-alpha",
+ license="其他",
+ source="互联网",
+ source_details="https://example.com/image.jpg",
+ generated_data_indicator=0,
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_format_compliance_default_schema_requires_mandatory_fields():
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(
+ id="dataset-id",
+ data_content={},
+ )
+ )
+
+ assert result.status is True
+ assert "data_content: expected list, got dict" in result.reason
+ assert "rid: required field is missing" in result.reason
+ assert "annotation: required field is missing" in result.reason
+ assert "original_time: required field is missing" in result.reason
+
+
+def test_format_compliance_reports_missing_and_wrong_type(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ field_schema={
+ "data_id": "str",
+ "content": "str",
+ "dt": "str",
+ }
+ ),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(data_id=1, content="example")
+ )
+
+ assert result.status is True
+ assert result.label == [
+ "QUALITY_BAD_TC609_0201.Rule_TC609_0201_FormatCompliance"
+ ]
+ assert result.reason == [
+ "data_id: expected str, got int",
+ "dt: required field is missing",
+ ]
+
+
+def test_format_compliance_reports_unexpected_fields(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ field_schema={"content": "str"},
+ allow_extra=False,
+ ),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(content="example", source="demo")
+ )
+
+ assert result.status is True
+ assert result.reason == ["source: unexpected field"]
+
+
+def test_format_compliance_allows_unexpected_fields_by_default(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(field_schema={"content": "str"}),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(
+ Data(content="example", source="demo")
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+@pytest.mark.parametrize(
+ "value, type_name",
+ [
+ (None, "Optional[str]"),
+ ("text", "Optional[str]"),
+ (None, "Optional[int]"),
+ (1, "Optional[int]"),
+ (None, "Optional[float]"),
+ (1.5, "Optional[float]"),
+ (None, "Optional[bool]"),
+ (True, "Optional[bool]"),
+ (None, "Optional[list]"),
+ ([], "Optional[list]"),
+ (None, "Optional[dict]"),
+ ({}, "Optional[dict]"),
+ ],
+)
+def test_format_compliance_accepts_optional_types(
+ monkeypatch, value, type_name
+):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(field_schema={"value": type_name}),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(Data(value=value))
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_format_compliance_still_requires_optional_field(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(field_schema={"value": "Optional[str]"}),
+ )
+
+ result = Rule_TC609_0201_FormatCompliance.eval(Data())
+
+ assert result.status is True
+ assert result.reason == ["value: required field is missing"]
+
+
+@pytest.mark.parametrize(
+ "schema, error",
+ [
+ (None, "requires a non-empty dynamic_config.field_schema"),
+ ({}, "requires a non-empty dynamic_config.field_schema"),
+ ({"content": "string"}, "Unsupported schema type"),
+ ({"content": {"type": "str"}}, "Unsupported schema type"),
+ ({"content": "optional_string"}, "Unsupported schema type"),
+ ({"content": "optional_str"}, "Unsupported schema type"),
+ ({"content": "optional[str]"}, "Unsupported schema type"),
+ ({"content": "typing.Optional[str]"}, "Unsupported schema type"),
+ ({"content": "optianal[str]"}, "Unsupported schema type"),
+ ],
+)
+def test_format_compliance_rejects_invalid_schema(monkeypatch, schema, error):
+ monkeypatch.setattr(
+ Rule_TC609_0201_FormatCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(field_schema=schema),
+ )
+
+ with pytest.raises(ValueError, match=error):
+ Rule_TC609_0201_FormatCompliance.eval(Data(content="example"))
+
+
+def test_composite_rule_maps_component_failure_to_tc609_label(monkeypatch):
+ class PassingRule:
+ @classmethod
+ def eval(cls, input_data):
+ return EvalDetail(
+ metric=cls.__name__,
+ label=[QualityLabel.QUALITY_GOOD],
+ )
+
+ class FailingRule:
+ @classmethod
+ def eval(cls, input_data):
+ return EvalDetail(
+ metric=cls.__name__,
+ status=True,
+ label=["QUALITY_BAD_TEST.FailingRule"],
+ reason=["component failed"],
+ )
+
+ component_map = {
+ Rule_TC609_0202_SafetyCompliance.component_rules[0]: PassingRule,
+ Rule_TC609_0202_SafetyCompliance.component_rules[1]: FailingRule,
+ Rule_TC609_0202_SafetyCompliance.component_rules[2]: PassingRule,
+ }
+ monkeypatch.setattr(
+ Rule_TC609_0202_SafetyCompliance,
+ "_resolve_rule",
+ classmethod(lambda cls, path: component_map[path]),
+ )
+
+ result = Rule_TC609_0202_SafetyCompliance.eval(
+ Data(
+ data_id="safety",
+ data_content=[{"media_type": "text", "content": "test"}],
+ )
+ )
+
+ assert result.status is True
+ assert result.label == [
+ "QUALITY_BAD_TC609_0202.Rule_TC609_0202_SafetyCompliance"
+ ]
+ assert result.reason == ["FailingRule: component failed"]
+
+
+def test_safety_compliance_passes_words_config_to_unsafe_rule(monkeypatch):
+ class UnsafeWordsRule:
+ dynamic_config = EvaluatorRuleArgs()
+ _unsafe_words_list = None
+ _unsafe_words_automaton = None
+
+ @classmethod
+ def eval(cls, input_data):
+ if "unsafe" in cls.dynamic_config.key_list:
+ return EvalDetail(
+ metric=cls.__name__,
+ status=True,
+ label=["QUALITY_BAD_SECURITY.UnsafeWordsRule"],
+ reason=["unsafe"],
+ )
+ return EvalDetail(
+ metric=cls.__name__,
+ label=[QualityLabel.QUALITY_GOOD],
+ )
+
+ class PassingRule:
+ @classmethod
+ def eval(cls, input_data):
+ return EvalDetail(
+ metric=cls.__name__,
+ label=[QualityLabel.QUALITY_GOOD],
+ )
+
+ component_map = {
+ Rule_TC609_0202_SafetyCompliance.component_rules[0]: UnsafeWordsRule,
+ Rule_TC609_0202_SafetyCompliance.component_rules[1]: PassingRule,
+ Rule_TC609_0202_SafetyCompliance.component_rules[2]: PassingRule,
+ }
+ monkeypatch.setattr(
+ Rule_TC609_0202_SafetyCompliance,
+ "_resolve_rule",
+ classmethod(lambda cls, path: component_map[path]),
+ )
+ monkeypatch.setattr(
+ Rule_TC609_0202_SafetyCompliance,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=["unsafe"], refer_path=[]),
+ )
+
+ result = Rule_TC609_0202_SafetyCompliance.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "safe"},
+ {"media_type": "image", "content": "unsafe"},
+ {"media_type": "text", "content": "unsafe"},
+ ]
+ )
+ )
+
+ assert UnsafeWordsRule.dynamic_config.key_list == ["unsafe"]
+ assert result.status is True
+ assert result.reason == ["UnsafeWordsRule: unsafe"]
+
+
+def test_safety_compliance_has_usable_default_words(monkeypatch):
+ from dingo.model.rule.rule_common import RuleUnsafeWords
+
+ assert Rule_TC609_0202_SafetyCompliance.dynamic_config.key_list
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None)
+ monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None)
+
+ result = Rule_TC609_0202_SafetyCompliance.eval(
+ Data(
+ data_content=[
+ {
+ "media_type": "text",
+ "content": "该内容提供制作炸弹的具体步骤",
+ }
+ ]
+ )
+ )
+
+ assert result.status is True
+ assert "RuleUnsafeWords: 制作炸弹" in result.reason
+
+
+def test_safety_compliance_requires_text_content():
+ result = Rule_TC609_0202_SafetyCompliance.eval(
+ Data(
+ data_content=[
+ {"media_type": "image", "content": "制作炸弹"}
+ ]
+ )
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "data_content: at least one text item is required"
+ ]
+
+
+def test_safety_compliance_declares_data_content():
+ assert Rule_TC609_0202_SafetyCompliance._required_fields == [
+ RequiredField.DATA_CONTENT
+ ]
+
+
+def test_annotation_compliance_accepts_valid_metadata():
+ result = Rule_TC609_0203_AnnotationCompliance.eval(
+ Data(
+ annotation={
+ "label": [
+ {
+ "iscrowd": 0,
+ "bbox": [20, 20, 20, 20],
+ "category": "human",
+ }
+ ],
+ "annotation_method": "人工标注",
+ "annotator": "普通标注员",
+ }
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_annotation_compliance_accepts_none_for_unannotated_data():
+ result = Rule_TC609_0203_AnnotationCompliance.eval(
+ Data(annotation=None)
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_annotation_compliance_declares_required_field():
+ assert Rule_TC609_0203_AnnotationCompliance._required_fields == [
+ RequiredField.ANNOTATION
+ ]
+
+
+def test_annotation_compliance_reports_invalid_nested_metadata():
+ result = Rule_TC609_0203_AnnotationCompliance.eval(
+ Data(
+ annotation={
+ "label": [],
+ "annotation_method": "众包标注",
+ "annotator": 1,
+ }
+ )
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "annotation.label: empty value is not allowed",
+ (
+ "annotation.annotation_method: unsupported value '众包标注'; "
+ "allowed values: 人工标注, 其他, 半自动标注, 自动标注"
+ ),
+ "annotation.annotator: expected str or None, got int",
+ ]
+
+
+def test_annotation_compliance_requires_all_nested_fields():
+ result = Rule_TC609_0203_AnnotationCompliance.eval(
+ Data(annotation={})
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "annotation.label: required field is missing",
+ "annotation.annotation_method: required field is missing",
+ "annotation.annotator: required field is missing",
+ ]
+
+
+def test_content_cleanliness_propagates_empty_watermark_config(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0208_ContentCleanliness,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=[]),
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="RuleWatermark requires non-empty dynamic_config.key_list",
+ ):
+ Rule_TC609_0208_ContentCleanliness.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "safe text"}
+ ]
+ )
+ )
+
+
+def test_content_cleanliness_passes_key_list_to_watermark(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0208_ContentCleanliness,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=["DINGO-WATERMARK"]),
+ )
+ monkeypatch.setattr(
+ RuleWatermark,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=["stale-value"]),
+ )
+
+ result = Rule_TC609_0208_ContentCleanliness.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "text with"},
+ {"media_type": "image", "content": "DINGO-WATERMARK"},
+ {"media_type": "text", "content": "DINGO-WATERMARK"},
+ ]
+ )
+ )
+
+ assert RuleWatermark.dynamic_config.key_list == ["DINGO-WATERMARK"]
+ assert result.status is True
+ assert result.reason == ["RuleWatermark: DINGO-WATERMARK"]
+
+
+def test_content_cleanliness_returns_good_without_watermark(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0208_ContentCleanliness,
+ "dynamic_config",
+ EvaluatorRuleArgs(key_list=["DINGO-WATERMARK"]),
+ )
+
+ result = Rule_TC609_0208_ContentCleanliness.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "ordinary"},
+ {"media_type": "text", "content": "clean text"},
+ {"media_type": "image", "content": "DINGO-WATERMARK"},
+ ]
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_content_cleanliness_has_usable_default_watermarks(monkeypatch):
+ assert Rule_TC609_0208_ContentCleanliness.dynamic_config.key_list
+
+ result = Rule_TC609_0208_ContentCleanliness.eval(
+ Data(
+ data_content=[
+ {
+ "media_type": "text",
+ "content": "本文版权所有，未经授权不得转载。",
+ }
+ ]
+ )
+ )
+
+ assert result.status is True
+ assert "RuleWatermark: 版权所有" in result.reason
+
+
+def test_content_cleanliness_requires_text_content():
+ result = Rule_TC609_0208_ContentCleanliness.eval(
+ Data(
+ data_content=[
+ {"media_type": "image", "content": "image.png"}
+ ]
+ )
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "data_content: at least one text item is required"
+ ]
+
+
+def test_structural_completeness_accepts_present_values(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0204_StructuralCompleteness,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=["content", "labels", "metadata"],
+ allow_none=False,
+ allow_empty=False,
+ ),
+ )
+
+ result = Rule_TC609_0204_StructuralCompleteness.eval(
+ Data(content="example", labels=["valid"], metadata={"source": "demo"})
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_structural_completeness_has_tc609_required_fields_by_default():
+ assert Rule_TC609_0204_StructuralCompleteness.dynamic_config.key_list == [
+ "id",
+ "data_content",
+ "original_time",
+ "last_modified_time",
+ "version",
+ "license",
+ "source",
+ "source_details",
+ "generated_data_indicator",
+ ]
+
+ result = Rule_TC609_0204_StructuralCompleteness.eval(
+ Data(
+ id="dataset-id",
+ data_content=[{"media_type": "text", "content": "example"}],
+ original_time="2025-01-01",
+ last_modified_time="2025-01-01",
+ version="1.0.0",
+ license="其他",
+ source="互联网",
+ source_details="https://example.com/data",
+ generated_data_indicator=0,
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_structural_completeness_reports_missing_none_and_empty(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0204_StructuralCompleteness,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=["missing", "nullable", "text", "items", "metadata"],
+ allow_none=False,
+ allow_empty=False,
+ ),
+ )
+
+ result = Rule_TC609_0204_StructuralCompleteness.eval(
+ Data(nullable=None, text="", items=[], metadata={})
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "missing: required field is missing",
+ "nullable: None is not allowed",
+ "text: empty value is not allowed",
+ "items: empty value is not allowed",
+ "metadata: empty value is not allowed",
+ ]
+
+
+def test_structural_completeness_allows_none_and_empty(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0204_StructuralCompleteness,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=["nullable", "text", "items", "metadata"],
+ allow_none=True,
+ allow_empty=True,
+ ),
+ )
+
+ result = Rule_TC609_0204_StructuralCompleteness.eval(
+ Data(nullable=None, text="", items=[], metadata={})
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_structural_completeness_requires_key_list(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0204_StructuralCompleteness,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ key_list=[],
+ allow_none=False,
+ allow_empty=False,
+ ),
+ )
+
+ with pytest.raises(ValueError, match="non-empty dynamic_config.key_list"):
+ Rule_TC609_0204_StructuralCompleteness.eval(Data(content="example"))
+
+
+@pytest.mark.parametrize(
+ "source_details",
+ [
+ "https://example.com/data/1",
+ "http://localhost:8080/record?id=1",
+ ],
+)
+def test_content_authenticity_accepts_valid_internet_url(source_details):
+ result = Rule_TC609_0205_ContentAuthenticity.eval(
+ Data(source="互联网", source_details=source_details)
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_content_authenticity_accepts_non_url_source_details():
+ result = Rule_TC609_0205_ContentAuthenticity.eval(
+ Data(
+ source="图书",
+ source_details="ISBN 978-7-121-15535-2，第 10 页",
+ )
+ )
+
+ assert result.status is False
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+@pytest.mark.parametrize(
+ "source, source_details, reason",
+ [
+ (None, "detail", "source: expected a non-empty string"),
+ ("", "detail", "source: expected a non-empty string"),
+ ("图书", None, "source_details: expected a non-empty string"),
+ ("图书", "", "source_details: expected a non-empty string"),
+ ],
+)
+def test_content_authenticity_rejects_empty_source_metadata(
+ source, source_details, reason
+):
+ result = Rule_TC609_0205_ContentAuthenticity.eval(
+ Data(source=source, source_details=source_details)
+ )
+
+ assert result.status is True
+ assert result.reason == [reason]
+
+
+@pytest.mark.parametrize(
+ "source, source_details",
+ [
+ ("互联网", "example.com/data/1"),
+ ("互联网", "ftp://example.com/data/1"),
+ ("互联网", "https://"),
+ ("互联网", "https://exa mple.com/data/1"),
+ ("互联网", "https://example.com:invalid/data/1"),
+ ],
+)
+def test_content_authenticity_rejects_invalid_url(source, source_details):
+ result = Rule_TC609_0205_ContentAuthenticity.eval(
+ Data(source=source, source_details=source_details)
+ )
+
+ assert result.status is True
+ assert result.reason == [
+ "source_details: expected a valid HTTP or HTTPS URL"
+ ]
+
+
+def test_content_authenticity_declares_required_fields():
+ assert Rule_TC609_0205_ContentAuthenticity._required_fields == [
+ RequiredField.SOURCE,
+ RequiredField.SOURCE_DETAILS,
+ ]
+
+
+def test_content_consistency_accepts_consistent_text_items(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0206_ContentConsistency,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ threshold=0.5,
+ model="test-model",
+ device=-1,
+ ),
+ )
+ monkeypatch.setattr(
+ rule_tc609_quality,
+ "calculate_text_consistency",
+ lambda **kwargs: {
+ "score": 0.85,
+ "is_consistent": True,
+ "item_scores": [0.9, 0.85, 0.8],
+ "outlier_indexes": [],
+ },
+ )
+
+ result = Rule_TC609_0206_ContentConsistency.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "海底管道砂袋防护"},
+ {"media_type": "image", "content": "pipeline.jpg"},
+ {"media_type": "text", "content": "砂袋用于保护海底管道"},
+ {"media_type": "text", "content": "模拟砂袋周围流场"},
+ ]
+ )
+ )
+
+ assert result.status is False
+ assert result.score == 0.85
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+def test_content_consistency_rejects_inconsistent_text_items(monkeypatch):
+ monkeypatch.setattr(
+ Rule_TC609_0206_ContentConsistency,
+ "dynamic_config",
+ EvaluatorRuleArgs(
+ threshold=0.5,
+ model="test-model",
+ device=-1,
+ ),
+ )
+ monkeypatch.setattr(
+ rule_tc609_quality,
+ "calculate_text_consistency",
+ lambda **kwargs: {
+ "score": 0.2,
+ "is_consistent": False,
+ "item_scores": [0.9, 0.2],
+ "outlier_indexes": [1],
+ },
+ )
+
+ result = Rule_TC609_0206_ContentConsistency.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "健康知识"},
+ {"media_type": "image", "content": "health.jpg"},
+ {"media_type": "text", "content": "金融市场"},
+ ]
+ )
+ )
+
+ assert result.status is True
+ assert result.score == 0.2
+ assert result.reason == [
+ "Text items in data_content are inconsistent "
+ "(score: 0.2000, threshold: 0.5000, outlier indexes: [2])"
+ ]
+
+
+def test_content_consistency_skips_comparison_for_one_text_item():
+ result = Rule_TC609_0206_ContentConsistency.eval(
+ Data(
+ data_content=[
+ {"media_type": "text", "content": "文本内容"},
+ {"media_type": "image", "content": "image.jpg"},
+ ]
+ )
+ )
+
+ assert result.status is False
+ assert result.score is None
+ assert result.label == [QualityLabel.QUALITY_GOOD]
+
+
+@pytest.mark.parametrize(
+ "data, reason",
+ [
+ (
+ Data(data_content=[]),
+ "data_content: expected a non-empty list",
+ ),
+ (
+ Data(data_content=["text"]),
+ "data_content[0]: expected dict, got str",
+ ),
+ (
+ Data(data_content=[{"media_type": ["text"], "content": "文本"}]),
+ "data_content[0].media_type: expected a non-empty string",
+ ),
+ (
+ Data(data_content=[{"media_type": "text", "content": ""}]),
+ (
+ "data_content[0].content: "
+ "expected a non-empty string for text media"
+ ),
+ ),
+ ],
+)
+def test_content_consistency_rejects_invalid_data_content(data, reason):
+ result = Rule_TC609_0206_ContentConsistency.eval(data)
+
+ assert result.status is True
+ assert result.reason == [reason]
+
+
+def test_calculate_text_consistency_compares_two_texts_directly(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ monkeypatch.setattr(
+ rule_tc609_quality_base,
+ "_encode_texts",
+ lambda *args, **kwargs: torch.tensor(
+ [[1.0, 0.0], [0.8, 0.6]]
+ ),
+ )
+
+ result = rule_tc609_quality_base.calculate_text_consistency(
+ ["标题", "摘要"],
+ model_name="test-model",
+ threshold=0.75,
+ )
+
+ assert result["score"] == pytest.approx(0.8)
+ assert result["is_consistent"] is True
+ assert result["outlier_indexes"] == []
+
+
+def test_calculate_text_consistency_uses_robust_center(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ encoded = torch.tensor(
+ [
+ [1.0, 0.0],
+ [0.99, 0.1],
+ [0.98, -0.1],
+ [0.97, 0.05],
+ [0.0, 1.0],
+ ]
+ )
+ encoded = torch.nn.functional.normalize(encoded, p=2, dim=1)
+ monkeypatch.setattr(
+ rule_tc609_quality_base,
+ "_encode_texts",
+ lambda *args, **kwargs: encoded,
+ )
+
+ result = rule_tc609_quality_base.calculate_text_consistency(
+ ["文本1", "文本2", "文本3", "文本4", "离群文本"],
+ model_name="test-model",
+ threshold=0.5,
+ consensus_keep_ratio=0.8,
+ )
+
+ assert result["score"] < 0.5
+ assert result["is_consistent"] is False
+ assert result["outlier_indexes"] == [4]
+
+
+def test_content_consistency_declares_data_content_required():
+ assert Rule_TC609_0206_ContentConsistency._required_fields == [
+ RequiredField.DATA_CONTENT
+ ]
+
+
+def test_uncovered_rule_is_explicit_placeholder():
+ assert Rule_TC609_0301_ContentDiversity.group == []
+ with pytest.raises(NotImplementedError, match="placeholder"):
+ Rule_TC609_0301_ContentDiversity.eval(
+ Data(data_id="diversity", content="test")
+ )
diff --git a/test/scripts/retrieval/test_open_eval.py b/test/scripts/retrieval/test_open_eval.py
index 45bd507f..dacb0973 100644
--- a/test/scripts/retrieval/test_open_eval.py
+++ b/test/scripts/retrieval/test_open_eval.py
@@ -82,6 +82,49 @@ def test_invalid_json(self):
assert grade.error
assert "JSON parse failed" in grade.error
+ def test_json_embedded_in_text(self):
+ response = 'Here is the grade:\n{"score": 0.6, "query_relevance": 0.7, "result_quality": 0.5, "content_issues": false, "confidence": 0.8, "reasoning": "ok"}'
+ grade = _parse_grade_response(response)
+ assert grade.score == 0.6
+ assert grade.error == ""
+
+ def test_lenient_parse_unescaped_quotes_in_reasoning(self):
+ response = (
+ '{"reasoning": "The query "resilience" directly matches the result", '
+ '"query_relevance": 0.95, "result_quality": 0.9, '
+ '"content_issues": false, "confidence": 0.85, "score": 0.92}'
+ )
+ grade = _parse_grade_response(response)
+ assert grade.error == ""
+ assert grade.score == 0.92
+ assert grade.query_relevance == 0.95
+ assert grade.result_quality == 0.9
+ assert grade.content_issues is False
+ assert grade.confidence == 0.85
+
+ def test_repairs_unescaped_quotes_when_reasoning_is_last(self):
+ response = (
+ '{"score": 0.92, "query_relevance": 0.95, "result_quality": 0.9, '
+ '"content_issues": false, "confidence": 0.85, '
+ '"reasoning": "The query "PBPK" directly matches"}'
+ )
+
+ grade = _parse_grade_response(response)
+
+ assert grade.error == ""
+ assert grade.score == 0.92
+ assert grade.reasoning == 'The query "PBPK" directly matches'
+
+ def test_lenient_parse_when_reasoning_is_last(self):
+ response = '{"score": 0.7, "query_relevance": 0.8, "broken": ???, "reasoning": "final reason"}'
+
+ grade = _parse_grade_response(response)
+
+ assert grade.error == ""
+ assert grade.score == 0.7
+ assert grade.query_relevance == 0.8
+ assert grade.reasoning == "final reason"
+
def test_missing_fields_default_to_zero(self):
grade = _parse_grade_response('{"score": 0.5}')
assert grade.score == 0.5
diff --git a/test/scripts/retrieval/test_search_client.py b/test/scripts/retrieval/test_search_client.py
index 05437276..3bde4619 100644
--- a/test/scripts/retrieval/test_search_client.py
+++ b/test/scripts/retrieval/test_search_client.py
@@ -2,7 +2,14 @@
import pytest
-from dingo.retrieval.search_client import PaperResult, SearchClient, SearchResponse, create_client, list_backends, register_backend
+from dingo.retrieval.search_client import ( # isort: skip
+ PaperResult,
+ SearchClient,
+ SearchResponse,
+ create_client,
+ list_backends,
+ register_backend,
+)
class TestPaperResult:
@@ -95,6 +102,23 @@ def test_create_meta_search_client(self):
}
]
+ def test_meta_search_resource_type_becomes_metadata_filter(self):
+ client = create_client(
+ "meta_search",
+ api_url="https://api.sciverse.space",
+ search_type="ebook",
+ )
+
+ payload = client._build_public_payload("test query", 10)
+
+ assert payload["filters"] == [
+ {
+ "field": "metadata_type",
+ "operator": "FILTER_OP_EQ",
+ "value": "ebook",
+ }
+ ]
+
def test_create_openalex_client_defaults_to_search(self):
client = create_client(
"openalex",
@@ -189,6 +213,22 @@ def test_parse_serpapi_result(self):
class TestOpenAlexClient:
+ def test_accepts_full_works_endpoint(self):
+ from dingo.retrieval.backends.openalex import OpenAlexClient
+
+ client = OpenAlexClient(api_url="https://api.openalex.org/works")
+
+ assert client.base_url == "https://api.openalex.org"
+
+ def test_default_select_contains_quality_evaluation_fields(self):
+ from dingo.retrieval.backends.openalex import OpenAlexClient
+
+ select = OpenAlexClient()._build_params("test", 10)["select"]
+
+ assert "authorships" in select
+ assert "keywords" in select
+ assert "primary_location" in select
+
def test_abstract_from_inverted_index(self):
from dingo.retrieval.backends.openalex import OpenAlexClient
diff --git a/test/scripts/retrieval/test_search_result_eval_script.py b/test/scripts/retrieval/test_search_result_eval_script.py
new file mode 100644
index 00000000..ed300db0
--- /dev/null
+++ b/test/scripts/retrieval/test_search_result_eval_script.py
@@ -0,0 +1,272 @@
+"""Tests for the end-to-end search result evaluation example."""
+
+import json
+import os
+import sys
+from argparse import Namespace
+from pathlib import Path
+from types import SimpleNamespace
+
+EXAMPLE_DIR = Path(__file__).resolve().parents[3] / "examples" / "retrieval"
+if str(EXAMPLE_DIR) not in sys.path:
+ sys.path.insert(0, str(EXAMPLE_DIR))
+
+import sdk_eval_search_result # noqa: E402
+from sdk_eval_search_result import build_reports, clear_executor_classification_dirs, load_env_file, normalize_search_result, retrieve_queries # noqa: E402,E501
+from search_result_eval_utils import load_queries, write_classified_jsonl # noqa: E402
+
+from dingo.retrieval.search_client import PaperResult, SearchResponse # noqa: E402
+
+
+def _record(
+ query: str,
+ rank: int,
+ relevance: float,
+ effectiveness: float,
+ authority: float,
+) -> dict:
+ return {
+ "raw_data": {
+ "query": query,
+ "query_index": 1,
+ "rank": rank,
+ "title": f"Result {rank}",
+ "search_result": {
+ "title": f"Result {rank}",
+ "abstract": f"Abstract {rank}",
+ "_eval_query": query,
+ },
+ },
+ "eval_details": {
+ "search_result": [
+ {"metric": "LLMSearchResultRelevance", "score": relevance},
+ {"metric": "LLMSearchResultEffectiveness", "score": effectiveness},
+ {"metric": "LLMSearchResultAuthority", "score": authority},
+ ]
+ }
+ }
+
+
+def _args() -> Namespace:
+ return Namespace(
+ top_k=10,
+ threshold=0.15,
+ openai_model="test-model",
+ prompt_mode="detailed",
+ llm_max_tokens=1024,
+ openai_temperature=0.0,
+ llm_timeout=60.0,
+ llm_workers=1,
+ disable_effectiveness_llm_quality=False,
+ effectiveness_llm_max_tokens=512,
+ )
+
+
+def test_load_queries_supports_query_only_jsonl(tmp_path):
+ path = tmp_path / "queries.jsonl"
+ path.write_text(
+ '\n'.join((json.dumps({"query": "first"}), json.dumps({"query_text": "second"}))),
+ encoding="utf-8",
+ )
+
+ assert load_queries(path) == ["first", "second"]
+
+
+def test_load_queries_deduplicates_queries(tmp_path):
+ path = tmp_path / "queries.jsonl"
+ path.write_text(
+ "\n".join((json.dumps({"query": "same"}), json.dumps({"query": "same"}))),
+ encoding="utf-8",
+ )
+
+ assert load_queries(path) == ["same"]
+
+
+def test_load_env_file_does_not_override_process_environment(tmp_path, monkeypatch):
+ path = tmp_path / ".env"
+ path.write_text("EXISTING=value-from-file\nNEW_VALUE=loaded\n", encoding="utf-8")
+ monkeypatch.setenv("EXISTING", "value-from-process")
+ monkeypatch.delenv("NEW_VALUE", raising=False)
+
+ load_env_file(path)
+
+ assert os.environ["EXISTING"] == "value-from-process"
+ assert os.environ["NEW_VALUE"] == "loaded"
+
+
+def test_retrieve_queries_writes_reusable_results_and_request_log(tmp_path, monkeypatch):
+ input_path = tmp_path / "queries.jsonl"
+ result_path = tmp_path / "retrieval_results.jsonl"
+ log_path = tmp_path / "request_log.jsonl"
+ input_path.write_text(json.dumps({"query": "test query"}), encoding="utf-8")
+
+ class FakeClient:
+ def search(self, query, limit=10):
+ return SearchResponse(
+ query=query,
+ results=[PaperResult(paper_id="p1", title="Test result", raw={"title": "Test result"})],
+ response_time_ms=12.5,
+ status_code=200,
+ )
+
+ monkeypatch.setattr(sdk_eval_search_result, "_build_search_client", lambda args: FakeClient())
+ args = Namespace(
+ input_jsonl=input_path,
+ max_queries=None,
+ top_k=10,
+ search_workers=1,
+ retrieval_backend="meta_search",
+ )
+
+ summary = retrieve_queries(args, result_path, log_path)
+ saved = json.loads(result_path.read_text(encoding="utf-8"))
+ log = json.loads(log_path.read_text(encoding="utf-8"))
+
+ assert saved["query"] == "test query"
+ assert saved["results"][0]["title"] == "Test result"
+ assert log["result_count"] == 1
+ assert summary["success_count"] == 1
+
+
+def test_normalize_openalex_result_maps_metric_fields():
+ raw = {
+ "id": "https://openalex.org/W1",
+ "doi": "https://doi.org/10.1/example",
+ "cited_by_count": 12,
+ "publication_year": 2025,
+ "type": "article",
+ "language": "en",
+ "keywords": [{"display_name": "Search"}],
+ "authorships": [
+ {"author": {"display_name": "A. Author", "orcid": "https://orcid.org/1"}}
+ ],
+ "primary_location": {
+ "source": {
+ "display_name": "Journal of Testing",
+ "type": "journal",
+ "issn": ["1234-5678"],
+ "host_organization_name": "Test Publisher",
+ }
+ },
+ }
+ paper = PaperResult(
+ paper_id=raw["id"],
+ title="A result",
+ abstract="An abstract",
+ score=9.5,
+ year=2025,
+ raw=raw,
+ )
+
+ result = normalize_search_result(paper, "openalex")
+
+ assert result["citation_count"] == 12
+ assert result["keywords"] == ["Search"]
+ assert result["author"][0]["name"] == "A. Author"
+ assert result["publication_venue_name_unified"] == "Journal of Testing"
+ assert result["publication_venue_type"] == "journal"
+ assert result["publication_publisher"] == ["Test Publisher"]
+
+
+def test_meta_search_accepts_full_endpoint():
+ from dingo.retrieval.backends.agentic import MetaSearchClient
+
+ client = MetaSearchClient(api_url="https://api.sciverse.space/meta-search")
+
+ assert client.base_url == "https://api.sciverse.space"
+
+
+def test_query_report_uses_rank_weighted_means_and_embeds_full_results():
+ records = [
+ _record("query", 1, relevance=0.0, effectiveness=0.1, authority=0.3),
+ _record("query", 2, relevance=0.2, effectiveness=0.2, authority=0.5),
+ ]
+
+ summary, query_rows, _, _, classified = build_reports(
+ records,
+ _args(),
+ SimpleNamespace(output_path="output/test"),
+ )
+
+ assert summary["query_aggregation"] == "rank_discounted_mean"
+ assert query_rows[0]["relevance"] == 0.07737
+ assert query_rows[0]["effectiveness"] == 0.13869
+ assert query_rows[0]["authority"] == 0.37737
+ assert query_rows[0]["relevance_aggregation"] == "rank_discounted_mean"
+ assert classified[0]["labels"] == [
+ "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW",
+ ]
+ assert classified[0]["results"][0]["abstract"] == "Abstract 1"
+ assert classified[0]["results"][0]["_evaluation"] == {
+ "rank": 1,
+ "relevance": 0.0,
+ "effectiveness": 0.1,
+ "authority": 0.3,
+ }
+ assert "_eval_query" not in classified[0]["results"][0]
+
+
+def test_query_report_has_no_overall_and_empty_query_uses_three_low_labels():
+ summary, query_rows, result_rows, _, classified = build_reports(
+ [],
+ _args(),
+ SimpleNamespace(output_path="output/test"),
+ empty_queries=["empty query"],
+ )
+
+ assert "weights" not in summary
+ assert "overall" not in summary["metrics"]
+ assert "overall" not in query_rows[0]
+ assert result_rows == []
+ assert classified[0]["labels"] == [
+ "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW",
+ "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW",
+ ]
+
+
+def test_result_level_executor_summary_is_not_exposed():
+ summary, *_ = build_reports(
+ [],
+ _args(),
+ SimpleNamespace(output_path="output/test"),
+ )
+
+ assert "result_level" not in summary
+
+
+def test_clear_executor_classification_dirs(tmp_path):
+ (tmp_path / "bad" / "result_level").mkdir(parents=True)
+ (tmp_path / "good").mkdir()
+
+ clear_executor_classification_dirs(tmp_path)
+
+ assert not (tmp_path / "bad").exists()
+ assert not (tmp_path / "good").exists()
+
+
+def test_classified_output_contains_query_records_only(tmp_path):
+ records = [
+ {
+ "query": "first",
+ "eval_status": True,
+ "labels": ["QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW"],
+ "results": [{"title": "First result"}],
+ },
+ {
+ "query": "second",
+ "eval_status": True,
+ "labels": ["QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW"],
+ "results": [{"title": "Second result"}],
+ },
+ ]
+
+ write_classified_jsonl(tmp_path, records)
+
+ output = tmp_path / "bad" / "QUALITY_BAD" / "SEARCH_RESULT_RELEVANCE_LOW.jsonl"
+ lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()]
+ assert [line["query"] for line in lines] == ["first", "second"]
+ assert all(line["results"] for line in lines)
+ assert not (tmp_path / "bad" / "query_level").exists()
+ assert not (tmp_path / "bad" / "result_level").exists()