From 398089c2462a845e8cec7b403ffc2b049ca417b2 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 6 Jul 2026 16:32:16 +0800 Subject: [PATCH 01/80] =?UTF-8?q?feat:=20DatasetSqlArgs=E6=B7=BB=E5=8A=A0e?= =?UTF-8?q?ngine=5Fargs=E5=B1=9E=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/config/input_args.py | 1 + dingo/data/datasource/sql.py | 142 +++++++++++++++++---- docs/dataset/sql.md | 61 ++++++++- test/scripts/dataset/test_sql_dataset.py | 153 +++++++++++++++++++++-- 4 files changed, 319 insertions(+), 38 deletions(-) diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py index 7ec09736..c0226257 100644 --- a/dingo/config/input_args.py +++ b/dingo/config/input_args.py @@ -25,6 +25,7 @@ class DatasetSqlArgs(BaseModel): port: str = '' database: str = '' connect_args: str = '' # 连接参数，如 ?charset=utf8mb4 + engine_args: str = '' # SQLAlchemy 引擎参数，如 pool_recycle=1800&pool_pre_ping=true class DatasetExcelArgs(BaseModel): diff --git a/dingo/data/datasource/sql.py b/dingo/data/datasource/sql.py index ef8d7d90..752654ad 100644 --- a/dingo/data/datasource/sql.py +++ b/dingo/data/datasource/sql.py @@ -1,7 +1,8 @@ from typing import Any, Dict, Generator, Optional +from urllib.parse import parse_qsl from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine +from sqlalchemy.engine import Engine, URL from dingo.config import InputArgs from dingo.data.datasource.base import DataSource @@ -9,6 +10,14 @@ @DataSource.register() class SqlDataSource(DataSource): + _ENGINE_ARG_TYPES = { + "pool_pre_ping": "bool", + "pool_recycle": "int", + "pool_size": "int", + "max_overflow": "int", + "pool_timeout": "int", + } + def __init__( self, input_args: InputArgs = None, @@ -33,37 +42,120 @@ def _get_engine(sql_config) -> Engine: "must be set when using SQL datasource." ) - # 构建数据库连接URL - # SQLite 格式: sqlite:///path/to/database.db - # 其他数据库格式: dialect+driver://username:password@host:port/database - if sql_config.dialect.lower() == "sqlite": - driver_part = f"+{sql_config.driver}" if sql_config.driver else "" - connection_url = f"{sql_config.dialect}{driver_part}:///{sql_config.database}" - else: - # 对于非 SQLite 数据库，需要用户名、密码和主机 - if not sql_config.username or not sql_config.host: + dialect = sql_config.dialect.lower() + query_args = SqlDataSource._parse_connect_args(sql_config.connect_args) + + connection_url = SqlDataSource._build_connection_url(sql_config, query_args) + + engine_kwargs: Dict[str, Any] = {"pool_pre_ping": True} + if dialect in {"mysql", "mariadb"}: + engine_kwargs["pool_recycle"] = 1800 + engine_kwargs.update(SqlDataSource._parse_engine_args(sql_config.engine_args)) + + engine = create_engine(connection_url, **engine_kwargs) + return engine + + @staticmethod + def _build_driver_name(sql_config) -> str: + return ( + f"{sql_config.dialect}+{sql_config.driver}" + if sql_config.driver + else sql_config.dialect + ) + + @staticmethod + def _parse_connect_args(connect_args: str) -> Dict[str, str]: + return SqlDataSource._parse_query_arg_string(connect_args) + + @staticmethod + def _parse_query_arg_string(raw_arg_string: str) -> Dict[str, str]: + if not raw_arg_string: + return {} + normalized = raw_arg_string.strip() + if normalized.startswith("?"): + normalized = normalized[1:] + if not normalized: + return {} + return { + key: value + for key, value in parse_qsl(normalized, keep_blank_values=False) + if key + } + + @staticmethod + def _parse_bool_value(raw_value: str, key: str) -> bool: + normalized = raw_value.strip().lower() + if normalized == "true": + return True + if normalized == "false": + return False + raise RuntimeError( + f"SQL engine arg '{key}' expects 'true' or 'false', got: {raw_value}." + ) + + @staticmethod + def _parse_engine_args(engine_args: str) -> Dict[str, Any]: + raw_engine_args = SqlDataSource._parse_query_arg_string(engine_args) + if not raw_engine_args: + return {} + + parsed_engine_args: Dict[str, Any] = {} + for key, raw_value in raw_engine_args.items(): + expected_type = SqlDataSource._ENGINE_ARG_TYPES.get(key) + if expected_type is None: + allowed = ", ".join(sorted(SqlDataSource._ENGINE_ARG_TYPES.keys())) raise RuntimeError( - f"For {sql_config.dialect}, username and host must be set." + f"Unsupported SQL engine arg '{key}'. Allowed keys: {allowed}." ) - driver_part = f"+{sql_config.driver}" if sql_config.driver else "" - port_part = f":{sql_config.port}" if sql_config.port else "" - password_part = f":{sql_config.password}" if sql_config.password else "" + if expected_type == "int": + try: + parsed_engine_args[key] = int(raw_value) + except ValueError as exc: + raise RuntimeError( + f"SQL engine arg '{key}' expects an integer value, got: {raw_value}." + ) from exc + elif expected_type == "bool": + parsed_engine_args[key] = SqlDataSource._parse_bool_value(raw_value, key) + + return parsed_engine_args + + @staticmethod + def _parse_port(port: str) -> Optional[int]: + if not port: + return None + try: + return int(port) + except ValueError as exc: + raise RuntimeError("SQL connection parameter 'port' must be an integer.") from exc - connection_url = ( - f"{sql_config.dialect}{driver_part}://" - f"{sql_config.username}{password_part}@" - f"{sql_config.host}{port_part}/{sql_config.database}" + @staticmethod + def _build_connection_url(sql_config, query_args: Dict[str, str]) -> URL: + driver_name = SqlDataSource._build_driver_name(sql_config) + query = query_args or None + + if sql_config.dialect.lower() == "sqlite": + return URL.create( + drivername=driver_name, + database=sql_config.database, + query=query, ) - # 添加连接参数（如 ?charset=utf8mb4） - if sql_config.connect_args: - # 确保参数以 ? 开头 - args_part = sql_config.connect_args if sql_config.connect_args.startswith('?') else f"?{sql_config.connect_args}" - connection_url = f"{connection_url}{args_part}" + # 对于非 SQLite 数据库，需要用户名、密码和主机 + if not sql_config.username or not sql_config.host: + raise RuntimeError( + f"For {sql_config.dialect}, username and host must be set." + ) - engine = create_engine(connection_url) - return engine + return URL.create( + drivername=driver_name, + username=sql_config.username, + password=sql_config.password or None, + host=sql_config.host, + port=SqlDataSource._parse_port(sql_config.port), + database=sql_config.database, + query=query, + ) @staticmethod def get_source_type() -> str: diff --git a/docs/dataset/sql.md b/docs/dataset/sql.md index e03c8e8a..09a81193 100644 --- a/docs/dataset/sql.md +++ b/docs/dataset/sql.md @@ -10,6 +10,8 @@ - ✅ **多数据库支持**: 支持 PostgreSQL, MySQL, SQLite 等主流数据库 - ✅ **内存友好**: 逐行处理数据，适合处理大规模数据集 - ✅ **灵活查询**: 支持任意 SQL 查询语句（SELECT、JOIN、WHERE 等） +- ✅ **连接更稳健**: 默认启用 `pool_pre_ping=True`，MySQL/MariaDB 默认 `pool_recycle=1800` +- ✅ **URL 更安全**: 使用 SQLAlchemy `URL.create(...)` 构建连接，避免特殊字符导致的 URL 解析问题 ## 依赖安装 @@ -168,6 +170,7 @@ for data in dataset.get_data(): | `port` | str | 否 | 数据库端口 | | `database` | str | 是 | 数据库名称或文件路径（SQLite） | | `connect_args` | str | 否 | 连接参数，如 `?charset=utf8mb4`、`?sslmode=require` 等 | +| `engine_args` | str | 否 | SQLAlchemy 引擎参数，格式同 `connect_args`，如 `pool_recycle=3600&pool_pre_ping=true` | *注：对于 SQLite，`username` 和 `host` 不是必填项；对于其他数据库，这些是必填项。 @@ -260,9 +263,53 @@ sql_config = DatasetSqlArgs( ) ``` +### 4. 使用引擎参数（连接池/探活） + +对于需要控制 SQLAlchemy 引擎行为的场景，可以使用 `engine_args`（字符串格式，支持 `k=v&k2=v2`）： + +```python +# 覆盖默认连接池回收时间，显式开启连接探活 +sql_config = DatasetSqlArgs( + dialect="mysql", + driver="pymysql", + username="root", + password="password", + host="localhost", + port="3306", + database="test_db", + connect_args="charset=utf8mb4", + engine_args="pool_recycle=3600&pool_pre_ping=true" +) + +# 支持 ? 前缀写法 +sql_config = DatasetSqlArgs( + dialect="postgresql", + driver="psycopg2", + username="myuser", + password="mypassword", + host="localhost", + port="5432", + database="mydb", + engine_args="?pool_size=8&max_overflow=16&pool_timeout=30" +) +``` + +当前 `engine_args` 支持参数： + +- `pool_pre_ping`（bool，默认 `true`，仅支持 `true/false`） +- `pool_recycle`（int，MySQL/MariaDB 默认 `1800`） +- `pool_size`（int） +- `max_overflow`（int） +- `pool_timeout`（int） + +> ⚠️ Warning: 若在 `engine_args` 里显式设置同名参数，会覆盖默认值。 + ## 工作原理 1. **连接创建**: `SqlDataSource` 使用 SQLAlchemy 创建数据库引擎 + - 使用 `URL.create(...)` 安全构建 URL（兼容密码中的 `@/#/:` 等特殊字符） + - 默认开启 `pool_pre_ping=True`；MySQL/MariaDB 默认 `pool_recycle=1800` + - 可通过 `engine_args` 覆盖默认引擎参数 2. **流式查询**: 使用 `connection.execution_options(stream_results=True)` 启用服务器端游标 3. **逐行迭代**: SQLAlchemy 自动处理数据分页，逐行返回结果 4. **数据转换**: 每行数据通过 `jsonl` 转换器转换为 `Data` 对象 @@ -326,8 +373,20 @@ pip install pymysql # MySQL - 检查数据库服务是否运行 - 检查网络连接和防火墙设置 - 验证主机地址和端口号 +- 在 `connect_args` 中显式增加数据库驱动超时参数（如 `read_timeout`、`write_timeout`） + +### 问题4: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query') + +**现象**: 查询执行中连接被重置（例如 `Connection reset by peer`）。 + +**解决**: +- 检查数据库和网络链路（代理/LB/NAT）是否会中断长连接 +- 将大查询拆分为分页或分片查询，减少单次长事务时长 +- 检查服务端超时参数（如 `wait_timeout`、`net_read_timeout`） +- 确认已使用默认连接稳健配置（`pool_pre_ping=True`、MySQL 默认 `pool_recycle=1800`） +- 按需在 `engine_args` 中调整连接池策略（如 `pool_recycle`、`pool_timeout`） -### 问题4: TypeError: Data() argument after ** must be a mapping +### 问题5: TypeError: Data() argument after ** must be a mapping **解决**: 确保使用 `format="jsonl"` 而不是 `format="json"` diff --git a/test/scripts/dataset/test_sql_dataset.py b/test/scripts/dataset/test_sql_dataset.py index 8254ffb7..c44bfad2 100644 --- a/test/scripts/dataset/test_sql_dataset.py +++ b/test/scripts/dataset/test_sql_dataset.py @@ -7,6 +7,9 @@ import os import sqlite3 import tempfile +import uuid + +import pytest from dingo.config import DatasetArgs, DatasetSqlArgs, InputArgs from dingo.data.dataset.sql import SqlDataset @@ -16,7 +19,7 @@ def create_test_database(): """创建一个测试 SQLite 数据库""" # 创建临时数据库文件 - db_path = os.path.join(tempfile.gettempdir(), "test_dingo_sql.db") + db_path = os.path.join(tempfile.gettempdir(), f"test_dingo_sql_{uuid.uuid4().hex}.db") # 连接数据库并创建测试表 conn = sqlite3.connect(db_path) @@ -63,6 +66,7 @@ def test_sql_dataset(): db_path = create_test_database() print(f"✓ 创建测试数据库: {db_path}") + datasource = None try: # 配置 SQL 连接参数（SQLite） sql_config = DatasetSqlArgs( @@ -132,10 +136,15 @@ def test_sql_dataset(): print("=" * 60) finally: + if datasource is not None: + datasource.engine.dispose() # 清理测试数据库 if os.path.exists(db_path): - os.remove(db_path) - print(f"\n✓ 清理测试数据库: {db_path}") + try: + os.remove(db_path) + print(f"\n✓ 清理测试数据库: {db_path}") + except PermissionError: + print(f"\n! 跳过清理（文件占用）: {db_path}") def test_stream_results(): @@ -145,7 +154,10 @@ def test_stream_results(): print("=" * 60) # 创建一个包含更多数据的测试数据库 - db_path = os.path.join(tempfile.gettempdir(), "test_dingo_sql_stream.db") + db_path = os.path.join( + tempfile.gettempdir(), + f"test_dingo_sql_stream_{uuid.uuid4().hex}.db" + ) conn = sqlite3.connect(db_path) cursor = conn.cursor() @@ -164,6 +176,7 @@ def test_stream_results(): print(f"✓ 创建包含 1000 条数据的测试数据库") + datasource = None try: sql_config = DatasetSqlArgs( dialect="sqlite", @@ -195,19 +208,135 @@ def test_stream_results(): # 只读取前 10 条，验证流式读取（不会加载全部 1000 条到内存） print("开始流式读取（只读取前 10 条）:") count = 0 - for idx, data in enumerate(dataset.get_data()): - if idx < 10: - print(f" [{idx + 1}] {data}") - count += 1 - if idx >= 9: # 只读取前 10 条就停止 - break + data_iterator = iter(dataset.get_data()) + try: + for idx, data in enumerate(data_iterator): + if idx < 10: + print(f" [{idx + 1}] {data}") + count += 1 + if idx >= 9: # 只读取前 10 条就停止 + break + finally: + close_method = getattr(data_iterator, "close", None) + if callable(close_method): + close_method() print(f"\n✓ 流式读取验证通过（处理了 {count} 条数据后停止）") finally: + if datasource is not None: + datasource.engine.dispose() if os.path.exists(db_path): - os.remove(db_path) - print(f"✓ 清理测试数据库: {db_path}") + try: + os.remove(db_path) + print(f"✓ 清理测试数据库: {db_path}") + except PermissionError: + print(f"! 跳过清理（文件占用）: {db_path}") + + +def test_parse_connect_args_supports_prefix_and_multiple_pairs(): + query_args = SqlDataSource._parse_connect_args( + "?charset=utf8mb4&read_timeout=120&write_timeout=120" + ) + assert query_args["charset"] == "utf8mb4" + assert query_args["read_timeout"] == "120" + assert query_args["write_timeout"] == "120" + + +def test_mysql_engine_has_stability_pool_settings(): + sql_config = DatasetSqlArgs( + dialect="mysql", + driver="pymysql", + username="user", + password="pass", + host="localhost", + port="3306", + database="db", + connect_args="charset=utf8mb4" + ) + engine = SqlDataSource._get_engine(sql_config) + try: + assert engine.pool._pre_ping is True + assert engine.pool._recycle == 1800 + finally: + engine.dispose() + + +def test_mysql_does_not_inject_default_timeout_query_args(): + sql_config = DatasetSqlArgs( + dialect="mysql", + driver="pymysql", + username="user", + password="pass", + host="localhost", + port="3306", + database="db", + connect_args="charset=utf8mb4" + ) + query_args = SqlDataSource._parse_connect_args(sql_config.connect_args) + url = SqlDataSource._build_connection_url(sql_config, query_args) + assert url.query == {"charset": "utf8mb4"} + + +def test_parse_engine_args_with_supported_types(): + engine_args = SqlDataSource._parse_engine_args( + "pool_recycle=3600&pool_pre_ping=true&pool_size=8&max_overflow=16&pool_timeout=30" + ) + assert engine_args == { + "pool_recycle": 3600, + "pool_pre_ping": True, + "pool_size": 8, + "max_overflow": 16, + "pool_timeout": 30, + } + + +def test_engine_args_override_default_pool_recycle(): + sql_config = DatasetSqlArgs( + dialect="mysql", + driver="pymysql", + username="user", + password="pass", + host="localhost", + port="3306", + database="db", + engine_args="pool_recycle=7200&pool_pre_ping=false" + ) + engine = SqlDataSource._get_engine(sql_config) + try: + assert engine.pool._recycle == 7200 + assert engine.pool._pre_ping is False + finally: + engine.dispose() + + +def test_engine_args_rejects_unsupported_key(): + with pytest.raises(RuntimeError, match="Unsupported SQL engine arg"): + SqlDataSource._parse_engine_args("unsupported_key=1") + + +def test_engine_args_rejects_invalid_bool_value(): + with pytest.raises(RuntimeError, match="true' or 'false"): + SqlDataSource._parse_engine_args("pool_pre_ping=not_bool") + + +def test_engine_args_rejects_numeric_bool_value(): + with pytest.raises(RuntimeError, match="true' or 'false"): + SqlDataSource._parse_engine_args("pool_pre_ping=1") + + +def test_invalid_port_raises_runtime_error(): + sql_config = DatasetSqlArgs( + dialect="mysql", + driver="pymysql", + username="user", + password="pass", + host="localhost", + port="not_a_number", + database="db" + ) + with pytest.raises(RuntimeError, match="port"): + SqlDataSource._build_connection_url(sql_config, {}) if __name__ == "__main__": From f7044589c48be2101b73c0ff0273f051672e20c1 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 6 Jul 2026 08:33:52 +0000 Subject: [PATCH 02/80] =?UTF-8?q?=F0=9F=8E=A8=20Auto-format=20code=20with?= =?UTF-8?q?=20pre-commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/data/datasource/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dingo/data/datasource/sql.py b/dingo/data/datasource/sql.py index 752654ad..7b4c4df2 100644 --- a/dingo/data/datasource/sql.py +++ b/dingo/data/datasource/sql.py @@ -2,7 +2,7 @@ from urllib.parse import parse_qsl from sqlalchemy import create_engine, text -from sqlalchemy.engine import Engine, URL +from sqlalchemy.engine import URL, Engine from dingo.config import InputArgs from dingo.data.datasource.base import DataSource From d94c9cd7cdf871b9fc33894cfc044d2cd7784b77 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 10 Jul 2026 19:22:48 +0800 Subject: [PATCH 03/80] =?UTF-8?q?feat:=20score=E5=AE=9E=E6=97=B6=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/exec/local.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index ff48aaa5..892a3ef2 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -139,6 +139,14 @@ def execute(self) -> SummaryModel: else: self.summary.num_good += 1 self.summary.total += 1 + # Keep score updated during execution so get_summary() + # remains meaningful even if the task stops early. + if self.summary.total > 0: + self.summary.score = round( + self.summary.num_good / self.summary.total * 100, 2 + ) + else: + self.summary.score = 0.0 self.write_single_data( self.summary.output_path, self.input_args, result_info From a2933d0819fca470df7b4e44d2188469adc0a9c0 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 10 Jul 2026 19:26:46 +0800 Subject: [PATCH 04/80] feat: ci update --- dingo/exec/local.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 892a3ef2..95fd5cac 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -141,12 +141,9 @@ def execute(self) -> SummaryModel: self.summary.total += 1 # Keep score updated during execution so get_summary() # remains meaningful even if the task stops early. - if self.summary.total > 0: - self.summary.score = round( - self.summary.num_good / self.summary.total * 100, 2 - ) - else: - self.summary.score = 0.0 + self.summary.score = round( + self.summary.num_good / self.summary.total * 100, 2 + ) self.write_single_data( self.summary.output_path, self.input_args, result_info From 0ba34f056823ccda99f615c53fdad94b8b66b889 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 13 Jul 2026 14:41:44 +0800 Subject: [PATCH 05/80] =?UTF-8?q?feat:=20SummaryModel=E5=A2=9E=E5=8A=A0typ?= =?UTF-8?q?e=5Fcount?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/exec/local.py | 29 ++++++++++++++++++----------- dingo/io/output/summary_model.py | 4 +++- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 95fd5cac..d3ca433c 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -115,8 +115,8 @@ def execute(self) -> SummaryModel: # 统计eval_details，第一层key是字段名组合，第二层value是List[EvalDetail] # 错误类型从EvalDetail.label中获取 for field_key, eval_detail_list in result_info.eval_details.items(): - if field_key not in self.summary.type_ratio: - self.summary.type_ratio[field_key] = {} + if field_key not in self.summary.type_count: + self.summary.type_count[field_key] = {} # 遍历 List[EvalDetail]，同时收集指标分数和标签 label_set = set() @@ -131,8 +131,8 @@ def execute(self) -> SummaryModel: label_set.add(label) for label in label_set: - self.summary.type_ratio[field_key].setdefault(label, 0) - self.summary.type_ratio[field_key][label] += 1 + self.summary.type_count[field_key].setdefault(label, 0) + self.summary.type_count[field_key][label] += 1 if result_info.eval_status: self.summary.num_bad += 1 @@ -144,6 +144,7 @@ def execute(self) -> SummaryModel: self.summary.score = round( self.summary.num_good / self.summary.total * 100, 2 ) + self._refresh_type_ratio(self.summary) self.write_single_data( self.summary.output_path, self.input_args, result_info @@ -252,13 +253,7 @@ def summarize(self, summary: SummaryModel) -> SummaryModel: if new_summary.total == 0: return new_summary new_summary.score = round(new_summary.num_good / new_summary.total * 100, 2) - - # type_ratio是两层结构：第一层是字段名，第二层是具体错误类型 - for field_name in new_summary.type_ratio: - for eval_details in new_summary.type_ratio[field_name]: - new_summary.type_ratio[field_name][eval_details] = round( - new_summary.type_ratio[field_name][eval_details] / new_summary.total, 6 - ) + self._refresh_type_ratio(new_summary) # 计算指标分数的平均值、最小值、最大值、标准差等 new_summary.calculate_metrics_score_averages() @@ -266,6 +261,18 @@ def summarize(self, summary: SummaryModel) -> SummaryModel: new_summary.finish_time = time.strftime("%Y%m%d_%H%M%S", time.localtime()) return new_summary + @staticmethod + def _refresh_type_ratio(summary: SummaryModel): + if summary.total <= 0: + summary.type_ratio = {} + return + + summary.type_ratio = {} + for field_name, label_counts in summary.type_count.items(): + summary.type_ratio[field_name] = {} + for label, count in label_counts.items(): + summary.type_ratio[field_name][label] = round(count / summary.total, 6) + @staticmethod def _json_default(value): if isinstance(value, Decimal): diff --git a/dingo/io/output/summary_model.py b/dingo/io/output/summary_model.py index 3d231df7..f2de1df2 100644 --- a/dingo/io/output/summary_model.py +++ b/dingo/io/output/summary_model.py @@ -16,7 +16,8 @@ class SummaryModel(BaseModel): num_good: int = 0 num_bad: int = 0 total: int = 0 - type_ratio: Dict[str, Dict[str, int]] = {} + type_count: Dict[str, Dict[str, int]] = Field(default_factory=dict) + type_ratio: Dict[str, Dict[str, float]] = Field(default_factory=dict) # 新增：指标分数统计（用于RAG等评估场景） # 结构：{field_key: {metric_name: {scores, score_average, ...}}} @@ -117,6 +118,7 @@ def to_dict(self): 'num_good': self.num_good, 'num_bad': self.num_bad, 'total': self.total, + 'type_count': self.type_count, 'type_ratio': self.type_ratio, } From e9023954db45934f0ef140df192ad460c18133a6 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 13 Jul 2026 16:24:07 +0800 Subject: [PATCH 06/80] =?UTF-8?q?feat:=20refresh=5Ftype=5Fratio=E4=BC=A0?= =?UTF-8?q?=E5=8F=82=E9=80=BB=E8=BE=91=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/exec/local.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index d3ca433c..6db0a65f 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -144,7 +144,6 @@ def execute(self) -> SummaryModel: self.summary.score = round( self.summary.num_good / self.summary.total * 100, 2 ) - self._refresh_type_ratio(self.summary) self.write_single_data( self.summary.output_path, self.input_args, result_info @@ -253,7 +252,7 @@ def summarize(self, summary: SummaryModel) -> SummaryModel: if new_summary.total == 0: return new_summary new_summary.score = round(new_summary.num_good / new_summary.total * 100, 2) - self._refresh_type_ratio(new_summary) + new_summary = self.refresh_type_ratio(new_summary) # 计算指标分数的平均值、最小值、最大值、标准差等 new_summary.calculate_metrics_score_averages() @@ -261,17 +260,19 @@ def summarize(self, summary: SummaryModel) -> SummaryModel: new_summary.finish_time = time.strftime("%Y%m%d_%H%M%S", time.localtime()) return new_summary - @staticmethod - def _refresh_type_ratio(summary: SummaryModel): + def refresh_type_ratio(self, summary: SummaryModel) -> SummaryModel: if summary.total <= 0: summary.type_ratio = {} - return + return summary summary.type_ratio = {} for field_name, label_counts in summary.type_count.items(): summary.type_ratio[field_name] = {} for label, count in label_counts.items(): - summary.type_ratio[field_name][label] = round(count / summary.total, 6) + summary.type_ratio[field_name][label] = round( + count / summary.total, 6 + ) + return summary @staticmethod def _json_default(value): @@ -392,6 +393,7 @@ def write_summary(self, path: str, input_args: InputArgs, summary: SummaryModel) ) def get_summary(self): + self.summary = self.refresh_type_ratio(self.summary) return self.summary def get_info_list(self, high_quality: bool) -> list: From 450c3d7f3e4ae562e96a3e09348b699b60c1cd0b Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 13 Jul 2026 17:41:22 +0800 Subject: [PATCH 07/80] =?UTF-8?q?feat:=20summarize=E5=A2=9E=E5=8A=A0refres?= =?UTF-8?q?h=5Ftype=5Fratio=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/exec/local.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 6db0a65f..18f40328 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -152,7 +152,7 @@ def execute(self) -> SummaryModel: self.write_summary( self.summary.output_path, self.input_args, - self.summarize(self.summary), + self.summarize(self.summary, refresh_type_ratio=False), ) log.debug("[Summary]: " + str(self.summary)) @@ -247,12 +247,15 @@ def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInf return existing_list - def summarize(self, summary: SummaryModel) -> SummaryModel: + def summarize( + self, summary: SummaryModel, refresh_type_ratio: bool = True + ) -> SummaryModel: new_summary = copy.deepcopy(summary) if new_summary.total == 0: return new_summary new_summary.score = round(new_summary.num_good / new_summary.total * 100, 2) - new_summary = self.refresh_type_ratio(new_summary) + if refresh_type_ratio: + new_summary = self.refresh_type_ratio(new_summary) # 计算指标分数的平均值、最小值、最大值、标准差等 new_summary.calculate_metrics_score_averages() From f3a10236ed80e38997932e85bdd53b53aa45d8e6 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Tue, 14 Jul 2026 11:32:14 +0800 Subject: [PATCH 08/80] feat(retrieval): add search result quality evaluation - add relevance, effectiveness, and authority evaluators - add standalone and combined executor-based evaluation scripts - support query-level and result-level classified outputs - improve LLM response parsing and content issue detection - set overall weights to 0.7/0.2/0.1 - add evaluator tests and usage documentation --- .../model/llm/llm_search_result_authority.py | 202 +++++ .../llm/llm_search_result_effectiveness.py | 729 ++++++++++++++++++ .../model/llm/llm_search_result_relevance.py | 345 ++++++++- docs/search_result_authority_executor.md | 162 ++++ docs/search_result_effectiveness_executor.md | 131 ++++ docs/search_result_quality_metrics.md | 641 +++++++++++++++ docs/search_result_relevance_executor.md | 32 + examples/retrieval/sdk_eval_authority.py | 105 +++ examples/retrieval/sdk_eval_effectiveness.py | 127 +++ examples/retrieval/sdk_eval_relevancy.py | 126 +++ examples/retrieval/sdk_eval_search_result.py | 565 ++++++++++++++ .../retrieval/search_result_eval_utils.py | 118 +++ .../test_llm_search_result_effectiveness.py | 51 ++ .../llm/test_llm_search_result_relevance.py | 47 ++ test/scripts/retrieval/test_open_eval.py | 20 + 15 files changed, 3372 insertions(+), 29 deletions(-) create mode 100644 dingo/model/llm/llm_search_result_authority.py create mode 100644 dingo/model/llm/llm_search_result_effectiveness.py create mode 100644 docs/search_result_authority_executor.md create mode 100644 docs/search_result_effectiveness_executor.md create mode 100644 docs/search_result_quality_metrics.md create mode 100644 docs/search_result_relevance_executor.md create mode 100644 examples/retrieval/sdk_eval_authority.py create mode 100644 examples/retrieval/sdk_eval_effectiveness.py create mode 100644 examples/retrieval/sdk_eval_relevancy.py create mode 100644 examples/retrieval/sdk_eval_search_result.py create mode 100644 examples/retrieval/search_result_eval_utils.py create mode 100644 test/scripts/model/llm/test_llm_search_result_effectiveness.py create mode 100644 test/scripts/model/llm/test_llm_search_result_relevance.py diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py new file mode 100644 index 00000000..cfd42779 --- /dev/null +++ b/dingo/model/llm/llm_search_result_authority.py @@ -0,0 +1,202 @@ +"""Rule-based search result authority grader.""" + +from __future__ import annotations + +import json +import math +import statistics +from dataclasses import dataclass +from typing import Any + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + + +HIGH_AUTHORITY_VENUE_HINTS = ( + "nature", + "science", + "cell", + "nejm", + "lancet", + "jama", + "acm", + "ieee", + "springer", + "elsevier", + "wiley", + "neurips", + "icml", + "iclr", + "cvpr", + "acl", + "emnlp", + "aaai", + "ijcai", + "sigir", +) + + +def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: + return max(low, min(high, value)) + + +def _normalize_text(text: Any) -> str: + return " ".join(str(text or "").lower().split()) + + +def extract_venue(result: dict[str, Any]) -> str: + return str( + result.get("publication_venue_name_unified") + or result.get("publication_venue_name") + or result.get("venue") + or result.get("source") + or "" + ) + + +def extract_citations(result: dict[str, Any], key: str = "citation_count") -> float: + try: + return float(result.get(key) or 0) + except (TypeError, ValueError): + return 0.0 + + +@dataclass +class AuthorityGrade: + """Structured authority score for one search result.""" + + score: float = 0.0 + citation_score: float = 0.0 + influential_citation_score: float = 0.0 + venue_score: float = 0.0 + doi_score: float = 0.0 + reason: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "score": round(self.score, 5), + "citation_score": round(self.citation_score, 5), + "influential_citation_score": round(self.influential_citation_score, 5), + "venue_score": round(self.venue_score, 5), + "doi_score": round(self.doi_score, 5), + "reason": self.reason, + } + + +@dataclass +class AuthoritySummary: + mean_score: float = 0.0 + median_score: float = 0.0 + mean_citation_score: float = 0.0 + mean_influential_citation_score: float = 0.0 + mean_venue_score: float = 0.0 + mean_doi_score: float = 0.0 + graded_pairs: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "authority_mean_score": round(self.mean_score, 5), + "authority_median_score": round(self.median_score, 5), + "authority_mean_citation_score": round(self.mean_citation_score, 5), + "authority_mean_influential_citation_score": round(self.mean_influential_citation_score, 5), + "authority_mean_venue_score": round(self.mean_venue_score, 5), + "authority_mean_doi_score": round(self.mean_doi_score, 5), + "authority_graded_pairs": self.graded_pairs, + } + + +@Model.llm_register("LLMSearchResultAuthority") +class LLMSearchResultAuthority: + """Authority scorer based on citation impact, venue, and DOI metadata.""" + + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + + def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: + venue = _normalize_text(extract_venue(result)) + venue_type = _normalize_text(result.get("publication_venue_type") or "") + citations = extract_citations(result, "citation_count") + influential = extract_citations(result, "influential_citation_count") + + citation_score = _clamp(math.log1p(citations) / math.log1p(500.0)) + influential_score = _clamp(math.log1p(influential) / math.log1p(50.0)) + + venue_score = 0.25 + reason = "unknown_or_low_signal_venue" + if any(hint in venue for hint in HIGH_AUTHORITY_VENUE_HINTS): + venue_score = 0.85 + reason = "high_authority_venue_hint" + elif "journal" in venue_type or "conference" in venue_type: + venue_score = 0.65 + reason = "journal_or_conference" + elif "repository" in venue_type or "preprint" in venue: + venue_score = 0.45 + reason = "repository_or_preprint" + + doi_score = 1.0 if result.get("doi") or "doi.org" in json.dumps(result.get("locations", [])) else 0.0 + score = ( + 0.45 * citation_score + + 0.20 * influential_score + + 0.25 * venue_score + + 0.10 * doi_score + ) + return AuthorityGrade( + score=_clamp(score), + citation_score=citation_score, + influential_citation_score=influential_score, + venue_score=venue_score, + doi_score=doi_score, + reason=reason, + ) + + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened search result row.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + + grade = cls().grade(result=result) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + + labels: list[str] = [] + if grade.score < threshold: + labels.append("Authority.Error_Authority_Low") + if grade.citation_score <= 0.0: + labels.append("Authority.Error_Citation_Miss") + if grade.venue_score <= 0.25: + labels.append("Authority.Error_Venue_Low_Signal") + if grade.doi_score <= 0.0: + labels.append("Authority.Error_DOI_Miss") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[grade.to_dict()], + ) + + +def aggregate_grades(grades: list[AuthorityGrade]) -> AuthoritySummary: + if not grades: + return AuthoritySummary() + return AuthoritySummary( + mean_score=statistics.mean(g.score for g in grades), + median_score=statistics.median(g.score for g in grades), + mean_citation_score=statistics.mean(g.citation_score for g in grades), + mean_influential_citation_score=statistics.mean(g.influential_citation_score for g in grades), + mean_venue_score=statistics.mean(g.venue_score for g in grades), + mean_doi_score=statistics.mean(g.doi_score for g in grades), + graded_pairs=len(grades), + ) diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py new file mode 100644 index 00000000..310f6eb3 --- /dev/null +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -0,0 +1,729 @@ +"""Search result effectiveness grader. + +This grader scores whether a returned search result has enough usable +bibliographic content for a user to judge and consume it. Missing-field and +basic information-density checks are deterministic. Readability and corruption +checks can be delegated to an LLM judge to avoid over-penalizing normal academic +formulas, units, and symbols. + +It intentionally does not judge topical relevance; use +``LLMSearchResultRelevance`` for that. +""" + +from __future__ import annotations + +import json +import logging +import re +import statistics +from dataclasses import dataclass +from typing import Any + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + + +logger = logging.getLogger(__name__) + + +RULE_SPECIAL_CHARACTER_PATTERNS = ( + r"u200e", + r"÷|\? :", + r"[锟解枴閿熻В鏋碷�]|\{\/U\}", + r"U\+26[0-F][0-D]|U\+273[3-4]|U\+1F[3-6][0-4][0-F]|U\+1F6[8-F][0-F]", + r"<\|.*?\|>", + r"<[^>]+>", +) +RULE_INVISIBLE_CHAR_PATTERN = r"[\u0080-\u009F\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" +RULE_ABNORMAL_CHAR_THRESHOLD = 0.01 +HTML_TAG_PATTERN = r"<[^>]+>" +MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}" +UTF8_LATIN1_SEQUENCE_PATTERN = re.compile(r"[\u00C2\u00C3\u00D0\u00D1][\u0080-\u00BF]") +C1_CONTROL_PATTERN = re.compile(r"[\u0080-\u009F]") + + +LLM_FIELD_QUALITY_SYSTEM_PROMPT = """You are a strict but practical data quality evaluator for academic search result metadata. + +Judge whether each supplied metadata field is readable and clean enough to show to users. +Focus on real text-quality problems: +- missing or empty field +- invisible/control characters +- mojibake or garbled encoding, such as replacement characters, unreadable CJK mojibake, + or UTF-8 text decoded as Latin-1 with repeated sequences like Ð... or Ñ... +- raw HTML/XML markup leaked into visible text, such as ... +- suspicious special-character noise that materially hurts readability + +Do NOT penalize normal academic content: +- mathematical formulas, LaTeX, chemical symbols, units, Greek letters +- punctuation, pipes used as separators, parentheses, slashes, hyphens +- mixed Chinese/English titles, journal names, abbreviations, DOI-like text + +Return compact JSON only. Do not use markdown. Keep each reason within 12 words and do not use double quotes inside reasons. +Schema: +{ + "fields": { + "title": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "abstract": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "keywords": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "venue": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."} + }, + "overall_issues": ["..."], + "reason": "short overall reason" +} + +Use issue names from: +- missing_field +- invisible_char +- mojibake +- html_tag +- unreadable_text +- special_char_noise +- none + +Scoring guidance: +- 1.0: clean, readable field; normal formulas and units are allowed. +- 0.7: mostly readable with minor display artifacts. +- 0.4: readable but contains visible markup or notable noise requiring cleanup. +- 0.1: unreadable garbled text, heavy mojibake, or severe invisible/control-character corruption. +- 0.0: missing/empty field. +""" + + +def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: + return max(low, min(high, value)) + + +def _token_count(text: str) -> int: + return len(re.findall(r"[\w\u4e00-\u9fff]+", text or "")) + + +def _field_quality(text: Any, *, min_chars: int, good_chars: int) -> float: + value = str(text or "").strip() + if not value: + return 0.0 + if len(value) < min_chars: + return 0.25 + if len(value) >= good_chars: + return 1.0 + return _clamp(0.35 + 0.65 * (len(value) - min_chars) / max(1, good_chars - min_chars)) + + +def _suspicious_latin1_mojibake_count(text: str) -> int: + return len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(text)) + len(C1_CONTROL_PATTERN.findall(text)) + + +def _looks_like_utf8_latin1_mojibake(text: str) -> bool: + """Detect UTF-8 text accidentally decoded as Latin-1. + + Literal characters such as ``Ð`` and ``Ñ`` can be valid text, so they are + not sufficient evidence by themselves. A value is treated as suspicious + when it contains repeated UTF-8/Latin-1 byte-shaped sequences or C1 control + characters and a reversible Latin-1-to-UTF-8 repair removes that evidence. + """ + value = str(text or "") + if not value: + return False + + suspicious_before = _suspicious_latin1_mojibake_count(value) + if suspicious_before == 0: + return False + + c1_count = len(C1_CONTROL_PATTERN.findall(value)) + repeated_sequences = len(UTF8_LATIN1_SEQUENCE_PATTERN.findall(value)) >= 2 + if not repeated_sequences and c1_count / max(1, len(value)) < RULE_ABNORMAL_CHAR_THRESHOLD: + return False + + try: + repaired = value.encode("latin-1").decode("utf-8") + except (UnicodeEncodeError, UnicodeDecodeError): + # Mixed-language metadata may contain legitimate non-Latin-1 text next + # to a corrupted fragment. Repeated byte-shaped pairs plus C1 controls + # are enough to send that field to the LLM judge for confirmation. + return repeated_sequences and c1_count > 0 + + suspicious_after = _suspicious_latin1_mojibake_count(repaired) + return repaired != value and suspicious_after < suspicious_before + + +def _has_mojibake_evidence(text: str) -> bool: + value = str(text or "") + return bool(re.search(MOJIBAKE_EVIDENCE_PATTERN, value)) or _looks_like_utf8_latin1_mojibake(value) + + +def _rule_abnormal_char_issues(text: str) -> list[str]: + value = str(text or "") + if not value: + return [] + + issues: list[str] = [] + special_matches: list[str] = [] + for pattern in RULE_SPECIAL_CHARACTER_PATTERNS: + special_matches.extend(re.findall(pattern, value)) + if len(special_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + issues.append("RuleSpecialCharacter") + + has_latin1_mojibake = _looks_like_utf8_latin1_mojibake(value) + if has_latin1_mojibake: + issues.append("RuleMojibake") + + invisible_matches = re.findall(RULE_INVISIBLE_CHAR_PATTERN, value) + if not has_latin1_mojibake and len(invisible_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + issues.append("RuleInvisibleChar") + return issues + + +def _has_confirmed_llm_issue(issues: list[str] | None) -> bool: + if not issues: + return False + ignored = {"none", "missing_field"} + return any(str(issue).split(":")[-1].strip().lower() not in ignored for issue in issues) + + +def _filter_llm_field_issues(field: str, value: str, issues: list[str]) -> list[str]: + """Keep only LLM issues supported by field-level evidence.""" + text = str(value or "") + filtered: list[str] = [] + for issue in issues: + issue_type = str(issue).split(":")[-1].strip().lower() + keep = False + if issue_type == "html_tag": + keep = bool(re.search(HTML_TAG_PATTERN, text)) + elif issue_type == "invisible_char": + keep = bool(re.search(RULE_INVISIBLE_CHAR_PATTERN, text)) + elif issue_type in {"mojibake", "unreadable_text"}: + keep = _has_mojibake_evidence(text) + elif issue_type == "special_char_noise": + keep = bool(_rule_abnormal_char_issues(text)) + else: + keep = True + + if keep and issue not in filtered: + filtered.append(issue) + return filtered + + +def _strip_json_fence(text: str) -> str: + value = (text or "").strip() + if value.startswith("```json"): + value = value[7:].strip() + elif value.startswith("```"): + value = value[3:].strip() + if value.endswith("```"): + value = value[:-3].strip() + return value + + +def _extract_json_object(text: str) -> str: + value = _strip_json_fence(text) + start = value.find("{") + end = value.rfind("}") + if start >= 0 and end > start: + return value[start : end + 1] + return value + + +def _safe_float(value: Any, default: float = 1.0) -> float: + try: + return _clamp(float(value)) + except (TypeError, ValueError): + return default + + +def _normalize_issues(value: Any) -> list[str]: + if not value: + return [] + if isinstance(value, str): + return [] if value.lower() == "none" else [value] + if isinstance(value, list): + issues = [] + for item in value: + item_text = str(item).strip() + if item_text and item_text.lower() != "none": + issues.append(item_text) + return issues + return [str(value)] + + +EFFECTIVENESS_LABEL_MAP = { + "missing_title": "Effectiveness.Error_Title_Miss", + "missing_abstract": "Effectiveness.Error_Abstract_Miss", + "missing_keywords": "Effectiveness.Error_Keywords_Miss", + "missing_venue": "Effectiveness.Error_Venue_Miss", + "html_tag": "Effectiveness.Error_HTML_Tag", + "mojibake": "Effectiveness.Error_Mojibake", + "invisible_char": "Effectiveness.Error_Invisible_Char", + "unreadable_text": "Effectiveness.Error_Unreadable_Text", + "special_char_noise": "Effectiveness.Error_Special_Char_Noise", + "llm_quality_parse_error": "Effectiveness.Error_LLM_Quality_Parse", + "RuleSpecialCharacter": "Effectiveness.Error_Rule_Special_Character", + "RuleInvisibleChar": "Effectiveness.Error_Rule_Invisible_Char", + "RuleMojibake": "Effectiveness.Error_Mojibake", +} + + +def _issue_to_label(issue: str) -> str | None: + issue_text = str(issue or "").strip() + if not issue_text: + return None + issue_type = issue_text.split(":")[-1] + return EFFECTIVENESS_LABEL_MAP.get(issue_type) + + +def _issues_to_labels(issues: list[str] | None) -> list[str]: + """Map issues to final business labels. + + RuleSpecialCharacter and RuleInvisibleChar are candidate triggers. When LLM + confirms a concrete issue such as title:html_tag, keep the concrete + business label and suppress the intermediate rule label to avoid duplicate + output files for the same problem. + """ + labels: list[str] = [] + normalized_issues = [str(issue or "").strip() for issue in (issues or []) if str(issue or "").strip()] + has_confirmed_quality_issue = any( + ":" in issue and _issue_to_label(issue) is not None + for issue in normalized_issues + ) + + for issue in normalized_issues: + issue_type = issue.split(":")[-1] + if has_confirmed_quality_issue and issue_type in { + "RuleSpecialCharacter", + "RuleInvisibleChar", + "RuleMojibake", + }: + continue + label = _issue_to_label(issue) + if label and label not in labels: + labels.append(label) + return labels + + +def _truncate_for_llm(value: str, max_chars: int = 1000) -> str: + text = str(value or "") + if len(text) <= max_chars: + return text + return text[:max_chars] + "...[truncated]" + + +def extract_keywords(result: dict[str, Any]) -> list[str]: + value = result.get("keywords") or result.get("keyword") or result.get("concepts") or [] + if isinstance(value, str): + return [item.strip() for item in re.split(r"[,;|]", value) if item.strip()] + if isinstance(value, list): + keywords: list[str] = [] + for item in value: + if isinstance(item, dict): + name = item.get("name") or item.get("display_name") or item.get("keyword") + if name: + keywords.append(str(name)) + elif item not in (None, ""): + keywords.append(str(item)) + return keywords + return [] + + +def extract_venue(result: dict[str, Any]) -> str: + return str( + result.get("publication_venue_name_unified") + or result.get("publication_venue_name") + or result.get("venue") + or result.get("source") + or "" + ) + + +@dataclass +class LLMFieldQuality: + """LLM readability and corruption judgment for one search result.""" + + title_score: float = 1.0 + abstract_score: float = 1.0 + keywords_score: float = 1.0 + venue_score: float = 1.0 + issues: list[str] | None = None + reason: str = "" + error: str = "" + + def field_score(self, field: str) -> float: + return { + "title": self.title_score, + "abstract": self.abstract_score, + "keywords": self.keywords_score, + "venue": self.venue_score, + }.get(field, 1.0) + + +def _parse_llm_field_quality_response(text: str) -> LLMFieldQuality: + candidate = _extract_json_object(text) + try: + data = json.loads(candidate) + except json.JSONDecodeError as e: + return LLMFieldQuality(error=f"JSON parse failed: {e}. Text: {text[:200]}") + + fields = data.get("fields") if isinstance(data, dict) else {} + if not isinstance(fields, dict): + return LLMFieldQuality(error=f"Missing fields object. Text: {candidate[:200]}") + + issues: list[str] = [] + scores: dict[str, float] = {} + reasons: list[str] = [] + for field in ("title", "abstract", "keywords", "venue"): + field_data = fields.get(field) or {} + if not isinstance(field_data, dict): + field_data = {} + scores[field] = _safe_float(field_data.get("score"), default=1.0) + for issue in _normalize_issues(field_data.get("issues")): + issues.append(f"{field}:{issue}") + reason = str(field_data.get("reason") or "").strip() + if reason: + reasons.append(f"{field}: {reason}") + + for issue in _normalize_issues(data.get("overall_issues")): + issues.append(issue) + + return LLMFieldQuality( + title_score=scores["title"], + abstract_score=scores["abstract"], + keywords_score=scores["keywords"], + venue_score=scores["venue"], + issues=issues, + reason=str(data.get("reason") or "; ".join(reasons))[:500], + ) + + +@dataclass +class EffectivenessGrade: + """Structured score for one search result.""" + + score: float = 0.0 + title_score: float = 0.0 + abstract_score: float = 0.0 + keywords_score: float = 0.0 + venue_score: float = 0.0 + issues: list[str] | None = None + llm_quality_reason: str = "" + llm_quality_error: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "score": round(self.score, 5), + "title_score": round(self.title_score, 5), + "abstract_score": round(self.abstract_score, 5), + "keywords_score": round(self.keywords_score, 5), + "venue_score": round(self.venue_score, 5), + "issues": self.issues or [], + "llm_quality_reason": self.llm_quality_reason, + "llm_quality_error": self.llm_quality_error, + } + + +@dataclass +class EffectivenessSummary: + mean_score: float = 0.0 + median_score: float = 0.0 + mean_title_score: float = 0.0 + mean_abstract_score: float = 0.0 + mean_keywords_score: float = 0.0 + mean_venue_score: float = 0.0 + graded_pairs: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "effectiveness_mean_score": round(self.mean_score, 5), + "effectiveness_median_score": round(self.median_score, 5), + "effectiveness_mean_title_score": round(self.mean_title_score, 5), + "effectiveness_mean_abstract_score": round(self.mean_abstract_score, 5), + "effectiveness_mean_keywords_score": round(self.mean_keywords_score, 5), + "effectiveness_mean_venue_score": round(self.mean_venue_score, 5), + "effectiveness_graded_pairs": self.graded_pairs, + } + + +@Model.llm_register("LLMSearchResultEffectiveness") +class LLMSearchResultEffectiveness: + """Effectiveness scorer for title, abstract, keywords, and venue.""" + + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + + def __init__( + self, + *, + model: str | None = None, + api_key: str | None = None, + api_url: str | None = None, + max_tokens: int = 512, + temperature: float = 0.0, + timeout: float | None = None, + enable_llm_quality: bool = False, + ): + self.model = model or "gpt-4o" + self.api_key = api_key + self.api_url = api_url + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout + self.enable_llm_quality = enable_llm_quality + self._client = None + + def _get_client(self): + if self._client is None: + from openai import OpenAI + + kwargs: dict[str, Any] = {} + if self.api_key: + kwargs["api_key"] = self.api_key + if self.api_url: + kwargs["base_url"] = self.api_url + self._client = OpenAI(**kwargs) + return self._client + + def _build_llm_quality_user_message( + self, + *, + title: str, + abstract: str, + keywords: list[str], + venue: str, + candidate_fields: set[str] | None = None, + ) -> str: + all_fields = { + "title": title, + "abstract": abstract, + "keywords": " | ".join(keywords), + "venue": venue, + } + selected = candidate_fields or set(all_fields) + payload = { + field: _truncate_for_llm(value) + for field, value in all_fields.items() + if field in selected + } + return ( + "Evaluate only the supplied fields for readability and corruption. " + "Omitted fields should not be judged. Return compact JSON.\n\n" + f"{json.dumps(payload, ensure_ascii=False, indent=2)}" + ) + + def _judge_llm_field_quality( + self, + *, + title: str, + abstract: str, + keywords: list[str], + venue: str, + candidate_fields: set[str] | None = None, + ) -> LLMFieldQuality: + if not self.enable_llm_quality: + return LLMFieldQuality() + try: + client = self._get_client() + completion = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": LLM_FIELD_QUALITY_SYSTEM_PROMPT}, + { + "role": "user", + "content": self._build_llm_quality_user_message( + title=title, + abstract=abstract, + keywords=keywords, + venue=venue, + candidate_fields=candidate_fields, + ), + }, + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + response_text = completion.choices[0].message.content or "" + return _parse_llm_field_quality_response(response_text) + except Exception as e: + logger.warning("LLM field quality judgment failed for title=%r: %s", title, e) + return LLMFieldQuality(error=str(e)) + + def grade( + self, + *, + title: str = "", + abstract: str = "", + keywords: list[str] | str | None = None, + venue: str = "", + result: dict[str, Any] | None = None, + ) -> EffectivenessGrade: + if result is not None: + title = str(result.get("title") or result.get("display_name") or title or "") + abstract = str(result.get("abstract") or abstract or "") + keywords = extract_keywords(result) if keywords is None else keywords + venue = extract_venue(result) or venue + + keyword_items = ( + [item.strip() for item in re.split(r"[,;|]", keywords) if item.strip()] + if isinstance(keywords, str) + else [str(item).strip() for item in (keywords or []) if str(item).strip()] + ) + + title_score = _field_quality(title, min_chars=6, good_chars=35) + if _token_count(title) <= 2 and len(str(title).strip()) < 15: + title_score *= 0.65 + + abstract_score = _field_quality(abstract, min_chars=80, good_chars=700) + if abstract and _token_count(abstract) < 25: + abstract_score *= 0.7 + + keywords_score = _clamp(len(keyword_items) / 5.0) if keyword_items else 0.0 + + venue_score = _field_quality(venue, min_chars=3, good_chars=30) + if venue and not re.search(r"[A-Za-z\u4e00-\u9fff]", venue): + venue_score *= 0.4 + + issues: list[str] = [] + if not str(title or "").strip(): + issues.append("missing_title") + if not str(abstract or "").strip(): + issues.append("missing_abstract") + if not keyword_items: + issues.append("missing_keywords") + if not str(venue or "").strip(): + issues.append("missing_venue") + + field_values = { + "title": str(title or ""), + "abstract": str(abstract or ""), + "keywords": " | ".join(keyword_items), + "venue": str(venue or ""), + } + rule_candidate_issues = { + field: _rule_abnormal_char_issues(value) + for field, value in field_values.items() + if value + } + rule_candidate_issues = { + field: field_issues + for field, field_issues in rule_candidate_issues.items() + if field_issues + } + + llm_quality = LLMFieldQuality() + if rule_candidate_issues and self.enable_llm_quality: + llm_quality = self._judge_llm_field_quality( + title=str(title or ""), + abstract=str(abstract or ""), + keywords=keyword_items, + venue=str(venue or ""), + candidate_fields=set(rule_candidate_issues), + ) + + def apply_confirmed_field_issue(field: str, score: float) -> float: + field_rule_issues = rule_candidate_issues.get(field) or [] + if not field_rule_issues: + return score + + if not self.enable_llm_quality: + issues.extend(field_rule_issues) + return min(score, 0.1) + + if llm_quality.error: + return score + + field_llm_issues = [ + issue for issue in (llm_quality.issues or []) + if str(issue).startswith(f"{field}:") + ] + field_llm_issues = _filter_llm_field_issues( + field, + field_values.get(field, ""), + field_llm_issues, + ) + llm_field_score = llm_quality.field_score(field) + if llm_field_score < 1.0 or _has_confirmed_llm_issue(field_llm_issues): + issues.extend(field_rule_issues) + issues.extend(field_llm_issues) + return min(score, llm_field_score) + return score + + title_score = apply_confirmed_field_issue("title", title_score) + abstract_score = apply_confirmed_field_issue("abstract", abstract_score) + keywords_score = apply_confirmed_field_issue("keywords", keywords_score) + venue_score = apply_confirmed_field_issue("venue", venue_score) + + if rule_candidate_issues and self.enable_llm_quality and llm_quality.error: + issues.append("llm_quality_parse_error") + + score = ( + 0.25 * title_score + + 0.45 * abstract_score + + 0.15 * keywords_score + + 0.15 * venue_score + ) + return EffectivenessGrade( + score=_clamp(score), + title_score=_clamp(title_score), + abstract_score=_clamp(abstract_score), + keywords_score=_clamp(keywords_score), + venue_score=_clamp(venue_score), + issues=issues, + llm_quality_reason=llm_quality.reason, + llm_quality_error=llm_quality.error, + ) + + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def _build_from_config(cls) -> "LLMSearchResultEffectiveness": + return cls( + model=cls.dynamic_config.model, + api_key=cls.dynamic_config.key, + api_url=cls.dynamic_config.api_url, + max_tokens=int(cls._config_value("max_tokens", 512) or 512), + temperature=float(cls._config_value("temperature", 0.0) or 0.0), + timeout=cls._config_value("timeout", None), + enable_llm_quality=bool(cls._config_value("enable_llm_quality", False)), + ) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened search result row.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + + grader = cls._build_from_config() + grade = grader.grade(result=result) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + + labels = _issues_to_labels(grade.issues) + + if grade.score < threshold and "Effectiveness.Error_Effectiveness_Low" not in labels: + labels.append("Effectiveness.Error_Effectiveness_Low") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[grade.to_dict()], + ) + + +def aggregate_grades(grades: list[EffectivenessGrade]) -> EffectivenessSummary: + if not grades: + return EffectivenessSummary() + return EffectivenessSummary( + mean_score=statistics.mean(g.score for g in grades), + median_score=statistics.median(g.score for g in grades), + mean_title_score=statistics.mean(g.title_score for g in grades), + mean_abstract_score=statistics.mean(g.abstract_score for g in grades), + mean_keywords_score=statistics.mean(g.keywords_score for g in grades), + mean_venue_score=statistics.mean(g.venue_score for g in grades), + graded_pairs=len(grades), + ) diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index d3f1a62b..ed43b018 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -17,12 +17,23 @@ from __future__ import annotations import json import logging +import re import statistics from dataclasses import dataclass from typing import Any +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model import Model + logger = logging.getLogger(__name__) +HTML_TAG_PATTERN = r"<[^>]+>" +MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}|�{2,}" +INVISIBLE_CHAR_PATTERN = r"[\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" +DOI_PATTERN = re.compile(r"(?i)(?:https?://(?:dx\.)?doi\.org/)?(10\.\d{4,9}/[^\s]+)") + STANDARD_SYSTEM_PROMPT = """\ You are a helpful assistant that grades the relevance of search results for given queries. Your task is to assign a relevance score between 0.0 and 1.0 to each result, based on @@ -30,12 +41,20 @@ For each search result, carefully read the query and the result. Assign a value for each criterion as follows: -- Provide a brief explanation of your reasoning. +- Provide a brief explanation of your reasoning in 20 words or fewer. - Assign a query_relevance score between 0.0 and 1.0. - Assign a result_quality score between 0.0 and 1.0. -- Indicate if there are any content_issues (true/false). +- Indicate if there are severe content_issues (true/false). - Assign a confidence score between 0.0 and 1.0. -- Assign an overall score between 0.0 and 1.0.""" +- Assign an overall score between 0.0 and 1.0. + +Set content_issues to true only for severe content corruption, such as garbled/mojibake text, +raw HTML/XML or parser residue that materially hurts readability, invisible/control characters, +or unreadable content. Do not mark normal snippets, short abstracts, missing abstracts, or +truncated previews as content_issues when the title/snippet is still readable. + +Return only one valid JSON object. Keep reasoning short and do not use double +quotes inside the reasoning string.""" DETAILED_SYSTEM_PROMPT = """\ You are a helpful assistant that grades the relevance of search results for given queries. @@ -73,9 +92,11 @@ 2. result_quality: The authority, accuracy, and trustworthiness of the result. High-quality \ results come from reputable sources, are well-written, and are not spammy or misleading. -3. content_issues: A boolean indicating whether there are problems with the content, such as \ -truncation, missing information, or improper parsing. If the result is incomplete or garbled, \ -set this to true. +3. content_issues: A boolean indicating severe content corruption only. Set this to true \ +when the visible title/content has garbled or mojibake text, raw HTML/XML or parser residue \ +that materially hurts readability, invisible/control characters, or unreadable text. Do NOT \ +set this to true merely because the abstract is missing, the snippet is short, or the preview \ +is truncated, if the visible title/snippet is still readable enough to judge relevance. 4. confidence: How certain you are about your grading. If the result snippet is clear and \ directly answers the query, confidence should be high. If you need external information to \ @@ -86,15 +107,18 @@ For each search result, carefully read the query and the result. Assign a value for \ each criterion as follows: -- Provide a brief explanation of your reasoning. +- Provide a brief explanation of your reasoning in 20 words or fewer. - Assign a query_relevance score between 0.0 and 1.0. - Assign a result_quality score between 0.0 and 1.0. -- Indicate if there are any content_issues (true/false). +- Indicate if there are severe content_issues (true/false). - Assign a confidence score between 0.0 and 1.0. - Assign an overall score between 0.0 and 1.0. Be consistent and use decimal points for fine-grained differentiation. If you are unsure \ -due to missing or unclear information, lower your confidence and make a best guess as to the score.""" +due to missing or unclear information, lower your confidence and make a best guess as to the score. + +Return only one valid JSON object. Keep reasoning short and do not use double quotes inside \ +the reasoning string. If you need quotation marks in reasoning, use single quotes.""" @dataclass @@ -179,11 +203,13 @@ def _build_user_message( '"result_quality": 0.0-1.0, "content_issues": true/false, ' '"confidence": 0.0-1.0, "score": 0.0-1.0}' ) + parts.append( + "Return JSON only. Keep reasoning under 20 words and do not use double quotes inside reasoning." + ) return "\n".join(parts) -def _parse_grade_response(response_text: str) -> RelevanceGrade: - """Parse LLM JSON response into a RelevanceGrade.""" +def _strip_json_fence(response_text: str) -> str: text = response_text.strip() if text.startswith("```json"): text = text[7:] @@ -191,28 +217,207 @@ def _parse_grade_response(response_text: str) -> RelevanceGrade: text = text[3:] if text.endswith("```"): text = text[:-3] - text = text.strip() + return text.strip() - try: - data = json.loads(text) - except json.JSONDecodeError: - return RelevanceGrade(error=f"JSON parse failed: {text[:200]}") +def _extract_json_object(text: str) -> str | None: + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end <= start: + return None + return text[start:end + 1].strip() + + +def _repair_unescaped_quotes_in_reasoning(text: str) -> str: + """Escape stray double quotes inside the reasoning JSON string. + + Some models emit otherwise valid JSON such as: + {"reasoning": "The query "PBPK" matches", "score": 0.9, ...} + The inner quotes break json.loads. This repair scopes the change to the + reasoning value and leaves the following JSON keys untouched. + """ + start_match = re.search(r'("reasoning"\s*:\s*")', text) + if not start_match: + return text + + value_start = start_match.end() + next_key = re.search( + r'"\s*,\s*"(query_relevance|result_quality|content_issues|confidence|score)"\s*:', + text[value_start:], + flags=re.DOTALL, + ) + if not next_key: + return text + + value_end = value_start + next_key.start() + value = text[value_start:value_end] + repaired_value = re.sub(r'(? float: try: - if not isinstance(data, dict): - return RelevanceGrade(error=f"JSON is not a dictionary: {text[:200]}") - return RelevanceGrade( - score=float(data.get("score", 0.0)), - query_relevance=float(data.get("query_relevance", 0.0)), - result_quality=float(data.get("result_quality", 0.0)), - content_issues=bool(data.get("content_issues", False)), - confidence=float(data.get("confidence", 0.0)), - reasoning=str(data.get("reasoning", "")), + return float(value) + except (TypeError, ValueError): + return 0.0 + + +def _clamp_score(value: Any) -> float: + return max(0.0, min(1.0, _coerce_float(value))) + + +def _grade_from_dict(data: dict[str, Any]) -> RelevanceGrade: + return RelevanceGrade( + score=_clamp_score(data.get("score", 0.0)), + query_relevance=_clamp_score(data.get("query_relevance", 0.0)), + result_quality=_clamp_score(data.get("result_quality", 0.0)), + content_issues=bool(data.get("content_issues", False)), + confidence=_clamp_score(data.get("confidence", 0.0)), + reasoning=str(data.get("reasoning", "")), + ) + + +def _parse_grade_fields_lenient(text: str) -> RelevanceGrade | None: + number_fields = {} + for field in ("query_relevance", "result_quality", "confidence", "score"): + match = re.search( + rf'"{field}"\s*:\s*([-+]?(?:\d+(?:\.\d*)?|\.\d+))', + text, + flags=re.IGNORECASE, ) - except (ValueError, TypeError) as e: - return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {text[:200]}") + if match: + number_fields[field] = _clamp_score(match.group(1)) + + if "score" not in number_fields and "query_relevance" not in number_fields: + return None + + issue_match = re.search(r'"content_issues"\s*:\s*(true|false)', text, flags=re.IGNORECASE) + reasoning = "" + reasoning_match = re.search( + r'"reasoning"\s*:\s*"(.*?)"\s*,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"', + text, + flags=re.DOTALL, + ) + if reasoning_match: + reasoning = " ".join(reasoning_match.group(1).split()) + + return RelevanceGrade( + score=number_fields.get("score", number_fields.get("query_relevance", 0.0)), + query_relevance=number_fields.get("query_relevance", 0.0), + result_quality=number_fields.get("result_quality", 0.0), + content_issues=issue_match.group(1).lower() == "true" if issue_match else False, + confidence=number_fields.get("confidence", 0.0), + reasoning=reasoning, + ) + +def _parse_grade_response(response_text: str) -> RelevanceGrade: + """Parse LLM JSON response into a RelevanceGrade.""" + text = _strip_json_fence(response_text) + candidates = [text] + extracted = _extract_json_object(text) + if extracted and extracted != text: + candidates.append(extracted) + + repaired_candidates = [] + for candidate in candidates: + repaired = _repair_unescaped_quotes_in_reasoning(candidate) + if repaired != candidate: + repaired_candidates.append(repaired) + candidates.extend(repaired_candidates) + + for candidate in candidates: + try: + data = json.loads(candidate) + except json.JSONDecodeError: + continue + try: + if not isinstance(data, dict): + return RelevanceGrade(error=f"JSON is not a dictionary: {candidate[:200]}") + return _grade_from_dict(data) + except (ValueError, TypeError) as e: + return RelevanceGrade(error=f"Failed to parse grade response: {e}. Text: {candidate[:200]}") + + lenient = _parse_grade_fields_lenient(extracted or text) + if lenient: + return lenient + return RelevanceGrade(error=f"JSON parse failed: {text[:200]}") + + +def _content_issue_evidence(title: str, abstract: str) -> list[str]: + text = "\n".join([str(title or ""), str(abstract or "")]) + issues: list[str] = [] + if re.search(MOJIBAKE_EVIDENCE_PATTERN, text): + issues.append("mojibake_or_garbled_text") + if re.search(INVISIBLE_CHAR_PATTERN, text): + issues.append("invisible_or_control_char") + html_matches = re.findall(HTML_TAG_PATTERN, text) + if html_matches: + issues.append("html_or_xml_tag_residue") + if re.search(r"(/docserver/|<\?xml| bool: + return bool(_content_issue_evidence(title, abstract)) + + +def _normalize_doi(value: Any) -> str: + """Extract and normalize a DOI from a query or result field.""" + text = str(value or "").strip() + match = DOI_PATTERN.search(text) + if not match: + return "" + return match.group(1).rstrip(".,;:)]}").lower() + + +def is_doi_query(query: str) -> bool: + return bool(_normalize_doi(query)) + + +def _extract_result_dois(result: dict[str, Any]) -> list[str]: + candidates: list[Any] = [result.get("doi"), result.get("unique_id")] + for location in result.get("locations") or []: + if isinstance(location, dict): + candidates.extend([location.get("doi"), location.get("url"), location.get("landing_page_url")]) + + dois: list[str] = [] + for candidate in candidates: + values = candidate if isinstance(candidate, list) else [candidate] + for value in values: + doi = _normalize_doi(value) + if doi and doi not in dois: + dois.append(doi) + return dois + + +def _grade_doi_result(query: str, result: dict[str, Any]) -> RelevanceGrade | None: + """Use deterministic identifier matching when the query is a DOI.""" + query_doi = _normalize_doi(query) + if not query_doi: + return None + + result_dois = _extract_result_dois(result) + exact_match = query_doi in result_dois + score = 1.0 if exact_match else 0.0 + result_text = ", ".join(result_dois) if result_dois else "missing" + return RelevanceGrade( + score=score, + query_relevance=score, + result_quality=1.0 if exact_match else 0.0, + content_issues=False, + confidence=1.0, + reasoning=( + f"Exact DOI match: {query_doi}." + if exact_match + else f"DOI mismatch: expected {query_doi}; result {result_text}." + ), + ) + + +@Model.llm_register("LLMSearchResultRelevance") class LLMSearchResultRelevance: """Exa-style pointwise search result relevance grader. @@ -220,6 +425,9 @@ class LLMSearchResultRelevance: ``BaseOpenAI`` evaluator hierarchy. """ + dynamic_config = EvaluatorLLMArgs() + default_threshold = 0.15 + def __init__( self, *, @@ -228,12 +436,18 @@ def __init__( api_url: str | None = None, prompt_mode: str = "standard", expected_criteria: str | None = None, + max_tokens: int = 1024, + temperature: float = 0.0, + timeout: float | None = None, ): self.model = model or "gpt-4o" self.api_key = api_key self.api_url = api_url self.prompt_mode = prompt_mode self.expected_criteria = expected_criteria + self.max_tokens = max_tokens + self.temperature = temperature + self.timeout = timeout self._client = None def _get_client(self): @@ -269,8 +483,9 @@ def grade( {"role": "system", "content": system_prompt}, {"role": "user", "content": user_message}, ], - temperature=0.0, - max_tokens=512, + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, ) response_text = completion.choices[0].message.content or "" return _parse_grade_response(response_text) @@ -278,6 +493,78 @@ def grade( logger.warning("LLM grading failed for query=%r title=%r: %s", query, title, e) return RelevanceGrade(error=str(e)) + @classmethod + def _config_value(cls, name: str, default: Any = None) -> Any: + return getattr(cls.dynamic_config, name, default) + + @classmethod + def _build_from_config(cls) -> "LLMSearchResultRelevance": + return cls( + model=cls.dynamic_config.model, + api_key=cls.dynamic_config.key, + api_url=cls.dynamic_config.api_url, + prompt_mode=str(cls._config_value("prompt_mode", "detailed") or "detailed"), + expected_criteria=cls._config_value("expected_criteria", None), + max_tokens=int(cls._config_value("max_tokens", 1024) or 1024), + temperature=float(cls._config_value("temperature", 0.0) or 0.0), + timeout=cls._config_value("timeout", None), + ) + + @staticmethod + def _extract_title(result: dict[str, Any]) -> str: + return str(result.get("title") or result.get("display_name") or "") + + @staticmethod + def _extract_abstract(result: dict[str, Any]) -> str: + return str(result.get("abstract") or result.get("summary") or result.get("content") or "") + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + """Executor entry point: evaluate one flattened query-result pair.""" + result = getattr(input_data, "search_result", None) + if not isinstance(result, dict): + result = input_data.to_dict() + query = str(getattr(input_data, "query", "") or result.get("_eval_query") or result.get("query") or "") + + title = cls._extract_title(result) + abstract = cls._extract_abstract(result) + grade = _grade_doi_result(query, result) + if grade is None: + grader = cls._build_from_config() + grade = grader.grade( + query=query, + title=title, + abstract=abstract, + ) + threshold = float(cls._config_value("threshold", cls.default_threshold) or cls.default_threshold) + content_issue_evidence = _content_issue_evidence(title, abstract) if grade.content_issues else [] + effective_content_issues = bool(content_issue_evidence) + + labels: list[str] = [] + if grade.error: + labels.append("Relevance.Error_Parse") + if grade.score < threshold: + labels.append("Relevance.Error_Relevance_Low") + if effective_content_issues: + labels.append("Relevance.Error_Content_Issues") + + status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + + reason = grade.to_dict() + reason["raw_content_issues"] = grade.content_issues + reason["content_issues"] = effective_content_issues + reason["content_issue_evidence"] = content_issue_evidence + + return EvalDetail( + metric=cls.__name__, + status=status, + score=round(grade.score, 5), + label=labels, + reason=[reason], + ) + def aggregate_grades( grades: list[RelevanceGrade], diff --git a/docs/search_result_authority_executor.md b/docs/search_result_authority_executor.md new file mode 100644 index 00000000..17f3a6d0 --- /dev/null +++ b/docs/search_result_authority_executor.md @@ -0,0 +1,162 @@ +# Search Result Authority Executor Usage + +This document describes the executor-based authority evaluation for meta search results. + +## What Changed + +`examples/retrieval/sdk_eval_authority.py` evaluates each retrieved result with `LLMSearchResultAuthority` through Dingo `LocalExecutor`. + +The standalone authority script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds. + +`LLMSearchResultAuthority` is rule-based despite being registered as an LLM evaluator. It does not call an external model. The score is based on citation impact, venue/source signals, and DOI availability. + +## Flow + +1. Read query-level meta search JSONL. +2. Flatten each query's top-k results into result-level JSONL rows. +3. Build `InputArgs`. +4. Run `Executor.exec_map["local"]`. +5. `LLMSearchResultAuthority` returns standard `EvalDetail(metric/status/score/label/reason)`. +6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Authority/Error_*.jsonl`. + +Each flattened input row contains: + +| Field | Meaning | +|---|---| +| `query` | Original query text | +| `query_index` | Query order in source JSONL | +| `rank` | Result rank under the query | +| `title` | Result title or display name | +| `search_result` | Full result payload passed to evaluator | + +## Commands + +Run authority evaluation: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_authority_97q_executor ` + --top-k 10 ` + --threshold 0.15 ` + --save-good +``` + +Smoke test with fewer queries: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_authority_smoke ` + --top-k 10 ` + --max-queries 5 ` + --threshold 0.15 ` + --save-good +``` + +Useful parameters: + +| Parameter | Default | Meaning | +|---|---:|---| +| `--top-k` | `10` | Number of results evaluated per query | +| `--max-queries` | `None` | Limit query count for smoke tests | +| `--threshold` | `0.15` | Result-level bad threshold | +| `--max-workers` | `4` | LocalExecutor worker count | +| `--batch-size` | `10` | LocalExecutor batch size | +| `--save-good` | off | Save passing samples | +| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows | + +## Output + +The executor creates a timestamped child directory under `--output-dir`, for example: + +```text +outputs/search_result_authority_97q_executor/20260709_172501_0f73c631/ +``` + +Main files: + +| Path | Meaning | +|---|---| +| `summary.json` | Executor summary | +| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` | +| `search_result/Authority/Error_*.jsonl` | Bad result-level samples grouped by error type | + +`summary.json` uses result-level statistics: + +- `total`: number of evaluated retrieved results. +- `num_good`: result count with no error labels. +- `num_bad`: result count with at least one error label. +- `score`: `num_good / total * 100`. +- `metrics_score.search_result.stats.LLMSearchResultAuthority`: result-level authority score distribution. +- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio. + +Error labels: + +| Label path | Trigger | +|---|---| +| `search_result/Authority/Error_Authority_Low.jsonl` | Final authority score below threshold | +| `search_result/Authority/Error_Citation_Miss.jsonl` | Authority is low and citation score is zero | +| `search_result/Authority/Error_Venue_Low_Signal.jsonl` | Authority is low and venue/source has only low signal | +| `search_result/Authority/Error_DOI_Miss.jsonl` | Authority is low and no DOI signal is present | + +The sub-labels are emitted only when the final authority score is below the threshold. For example, a result without DOI can still be `QUALITY_GOOD` if citation and venue signals are strong enough. + +## Scoring + +The metric score is: + +```text +authority = + 0.45 * citation_score ++ 0.20 * influential_citation_score ++ 0.25 * venue_score ++ 0.10 * doi_score +``` + +Citation scores use log normalization and are clamped to `[0, 1]`: + +```text +citation_score = log1p(citation_count) / log1p(500) +influential_citation_score = log1p(influential_citation_count) / log1p(50) +``` + +Venue is read from the first available field: + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +Venue scoring: + +| Condition | `venue_score` | Reason | +|---|---:|---| +| Venue name contains a high-authority hint | `0.85` | `high_authority_venue_hint` | +| `publication_venue_type` contains journal or conference | `0.65` | `journal_or_conference` | +| `publication_venue_type` contains repository, or venue contains preprint | `0.45` | `repository_or_preprint` | +| Unknown or low-signal source | `0.25` | `unknown_or_low_signal_venue` | + +High-authority venue hints include: + +```text +nature, science, cell, nejm, lancet, jama, +acm, ieee, springer, elsevier, wiley, +neurips, icml, iclr, cvpr, acl, emnlp, aaai, ijcai, sigir +``` + +DOI scoring: + +```text +doi_score = 1.0 +``` + +when the result has a `doi` field or `locations` contains `doi.org`; otherwise: + +```text +doi_score = 0.0 +``` + +Authority does not judge query relevance or content completeness. A low authority score means the result lacks academic trust signals such as citations, venue, or DOI. It does not necessarily mean the result is irrelevant or unusable. diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md new file mode 100644 index 00000000..d4031078 --- /dev/null +++ b/docs/search_result_effectiveness_executor.md @@ -0,0 +1,131 @@ +# Search Result Effectiveness Executor Usage + +This document describes the executor-based effectiveness evaluation for meta search results. + +## What Changed + +`examples/retrieval/sdk_eval_effectiveness.py` reuses Dingo `LocalExecutor`. It no longer hand-builds chunk-style `summary.json` or bad/good folders. + +The standalone effectiveness script evaluates each retrieved result as one test object. The combined script `sdk_eval_search_result.py` still evaluates at query level and applies query-level bad/good thresholds. + +## Flow + +1. Read query-level meta search JSONL. +2. Flatten each query's top-k results into result-level JSONL rows. +3. Build `InputArgs`. +4. Run `Executor.exec_map["local"]`. +5. `LLMSearchResultEffectiveness` returns standard `EvalDetail(metric/status/score/label/reason)`. +6. `LocalExecutor` writes timestamped output, `summary.json`, `search_result/QUALITY_GOOD.jsonl`, and `search_result/Effectiveness/Error_*.jsonl`. + +Each flattened input row contains: + +| Field | Meaning | +|---|---| +| `query` | Original query text | +| `query_index` | Query order in source JSONL | +| `rank` | Result rank under the query | +| `title` | Result title or display name | +| `search_result` | Full result payload passed to evaluator | + +## Commands + +Fast rule-only run: + +```powershell +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_effectiveness_97q_executor ` + --top-k 10 ` + --threshold 0.15 ` + --disable-llm-quality ` + --save-good +``` + +Run with LLM second judgment for abnormal-character candidates: + +```powershell +$env:OPENAI_API_KEY="..." +$env:OPENAI_BASE_URL="http://35.220.164.252:3888/v1/" +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0.7" + +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_effectiveness_97q_executor_llm ` + --top-k 10 ` + --threshold 0.15 ` + --llm-max-tokens 512 ` + --llm-workers 4 ` + --save-good +``` + +Useful parameters: + +| Parameter | Default | Meaning | +|---|---:|---| +| `--top-k` | `10` | Number of results evaluated per query | +| `--max-queries` | `None` | Limit query count for smoke tests | +| `--threshold` | `0.15` | Result-level bad threshold | +| `--save-good` | off | Save passing samples | +| `--disable-llm-quality` | off | Skip LLM second judgment and use deterministic rules only | +| `--llm-max-tokens` | `512` | Max tokens for LLM second judgment | +| `--llm-workers` | `4` | LocalExecutor worker count | +| `--batch-size` | `10` | LocalExecutor batch size | +| `--raw-output` | off | Merge raw data and Dingo result in output JSONL rows | + +## Output + +The executor creates a timestamped child directory under `--output-dir`, for example: + +```text +outputs/search_result_effectiveness_97q_executor/20260709_172501_0f73c631/ +``` + +Main files: + +| Path | Meaning | +|---|---| +| `summary.json` | Executor summary | +| `search_result/QUALITY_GOOD.jsonl` | Passing result-level samples, only with `--save-good` | +| `search_result/Effectiveness/Error_*.jsonl` | Bad result-level samples grouped by error type | + +`summary.json` uses result-level statistics: + +- `total`: number of evaluated retrieved results. +- `num_good`: result count with no error labels. +- `num_bad`: result count with at least one error label. +- `score`: `num_good / total * 100`. +- `metrics_score.search_result.stats.LLMSearchResultEffectiveness`: result-level effectiveness score distribution. +- `type_ratio.search_result`: label ratios. One result can have multiple error labels, so error ratios can sum to more than the bad ratio. + +Error labels: + +| Label path | Trigger | +|---|---| +| `search_result/Effectiveness/Error_Title_Miss.jsonl` | Missing title | +| `search_result/Effectiveness/Error_Abstract_Miss.jsonl` | Missing abstract | +| `search_result/Effectiveness/Error_Keywords_Miss.jsonl` | Missing keywords | +| `search_result/Effectiveness/Error_Venue_Miss.jsonl` | Missing publication venue/source | +| `search_result/Effectiveness/Error_HTML_Tag.jsonl` | LLM-confirmed HTML tag pollution | +| `search_result/Effectiveness/Error_Mojibake.jsonl` | LLM-confirmed mojibake | +| `search_result/Effectiveness/Error_Invisible_Char.jsonl` | LLM-confirmed invisible characters | +| `search_result/Effectiveness/Error_Unreadable_Text.jsonl` | LLM-confirmed unreadable text | +| `search_result/Effectiveness/Error_Special_Char_Noise.jsonl` | LLM-confirmed special-character noise | +| `search_result/Effectiveness/Error_Effectiveness_Low.jsonl` | Final effectiveness score below threshold | + +`RuleSpecialCharacter` and `RuleInvisibleChar` are treated as candidate triggers. If LLM confirms a concrete issue, for example `title:html_tag`, the final label keeps only the concrete business label such as `Effectiveness.Error_HTML_Tag` and suppresses the intermediate rule label. The original rule issue is still retained in `EvalDetail.reason` for traceability. + +## Scoring + +The metric score is unchanged: + +```text +Effectiveness = + title_score * 0.25 ++ abstract_score * 0.45 ++ keywords_score * 0.15 ++ venue_score * 0.15 +``` + +Field scores are based on missing-field checks, length/information-density checks, and abnormal-character handling. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. + diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md new file mode 100644 index 00000000..f45418bd --- /dev/null +++ b/docs/search_result_quality_metrics.md @@ -0,0 +1,641 @@ +# Search Result Quality 三指标评测说明 + +本文档说明 meta search 检索结果的三类评测指标：相关性、内容有效性、权威性，以及对应的单项评测脚本和综合评测脚本。该方案面向无人工 GT 的检索结果质量检查，输入为 query 及其 top-k 检索结果，输出 query 级和 result 级分数，并按阈值生成 Dingo 风格的 good/bad 分类目录。 + +## 1. 适用场景 + +该评测用于回答三个业务问题： + +| 指标 | 业务问题 | 评测方式 | +|---|---|---| +| 相关性 `relevance` | 检索结果是否回答了用户 query 的真实检索意图 | LLM 逐条判断 query-result 匹配程度 | +| 内容有效性 `effectiveness` | 结果记录本身是否完整、可读、可用于判断论文价值 | 规则检查字段缺失/信息量，RuleSpecialCharacter/RuleInvisibleChar 初筛异常候选，LLM 二次确认 | +| 权威性 `authority` | 结果是否具备学术可信度和来源影响力信号 | 规则检查 citation、influential citation、venue、DOI | + +三个指标关注点不同： + +- 相关性判断“是不是用户要找的内容”。 +- 内容有效性判断“这条结果记录是否有足够信息可读可用”。 +- 权威性判断“这条结果是否有论文影响力、来源、DOI 等可信信号”。 + +例如，用户搜索 `Wallace Chafe`，rank1 返回标题也是 `Wallace Chafe`，相关性可能较好；但如果该结果没有 abstract、keywords、publication venue，则内容有效性会较低。 + +## 2. 输入格式 + +输入文件为 JSONL，每行一个 query 及其检索结果。 + +```json +{"query": "PBPK Review", "results": [{"title": "...", "abstract": "..."}]} +``` + +脚本支持的 query 字段名： + +- `query` +- `query_text` +- `q` + +脚本支持的结果列表字段名： + +- `results` +- `top_results` +- `top_api_results` +- `search_results` + +常用输入路径示例： + +```bash +outputs/meta_search_97_query_results.jsonl +``` + +## 3. 输出文件 + +综合评测脚本 `sdk_eval_search_result.py` 会在 `--output-dir` 下生成一个时间戳子目录，核心结果都放在该子目录中，例如： + +```text +outputs/search_result_eval_97q/20260710_162652_1ac3f3be/ +``` + +默认核心文件如下： + +| 文件 | 粒度 | 说明 | +|---|---|---| +| `summary.json` | 全局 | 指标均值、中位数、最小值、最大值、bad/good 数量、阈值、LLM 配置等 | +| `query_scores.csv` | query 级 | 每个 query 的 rank-discount 汇总分、label、eval_status | +| `result_scores.csv` | result 级 | 每个 query 的每条 top-k 结果分数和诊断信息 | +| `all_results.jsonl` | result 级原始明细 | executor 输出的逐条评测结果，保留三个指标的完整 `eval_details` | +| `bad/` | query 级分类 | 低于阈值或运行异常的 query 记录 | +| `good/` | query 级分类 | 使用 `--save-good` 时保存通过的 query 记录 | + +`detailed_results.json` 默认不生成；需要完整嵌套诊断时加 `--save-detailed`。 + +单独运行 `sdk_eval_effectiveness.py`、`sdk_eval_relevancy.py`、`sdk_eval_authority.py` 时，会保留 Dingo executor 的单指标 result 级分类目录，例如 `search_result/Effectiveness/Error_*.jsonl`。 + +分类 label 示例： + +```text +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR +QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW +QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW +QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW +QUALITY_GOOD.SEARCH_RESULT_RELEVANCE_PASS +QUALITY_GOOD.SEARCH_RESULT_EFFECTIVENESS_PASS +QUALITY_GOOD.SEARCH_RESULT_AUTHORITY_PASS +QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS +``` + +单独运行 `sdk_eval_effectiveness.py` 时，`summary.json` 采用类似 `sdk_chunk_eval.py` 的 result 级结构：每个 query 的每条 top-k 检索文献都是一个测试对象，`total`、`num_good`、`num_bad` 和 `score` 都按 result 级计算。`type_ratio.search_result` 中会统计 `Effectiveness.Error_*` 和 `QUALITY_GOOD` 的比例，`metrics_score.search_result` 中会统计 `LLMSearchResultEffectiveness` 的 result 级分数分布。 + +内容有效性还会额外输出 result 级错误类型文件，例如： + +```text +Effectiveness/Error_Title_Miss.jsonl +Effectiveness/Error_Abstract_Miss.jsonl +Effectiveness/Error_Keywords_Miss.jsonl +Effectiveness/Error_Venue_Miss.jsonl +Effectiveness/Error_HTML_Tag.jsonl +Effectiveness/Error_Mojibake.jsonl +Effectiveness/Error_Invisible_Char.jsonl +Effectiveness/Error_Unreadable_Text.jsonl +Effectiveness/Error_Special_Char_Noise.jsonl +Effectiveness/Error_LLM_Quality_Parse.jsonl +QUALITY_GOOD.jsonl +``` + +只有实际出现的错误类型会生成对应文件。 + +## 4. Query 级汇总逻辑 + +三个指标都先对 top-k 中每条 result 打分，然后用 rank-discounted mean 汇总到 query 级。 + +第 `rank` 条结果的权重为： + +```text +weight(rank) = 1 / log2(rank + 1) +``` + +query 级分数为： + +```text +query_score = sum(result_score_i * weight_i) / sum(weight_i) +``` + +业务含义： + +- rank1 的影响最大。 +- rank 越靠后，对 query 总分影响越小。 +- 适合评估搜索排序质量，因为用户更关注前几条结果。 + +## 5. 相关性 Relevance + +### 5.1 业务逻辑 + +相关性判断每条检索结果与 query 是否匹配，重点看： + +- result 是否直接回答或覆盖 query 意图。 +- 标题和摘要是否围绕 query 主题。 +- 对短词、人名、论文题名、中文长 query 等非 DOI query，LLM 根据语义进行判断。 +- DOI query 使用结构化精确匹配：规范化 query DOI 与结果 `doi`、`unique_id` 或 location DOI，完全一致才算命中。 +- DOI 前缀相似、同出版社或主题相似但 DOI 不一致时，result 相关性为 0，不允许 LLM 猜测。 + +### 5.2 输入字段 + +| 输入 | 来源 | +|---|---| +| `query` | query 字段 | +| `title` | result 的 `title` 或 `display_name` | +| `abstract` | result 的 `abstract`、`summary` 或 `content` | +| `doi` | DOI query 的精确标识符匹配 | +| `unique_id`、`locations` | result 缺少顶层 DOI 时的标识符补充来源 | + +### 5.3 Result 级输出 + +| 字段 | 说明 | +|---|---| +| `relevance` | result 总相关性分数 | +| `query_relevance` | query 与 result 的语义匹配程度 | +| `result_quality` | result 内容质量辅助判断 | +| `content_issues` | LLM 判断是否存在内容问题 | +| `confidence` | LLM 对评分的置信度 | +| `error` | LLM 输出解析失败等错误 | +| `reasoning` | 简短原因 | + +DOI query 不调用 LLM。Result 级完全匹配为 `1.0`，否则为 `0.0`；reason 中记录 expected DOI 和 result DOI。 + +DOI 的 query 级得分按精确命中的排名折扣： + +```text +doi_relevance = max(exact_match / log2(rank + 1)) +``` + +因此 rank1 命中为 `1.0`，rank2 命中为 `0.63093`，没有精确命中为 `0.0`。普通 query 继续使用 top-k rank-discount mean。 + +### 5.4 Query 级异常 + +如果某个 query 的任意 rank 出现 LLM JSON 解析失败，会增加： + +```text +QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR +``` + +这类 label 表示运行/解析质量告警，不一定代表业务相关性低。分析低相关时建议区分： + +- `SEARCH_RESULT_RELEVANCE_LOW`：业务低相关。 +- `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`：LLM 输出格式或解析异常。 + +### 5.5 可调参数 + +| 参数 | 默认值 | 说明 | +|---|---:|---| +| `--top-k` | 10 | 每个 query 评测前 k 条结果 | +| `--threshold` | 0.15 | query 级 hard bad 阈值 | +| `--llm-max-tokens` | 1024 | LLM 输出最大 token 数 | +| `--llm-workers` | 4 | 并发 LLM 调用数 | +| `--llm-timeout` | 60 | 单次 LLM 请求超时秒数 | +| `--prompt-mode` | `detailed` | prompt 模式 | +| `OPENAI_MODEL` | `gpt-4o` | LLM 模型名，可通过环境变量覆盖 | +| `OPENAI_BASE_URL` | 空 | OpenAI compatible endpoint | +| `OPENAI_TEMPERATURE` | 0.0 | LLM temperature | + +## 6. 内容有效性 Effectiveness + +### 6.1 业务逻辑 + +内容有效性判断的是一条检索结果记录是否“可读、完整、可用于用户判断”，不判断它是否与 query 相关。 + +当前不按 `metadata_type` 做差异化处理。也就是说，`paper`、`ebook`、未来新增类型都用同一套字段完整性标准。这有利于持续观察数据库元数据补全质量：如果 ebook 未来补全 abstract、keywords、venue，有效性分数会自然提升。 + +### 6.2 字段权重 + +单条 result 的分数为： + +```text +Effectiveness = + title_score * 0.25 ++ abstract_score * 0.45 ++ keywords_score * 0.15 ++ venue_score * 0.15 +``` + +| 子项 | 权重 | 业务含义 | +|---|---:|---| +| `title_score` | 0.25 | 标题是否存在、长度是否合理、是否可读 | +| `abstract_score` | 0.45 | 摘要是否存在、信息量是否充足、是否可读 | +| `keywords_score` | 0.15 | 关键词是否存在、是否提供主题信息 | +| `venue_score` | 0.15 | 期刊/会议/来源名称是否存在、是否可读 | + +字段为空时，该字段直接得 0 分。 + +示例：如果 result 只有标题，其他字段为空，且 `title_score=0.32948`： + +```text +0.32948 * 0.25 + 0 + 0 + 0 = 0.08237 +``` + +### 6.3 字段评分逻辑 + +每个字段先做基础质量判断： + +- 为空：0 分。 +- 字段内容异常先由规则筛选，再由 LLM 判断：如果存在 HTML 泄漏、乱码、不可见字符、严重特殊字符噪声等，会按 LLM 字段质量分降低该字段分数。乱码筛选包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 +- 长度太短：低分。 +- 长度和信息量达到要求：接近或等于 1 分。 + +`keywords` 会把列表中的每个 keyword 视为一个主题信号；空列表计为缺失。 + +`venue` 读取优先级： + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +### 6.4 Issues 类型 + +| issue | 含义 | +|---|---| +| `missing_title` | 标题为空 | +| `missing_abstract` | 摘要为空 | +| `missing_keywords` | 关键词为空 | +| `missing_venue` | 期刊/会议/来源名为空 | +| `title:html_tag` / `abstract:html_tag` / `venue:html_tag` | LLM 判断字段中有 HTML/XML 标签泄漏 | +| `*:mojibake` | LLM 判断字段存在乱码或编码错误 | +| `*:invisible_char` | LLM 判断字段存在不可见/控制字符 | +| `*:unreadable_text` | LLM 判断字段整体不可读 | +| `*:special_char_noise` | LLM 判断字段中特殊字符噪声已经影响阅读 | +| `llm_quality_parse_error` | LLM 字段质量判断调用或解析失败 | + +注意： + +- `RuleSpecialCharacter` 和 `RuleInvisibleChar` 只用于快速召回疑似异常字段，不直接作为最终扣分依据。 +- LaTeX、化学符号、单位、希腊字母、`|` 分隔符等正常学术表达不应被 LLM 判为问题。 +- HTML 高亮标签、明显 mojibake、不可见字符、严重乱码会由 LLM 输出字段级 issue，并降低对应字段分数。 +- 分析 bad 样本时建议结合原始 title、abstract、venue 和 `llm_quality_reason` 进行人工抽查。 + +### 6.5 可调参数 + +| 参数 | 默认值 | 说明 | +|---|---:|---| +| `--top-k` | 10 | 每个 query 评测前 k 条结果 | +| `--threshold` | 0.15 | query 级 hard bad 阈值 | +| `--llm-max-tokens` | 1024 | 内容有效性 LLM 字段质量判断的最大 token 数 | +| `--llm-workers` | 4 | 内容有效性 LLM 二次复核并发数 | +| `--llm-timeout` | 60 | LLM 请求超时秒数 | +| `--disable-llm-quality` | false | 关闭 LLM 字段质量判断，仅保留字段缺失/长度规则 | + +阈值建议： + +- `0.15` 适合作 hard bad，只筛严重不可用结果。 +- 如果要发现 metadata 缺失、摘要不足等一般质量问题，可额外关注 `< 0.45` 的 warning 区间。 + +## 7. 权威性 Authority + +### 7.1 业务逻辑 + +权威性判断检索结果是否具备学术可信度和影响力信号，主要来自： + +- 引用数。 +- 高影响引用数。 +- 期刊/会议/来源类型。 +- DOI。 + +该指标不判断 query 相关性，也不判断内容字段是否完整。它适合作为 overall 的辅助指标，不建议单独用于硬判“结果错误”。 + +### 7.2 字段权重 + +单条 result 的权威性分数为： + +```text +authority = + 0.45 * citation_score ++ 0.20 * influential_citation_score ++ 0.25 * venue_score ++ 0.10 * doi_score +``` + +| 子项 | 权重 | 业务含义 | +|---|---:|---| +| `citation_score` | 0.45 | 普通引用影响力 | +| `influential_citation_score` | 0.20 | 高影响引用 | +| `venue_score` | 0.25 | 来源/期刊/会议可信度 | +| `doi_score` | 0.10 | 是否具备 DOI 标识 | + +### 7.3 Citation 归一化 + +引用数使用 log 归一化，避免高引用老论文过度碾压。 + +```text +citation_score = log(1 + citation_count) / log(1 + 500) + +influential_citation_score = + log(1 + influential_citation_count) / log(1 + 50) +``` + +分数会被限制在 `[0, 1]`。 + +### 7.4 Venue 评分 + +`venue` 读取优先级： + +```text +publication_venue_name_unified +publication_venue_name +venue +source +``` + +当前规则： + +| 条件 | `venue_score` | reason | +|---|---:|---| +| venue 名称包含高权威来源提示词 | 0.85 | `high_authority_venue_hint` | +| `publication_venue_type` 是 journal 或 conference | 0.65 | `journal_or_conference` | +| repository 或 preprint | 0.45 | `repository_or_preprint` | +| 未知或低信号来源 | 0.25 | `unknown_or_low_signal_venue` | + +高权威来源提示词包括： + +```text +nature, science, cell, nejm, lancet, jama, +acm, ieee, springer, elsevier, wiley, +neurips, icml, iclr, cvpr, acl, emnlp, aaai, ijcai, sigir +``` + +### 7.5 DOI 评分 + +```text +doi_score = 1.0 +``` + +条件： + +- result 有 `doi` 字段；或 +- `locations` 中包含 `doi.org`。 + +否则： + +```text +doi_score = 0.0 +``` + +### 7.6 使用注意 + +权威性对以下 query 类型可能偏保守： + +- 人名，例如 `Michael Pecht`、`张文宏`。 +- 泛词，例如 `Jerry`、`pam`。 +- 书籍、访谈、百科式结果。 +- 缺少 citation、DOI、venue 元数据但实际有用的结果。 + +因此，权威性低不一定表示检索结果不相关，只表示该结果缺少学术权威信号。 + +## 8. 综合评分 Overall + +综合脚本将三个 query 级指标加权： + +```text +overall = + 0.7 * relevance ++ 0.2 * effectiveness ++ 0.1 * authority +``` + +默认权重： + +| 指标 | 权重 | +|---|---:| +| `relevance` | 0.7 | +| `effectiveness` | 0.2 | +| `authority` | 0.1 | + +业务含义： + +- 相关性是核心，因此权重最高。 +- 内容有效性次之，保证结果有足够元数据可读。 +- 权威性作为辅助，不让 citation/DOI 过度主导搜索体验。 + +综合评估会同时检查： + +- `overall` 是否低于 overall 阈值。 +- `relevance` 是否低于相关性阈值。 +- 是否存在 LLM 解析错误。 +- `effectiveness` 是否低于有效性阈值。 +- `authority` 是否低于权威性阈值。 + +## 9. 使用命令 + +以下命令均在项目根目录执行。 + +综合脚本按评测对象拆分分类目录： + +```text +/ +├── bad/ +│ ├── query_level/ +│ │ └── QUALITY_BAD/ +│ │ ├── SEARCH_RESULT_RELEVANCE_LOW.jsonl +│ │ ├── SEARCH_RESULT_EFFECTIVENESS_LOW.jsonl +│ │ └── SEARCH_RESULT_AUTHORITY_LOW.jsonl +│ └── result_level/ +│ ├── Relevance/ +│ │ └── Error_Relevance_Low.jsonl +│ ├── Effectiveness/ +│ │ ├── Error_Effectiveness_Low.jsonl +│ │ └── Error_HTML_Tag.jsonl +│ └── Authority/ +│ ├── Error_Authority_Low.jsonl +│ ├── Error_Citation_Miss.jsonl +│ └── Error_DOI_Miss.jsonl +└── good/ # 仅使用 --save-good 时生成 + ├── query_level/ + └── result_level/ +``` + +- `query_level`：一个 query 的 top-k 聚合得分及其全部结果。 +- `result_level`：每一篇检索文献的原始数据和三个指标明细。 +- 同一 result 可以写入多个原因 label 文件；例如 Authority Low 可能同时进入 Citation Miss 和 DOI Miss。 + +### 9.1 单独跑相关性 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0.7" + +python examples/retrieval/sdk_eval_relevancy.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_relevancy_97q \ + --top-k 10 \ + --threshold 0.15 \ + --llm-max-tokens 1024 \ + --llm-workers 3 \ + --llm-timeout 60 \ + --save-good +``` + +Windows PowerShell 示例： + +```powershell +$env:OPENAI_API_KEY="" +$env:OPENAI_BASE_URL="" +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0.7" + +python examples/retrieval/sdk_eval_relevancy.py ` + --input-jsonl outputs/meta_search_97_query_results.jsonl ` + --output-dir outputs/search_result_relevancy_97q ` + --top-k 10 ` + --threshold 0.15 ` + --llm-max-tokens 1024 ` + --llm-workers 3 ` + --llm-timeout 60 ` + --save-good +``` + +### 9.2 单独跑内容有效性 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0.7" + +python examples/retrieval/sdk_eval_effectiveness.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_effectiveness_97q \ + --top-k 10 \ + --threshold 0.15 \ + --llm-max-tokens 1024 \ + --llm-workers 8 \ + --llm-timeout 60 \ + --save-good +``` + +### 9.3 单独跑权威性 + +```bash +python examples/retrieval/sdk_eval_authority.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_authority_97q \ + --top-k 10 \ + --threshold 0.15 \ + --save-good +``` + +### 9.4 跑综合评分 + +```bash +export OPENAI_API_KEY="" +export OPENAI_BASE_URL="" +export OPENAI_MODEL="deepseek-v4-flash" + +python examples/retrieval/sdk_eval_search_result.py \ + --input-jsonl outputs/meta_search_97_query_results.jsonl \ + --output-dir outputs/search_result_quality_97q \ + --top-k 10 \ + --relevance-threshold 0.15 \ + --effectiveness-threshold 0.15 \ + --authority-threshold 0.15 \ + --overall-threshold 0.15 \ + --llm-max-tokens 1024 \ + --effectiveness-llm-max-tokens 512 \ + --llm-timeout 60 \ + --save-good +``` + +## 10. 阈值解释 + +当前默认统一阈值为： + +```text +0.15 +``` + +该阈值的含义是 hard bad，即只标记严重问题： + +- 相关性几乎不匹配。 +- 内容字段严重缺失或不可读。 +- 权威信号极弱。 + +分析建议： + +| 分数段 | 建议解释 | +|---|---| +| `< 0.15` | hard bad，优先人工排查 | +| `0.15 - 0.30` | 低分边缘，适合抽样检查 | +| `0.30 - 0.45` | 一般质量问题，尤其适合看 metadata 缺失 | +| `>= 0.45` | 通常可接受，但仍需结合业务 query 类型 | + +不同指标的阈值敏感性不同： + +- 相关性 `0.15` 适合筛严重错召回；如果要分析一般低相关，可关注 `< 0.30`。 +- 内容有效性 `0.15` 很宽松；如果要推动元数据补全，可关注 `< 0.45`。 +- 权威性 `0.15` 不宜轻易提高太多，因为很多人名、书籍、访谈、非标准论文结果天然 citation/DOI/venue 信号弱。 + +## 11. 常见分析方法 + +### 11.1 查看 query 级低分 + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Sort-Object {[double]$_.relevance} | + Select-Object -First 20 query,relevance,error_count,label +``` + +### 11.2 查看 result 级有效性 issues + +```powershell +Import-Csv outputs/search_result_effectiveness_97q/result_scores.csv | + Where-Object {$_.issues -ne ""} | + Select-Object query,rank,title,Effectiveness,issues +``` + +### 11.3 查看权威性低分原因 + +```powershell +Import-Csv outputs/search_result_authority_97q/result_scores.csv | + Sort-Object {[double]$_.authority} | + Select-Object -First 20 query,rank,title,authority,citation_score,influential_citation_score,venue_score,doi_score,reason +``` + +### 11.4 区分相关性低分和解析错误 + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Where-Object {[double]$_.relevance -lt 0.15} | + Select-Object query,relevance,error_count,label +``` + +```powershell +Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | + Where-Object {[int]$_.error_count -gt 0} | + Select-Object query,relevance,error_count,label +``` + +## 12. 相关代码位置 + +评测器： + +- `dingo/model/llm/llm_search_result_relevance.py` +- `dingo/model/llm/llm_search_result_effectiveness.py` +- `dingo/model/llm/llm_search_result_authority.py` + +脚本： + +- `examples/retrieval/sdk_eval_relevancy.py` +- `examples/retrieval/sdk_eval_effectiveness.py` +- `examples/retrieval/sdk_eval_authority.py` +- `examples/retrieval/sdk_eval_search_result.py` +- `examples/retrieval/search_result_eval_utils.py` + +## 13. 已知注意事项 + +1. 相关性使用 LLM，temperature 大于 0 时，同一批数据重跑可能有轻微分数波动。 +2. LLM 相关性结果可能出现 JSON 解析失败，脚本会记录 `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`。 +3. 内容有效性当前不按 `metadata_type` 放宽字段要求，因此 ebook 缺少 abstract、keywords、venue 时会低分。 +4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。 +5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。 + diff --git a/docs/search_result_relevance_executor.md b/docs/search_result_relevance_executor.md new file mode 100644 index 00000000..609cc2eb --- /dev/null +++ b/docs/search_result_relevance_executor.md @@ -0,0 +1,32 @@ +# Search Result Relevance Executor Notes + +`examples/retrieval/sdk_eval_relevancy.py` evaluates each query-result pair with `LLMSearchResultRelevance` through Dingo `LocalExecutor`. + +## Content Issues + +`Relevance.Error_Content_Issues` now uses a strict standard. It is intended for severe visible content corruption, not for ordinary search-result incompleteness. + +The LLM prompt asks `content_issues=true` only when the visible title/content has severe problems such as: + +- mojibake or garbled text +- raw HTML/XML tag residue +- parser residue that materially hurts readability +- invisible or control characters +- unreadable text + +The evaluator also applies a deterministic evidence filter after the LLM response. Even if the LLM returns `content_issues=true`, the final executor label `Relevance.Error_Content_Issues` is emitted only when the result text contains supporting evidence. + +The following should not by itself trigger `Relevance.Error_Content_Issues`: + +- missing abstract +- short snippet +- truncated preview +- title-only result, if the title is still readable enough to judge relevance + +In `EvalDetail.reason`, the output keeps both: + +- `raw_content_issues`: the original LLM boolean +- `content_issues`: the post-filtered boolean used for final labels +- `content_issue_evidence`: the matched evidence list + +This keeps the LLM signal available for analysis while preventing ordinary truncation or sparse metadata from making an otherwise relevant result bad. diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py new file mode 100644 index 00000000..11d6d1ca --- /dev/null +++ b/examples/retrieval/sdk_eval_authority.py @@ -0,0 +1,105 @@ +"""Run search result authority evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from dingo.config import InputArgs +from dingo.exec import Executor + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl + + +EVALUATOR_NAME = "LLMSearchResultAuthority" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result authority with Dingo executor.") + add_common_args(parser) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument("--max-workers", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + return { + "task_name": "search_result_authority", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.max_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": {"threshold": args.threshold}}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_authority_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py new file mode 100644 index 00000000..d7bffe0b --- /dev/null +++ b/examples/retrieval/sdk_eval_effectiveness.py @@ -0,0 +1,127 @@ +"""Run search result effectiveness evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from dingo.config import InputArgs +from dingo.exec import Executor + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl + + +EVALUATOR_NAME = "LLMSearchResultEffectiveness" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result effectiveness with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--llm-max-tokens", type=int, default=512) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument( + "--disable-llm-quality", + action="store_true", + help="Disable LLM second judgment for abnormal-character candidates.", + ) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + llm_config: dict[str, Any] = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.threshold, + "enable_llm_quality": not args.disable_llm_quality, + } + return { + "task_name": "search_result_effectiveness", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": llm_config}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_effectiveness_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + input_data = build_input_data(args, flattened_path) + summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py new file mode 100644 index 00000000..d34f0305 --- /dev/null +++ b/examples/retrieval/sdk_eval_relevancy.py @@ -0,0 +1,126 @@ +"""Run search result relevancy evaluation through Dingo LocalExecutor.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from dingo.config import InputArgs +from dingo.exec import Executor + +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl + + +EVALUATOR_NAME = "LLMSearchResultRelevance" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result relevancy with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed") + parser.add_argument("--expected-criteria", default=None) + parser.add_argument("--llm-max-tokens", type=int, default=1024) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--threshold", type=float, default=0.15) + parser.add_argument( + "--raw-output", + action="store_true", + help="Write executor records with raw data merged into top-level JSONL rows.", + ) + return parser.parse_args() + + +def flatten_query_results(input_path: Path, output_path: Path, *, top_k: int, max_queries: int | None) -> int: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + for rank, result in enumerate(item["results"][:top_k], start=1): + result_payload = dict(result) + result_payload["_eval_query"] = query + row: dict[str, Any] = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result_payload, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count + + +def build_input_data(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + llm_config: dict[str, Any] = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "prompt_mode": args.prompt_mode, + "expected_criteria": args.expected_criteria, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.threshold, + } + return { + "task_name": "search_result_relevancy", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": args.save_good, + "all_labels": True, + "raw": args.raw_output, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [{"name": EVALUATOR_NAME, "config": llm_config}], + } + ], + } + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / "meta_search_flattened_relevancy_input.jsonl" + total = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + summary = Executor.exec_map["local"](InputArgs(**build_input_data(args, flattened_path))).execute() + print(summary) + print(f"[Done] flattened_input={flattened_path.resolve()}") + print(f"[Done] executor_output={summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py new file mode 100644 index 00000000..dc28745e --- /dev/null +++ b/examples/retrieval/sdk_eval_search_result.py @@ -0,0 +1,565 @@ +"""Evaluate search results with relevance, effectiveness, and authority metrics. + +This script uses Dingo LocalExecutor for result-level metric execution, then +aggregates executor outputs back to query-level CSV/JSON reports. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import time +from datetime import datetime +from pathlib import Path +from typing import Any + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from dingo.config import InputArgs +from dingo.exec import Executor +from dingo.model.llm.llm_search_result_relevance import is_doi_query + +from search_result_eval_utils import ( + add_common_args, + get_title, + load_query_result_jsonl, + rank_discounted_mean, + summarize, + write_classified_jsonl, + write_csv, + write_json, +) + + +WEIGHTS = { + "relevance": 0.7, + "effectiveness": 0.2, + "authority": 0.1, +} + +EFFECTIVENESS_LABEL_TO_ISSUE = { + "Effectiveness.Error_Title_Miss": "missing_title", + "Effectiveness.Error_Abstract_Miss": "missing_abstract", + "Effectiveness.Error_Keywords_Miss": "missing_keywords", + "Effectiveness.Error_Venue_Miss": "missing_venue", + "Effectiveness.Error_HTML_Tag": "html_tag", + "Effectiveness.Error_Mojibake": "mojibake", + "Effectiveness.Error_Invisible_Char": "invisible_char", + "Effectiveness.Error_Unreadable_Text": "unreadable_text", + "Effectiveness.Error_Special_Char_Noise": "special_char_noise", + "Effectiveness.Error_LLM_Quality_Parse": "llm_quality_parse_error", + "Effectiveness.Error_Effectiveness_Low": "effectiveness_low", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Evaluate search result quality with Dingo executor.") + add_common_args(parser) + parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) + parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) + parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed") + parser.add_argument("--llm-max-tokens", type=int, default=1024) + parser.add_argument("--llm-timeout", type=float, default=60.0) + parser.add_argument("--llm-workers", type=int, default=4) + parser.add_argument("--batch-size", type=int, default=10) + parser.add_argument("--effectiveness-llm-max-tokens", type=int, default=512) + parser.add_argument( + "--disable-effectiveness-llm-quality", + action="store_true", + help="Disable LLM readability/corruption judgment for effectiveness.", + ) + parser.add_argument("--relevance-threshold", type=float, default=0.15) + parser.add_argument("--effectiveness-threshold", type=float, default=0.15) + parser.add_argument("--authority-threshold", type=float, default=0.15) + parser.add_argument("--overall-threshold", type=float, default=0.15) + parser.add_argument( + "--save-detailed", + action="store_true", + help="Save detailed_results.json with query-level records and embedded result rows.", + ) + return parser.parse_args() + + +def flatten_query_results( + input_path: Path, + output_path: Path, + *, + top_k: int, + max_queries: int | None, +) -> tuple[int, list[str]]: + items = load_query_result_jsonl(input_path, max_queries) + output_path.parent.mkdir(parents=True, exist_ok=True) + count = 0 + empty_queries: list[str] = [] + with output_path.open("w", encoding="utf-8") as f: + for query_index, item in enumerate(items, start=1): + query = item["query"] + if not item["results"][:top_k]: + empty_queries.append(query) + for rank, result in enumerate(item["results"][:top_k], start=1): + result_payload = dict(result) + result_payload["_eval_query"] = query + row = { + "query": query, + "query_index": query_index, + "rank": rank, + "title": get_title(result), + "search_result": result_payload, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + count += 1 + return count, empty_queries + + +def build_executor_input(args: argparse.Namespace, flattened_path: Path) -> dict[str, Any]: + relevance_config = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "prompt_mode": args.prompt_mode, + "max_tokens": args.llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.relevance_threshold, + } + effectiveness_config = { + "model": args.openai_model, + "key": args.openai_api_key, + "api_url": args.openai_base_url, + "temperature": args.openai_temperature, + "max_tokens": args.effectiveness_llm_max_tokens, + "timeout": args.llm_timeout, + "threshold": args.effectiveness_threshold, + "enable_llm_quality": not args.disable_effectiveness_llm_quality, + } + authority_config = {"threshold": args.authority_threshold} + return { + "task_name": "search_result_quality", + "input_path": str(flattened_path), + "output_path": str(args.output_dir), + "dataset": {"source": "local", "format": "jsonl"}, + "executor": { + "max_workers": args.llm_workers, + "batch_size": args.batch_size, + "result_save": { + "bad": True, + "good": True, + "all_labels": True, + "merge": True, + }, + }, + "evaluator": [ + { + "fields": {"search_result": "search_result"}, + "evals": [ + {"name": "LLMSearchResultRelevance", "config": relevance_config}, + {"name": "LLMSearchResultEffectiveness", "config": effectiveness_config}, + {"name": "LLMSearchResultAuthority", "config": authority_config}, + ], + } + ], + } + + +def load_executor_records(executor_output_path: str) -> list[dict[str, Any]]: + all_results_path = Path(executor_output_path) / "all_results.jsonl" + if not all_results_path.exists(): + raise FileNotFoundError(f"Executor merged result file not found: {all_results_path}") + records = [] + with all_results_path.open("r", encoding="utf-8") as f: + for line in f: + if line.strip(): + records.append(json.loads(line)) + return records + + +def get_metric_detail(record: dict[str, Any], metric: str) -> dict[str, Any]: + details = record.get("eval_details", {}).get("search_result", []) + for detail in details: + if detail.get("metric") == metric: + return detail + return {} + + +def first_reason(detail: dict[str, Any]) -> dict[str, Any]: + reason = detail.get("reason") or [] + if reason and isinstance(reason[0], dict): + return reason[0] + return {} + + +def filtered_effectiveness_issues(detail: dict[str, Any]) -> str: + labels = detail.get("label") or [] + issues = [] + for label in labels: + issue = EFFECTIVENESS_LABEL_TO_ISSUE.get(label) + if issue and issue not in issues: + issues.append(issue) + return "|".join(issues) + + +def build_reports( + records: list[dict[str, Any]], + args: argparse.Namespace, + executor_summary, + executor_result_summary: dict[str, Any] | None = None, + empty_queries: list[str] | None = None, +) -> tuple[dict, list[dict], list[dict], list[dict], list[dict]]: + records = sorted( + records, + key=lambda r: ( + int(r.get("raw_data", {}).get("query_index") or 0), + int(r.get("raw_data", {}).get("rank") or 0), + ), + ) + + result_rows = [] + by_query: dict[str, list[dict[str, Any]]] = {} + rank_relevance_error_count = 0 + rank_effectiveness_llm_quality_error_count = 0 + + for record in records: + raw = record.get("raw_data", {}) + query = raw.get("query", "") + relevance_detail = get_metric_detail(record, "LLMSearchResultRelevance") + effectiveness_detail = get_metric_detail(record, "LLMSearchResultEffectiveness") + authority_detail = get_metric_detail(record, "LLMSearchResultAuthority") + relevance_reason = first_reason(relevance_detail) + effectiveness_reason = first_reason(effectiveness_detail) + authority_reason = first_reason(authority_detail) + + relevance = round(float(relevance_detail.get("score") or 0.0), 5) + effectiveness = round(float(effectiveness_detail.get("score") or 0.0), 5) + authority = round(float(authority_detail.get("score") or 0.0), 5) + overall = round( + WEIGHTS["relevance"] * relevance + + WEIGHTS["effectiveness"] * effectiveness + + WEIGHTS["authority"] * authority, + 5, + ) + + relevance_error = str(relevance_reason.get("error") or "") + effectiveness_error = str(effectiveness_reason.get("llm_quality_error") or "") + if relevance_error: + rank_relevance_error_count += 1 + if effectiveness_error: + rank_effectiveness_llm_quality_error_count += 1 + + row = { + "query": query, + "rank": raw.get("rank"), + "title": raw.get("title", ""), + "relevance": relevance, + "query_relevance": round(float(relevance_reason.get("query_relevance") or 0.0), 5), + "result_quality": round(float(relevance_reason.get("result_quality") or 0.0), 5), + "relevance_content_issues": bool(relevance_reason.get("content_issues", False)), + "relevance_content_issue_evidence": "|".join(relevance_reason.get("content_issue_evidence") or []), + "relevance_error": relevance_error, + "relevance_reasoning": relevance_reason.get("reasoning", ""), + "effectiveness": effectiveness, + "effectiveness_issues": filtered_effectiveness_issues(effectiveness_detail), + "effectiveness_llm_quality_reason": effectiveness_reason.get("llm_quality_reason", ""), + "effectiveness_llm_quality_error": effectiveness_error, + "authority": authority, + "citation_score": round(float(authority_reason.get("citation_score") or 0.0), 5), + "influential_citation_score": round(float(authority_reason.get("influential_citation_score") or 0.0), 5), + "venue_score": round(float(authority_reason.get("venue_score") or 0.0), 5), + "doi_score": round(float(authority_reason.get("doi_score") or 0.0), 5), + "authority_reason": authority_reason.get("reason", ""), + "overall": overall, + } + result_rows.append(row) + by_query.setdefault(query, []).append(row) + + query_rows = [] + detailed = [] + classified_records = [] + for query, rows in by_query.items(): + relevance_scores = [float(row["relevance"]) for row in rows] + effectiveness_scores = [float(row["effectiveness"]) for row in rows] + authority_scores = [float(row["authority"]) for row in rows] + if is_doi_query(query): + query_relevance = round( + max( + ( + float(row["relevance"]) + / math.log2(max(1, int(row.get("rank") or index)) + 1) + ) + for index, row in enumerate(rows, start=1) + ), + 5, + ) + relevance_aggregation = "doi_exact_match_rank_discount" + else: + query_relevance = round(rank_discounted_mean(relevance_scores), 5) + relevance_aggregation = "rank_discounted_mean" + query_effectiveness = round(rank_discounted_mean(effectiveness_scores), 5) + query_authority = round(rank_discounted_mean(authority_scores), 5) + query_overall = round( + WEIGHTS["relevance"] * query_relevance + + WEIGHTS["effectiveness"] * query_effectiveness + + WEIGHTS["authority"] * query_authority, + 5, + ) + + labels = [] + if query_overall < args.overall_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW") + if query_relevance < args.relevance_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW") + relevance_errors = sum(1 for row in rows if row["relevance_error"]) + if relevance_errors: + labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR") + if query_effectiveness < args.effectiveness_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW") + if query_authority < args.authority_threshold: + labels.append("QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW") + + eval_status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS"] + + query_row = { + "query": query, + "result_count": len(rows), + "valid_relevance_count": len(rows) - relevance_errors, + "relevance_error_count": relevance_errors, + "relevance": query_relevance, + "relevance_aggregation": relevance_aggregation, + "effectiveness": query_effectiveness, + "authority": query_authority, + "overall": query_overall, + "eval_status": eval_status, + "label": "|".join(labels), + } + query_rows.append(query_row) + detail = {**query_row, "results": rows} + detailed.append(detail) + classified_records.append({ + "query": query, + "metric": "search_result_quality", + "score": query_overall, + "threshold": args.overall_threshold, + "eval_status": eval_status, + "labels": labels, + "relevance": query_relevance, + "relevance_aggregation": relevance_aggregation, + "effectiveness": query_effectiveness, + "authority": query_authority, + "relevance_error_count": relevance_errors, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "results": rows, + }) + + for query in empty_queries or []: + labels = ["QUALITY_BAD.SEARCH_RESULT_EMPTY"] + query_row = { + "query": query, + "result_count": 0, + "valid_relevance_count": 0, + "relevance_error_count": 0, + "relevance": 0.0, + "relevance_aggregation": ( + "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" + ), + "effectiveness": 0.0, + "authority": 0.0, + "overall": 0.0, + "eval_status": True, + "label": labels[0], + } + query_rows.append(query_row) + detailed.append({**query_row, "results": []}) + classified_records.append({ + "query": query, + "metric": "search_result_quality", + "score": 0.0, + "threshold": args.overall_threshold, + "eval_status": True, + "labels": labels, + "relevance": 0.0, + "relevance_aggregation": ( + "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" + ), + "effectiveness": 0.0, + "authority": 0.0, + "relevance_error_count": 0, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "results": [], + }) + + summary = { + "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "metric": "search_result_quality", + "top_k": args.top_k, + "weights": WEIGHTS, + "thresholds": { + "relevance": args.relevance_threshold, + "effectiveness": args.effectiveness_threshold, + "authority": args.authority_threshold, + "overall": args.overall_threshold, + }, + "llm": { + "model": args.openai_model, + "prompt_mode": args.prompt_mode, + "max_tokens": args.llm_max_tokens, + "temperature": args.openai_temperature, + "timeout": args.llm_timeout, + "workers": args.llm_workers, + "effectiveness_llm_quality_enabled": not args.disable_effectiveness_llm_quality, + "effectiveness_llm_max_tokens": args.effectiveness_llm_max_tokens, + }, + "run_output_path": str(executor_summary.output_path), + "metrics": { + "relevance": summarize([float(row["relevance"]) for row in query_rows]), + "effectiveness": summarize([float(row["effectiveness"]) for row in query_rows]), + "authority": summarize([float(row["authority"]) for row in query_rows]), + "overall": summarize([float(row["overall"]) for row in query_rows]), + }, + "query_count": len(query_rows), + "result_count": len(result_rows), + "rank_relevance_error_count": rank_relevance_error_count, + "rank_effectiveness_llm_quality_error_count": rank_effectiveness_llm_quality_error_count, + "num_bad": sum(1 for row in query_rows if row["eval_status"]), + "num_good": sum(1 for row in query_rows if not row["eval_status"]), + } + if executor_result_summary: + summary["result_level"] = { + "score": executor_result_summary.get("score"), + "num_good": executor_result_summary.get("num_good"), + "num_bad": executor_result_summary.get("num_bad"), + "total": executor_result_summary.get("total"), + "type_ratio": executor_result_summary.get("type_ratio", {}), + "metrics_score": executor_result_summary.get("metrics_score", {}), + } + return summary, query_rows, result_rows, detailed, classified_records + + +def normalize_executor_summary(executor_output_path: str, records: list[dict[str, Any]]) -> dict[str, Any] | None: + """Make executor summary label ratios use record-level quality semantics. + + LocalExecutor counts every EvalDetail label. In this combined script one + result has three metrics, so a bad result can still contain QUALITY_GOOD + from the metrics that passed. This rewrite keeps error labels as + per-result occurrence rates, but makes QUALITY_GOOD mutually exclusive. + """ + total = len(records) + if total == 0: + return None + + field_key = "search_result" + counts: dict[str, int] = {} + for record in records: + labels = set() + for detail in record.get("eval_details", {}).get(field_key, []): + for label in detail.get("label") or []: + labels.add(label) + + if record.get("eval_status"): + labels.discard("QUALITY_GOOD") + else: + labels = {"QUALITY_GOOD"} + + for label in labels: + counts[label] = counts.get(label, 0) + 1 + + summary_path = Path(executor_output_path) / "summary.json" + if not summary_path.exists(): + return None + + summary = json.loads(summary_path.read_text(encoding="utf-8-sig")) + summary.setdefault("type_ratio", {})[field_key] = { + label: round(count / total, 6) + for label, count in sorted(counts.items()) + } + return summary + + +def build_result_classified_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Expose executor result labels for result-level directory output.""" + classified: list[dict[str, Any]] = [] + for record in records: + labels: list[str] = [] + for details in record.get("eval_details", {}).values(): + for detail in details: + for label in detail.get("label") or []: + if label != "QUALITY_GOOD" and label not in labels: + labels.append(label) + + eval_status = bool(labels) + if not labels: + labels = ["QUALITY_GOOD"] + classified.append({**record, "eval_status": eval_status, "labels": labels}) + return classified + + +def main() -> None: + args = parse_args() + os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") + args.output_dir.mkdir(parents=True, exist_ok=True) + flattened_path = args.output_dir / f".search_result_quality_input_{int(time.time())}_{os.getpid()}.jsonl" + try: + total, empty_queries = flatten_query_results( + args.input_jsonl, + flattened_path, + top_k=args.top_k, + max_queries=args.max_queries, + ) + if total == 0: + raise ValueError("No search results found to evaluate.") + + input_data = build_executor_input(args, flattened_path) + executor_summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() + run_dir = Path(executor_summary.output_path) + executor_records = load_executor_records(executor_summary.output_path) + executor_result_summary = normalize_executor_summary(executor_summary.output_path, executor_records) + summary, query_rows, result_rows, detailed, classified_records = build_reports( + executor_records, + args, + executor_summary, + executor_result_summary, + empty_queries, + ) + + write_json(run_dir / "summary.json", summary) + if args.save_detailed: + write_json(run_dir / "detailed_results.json", {"summary": summary, "queries": detailed}) + write_csv(run_dir / "query_scores.csv", query_rows) + write_csv(run_dir / "result_scores.csv", result_rows) + write_classified_jsonl( + run_dir, + classified_records, + save_good=args.save_good, + level="query_level", + ) + write_classified_jsonl( + run_dir, + build_result_classified_records(executor_records), + save_good=args.save_good, + level="result_level", + ) + print(f"Saved to {run_dir.resolve()}") + finally: + if flattened_path.exists(): + flattened_path.unlink() + + +if __name__ == "__main__": + main() diff --git a/examples/retrieval/search_result_eval_utils.py b/examples/retrieval/search_result_eval_utils.py new file mode 100644 index 00000000..14cde206 --- /dev/null +++ b/examples/retrieval/search_result_eval_utils.py @@ -0,0 +1,118 @@ +"""Utilities for search result JSONL evaluation examples.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import statistics +from pathlib import Path +from typing import Any + + +def load_query_result_jsonl(path: Path, max_queries: int | None = None) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + with path.open("r", encoding="utf-8-sig") as f: + for line_no, line in enumerate(f, start=1): + if not line.strip(): + continue + item = json.loads(line) + query = item.get("query") or item.get("query_text") or item.get("q") or "" + results = ( + item.get("results") + or item.get("top_results") + or item.get("top_api_results") + or item.get("search_results") + or [] + ) + if isinstance(results, dict): + results = results.get("results") or results.get("items") or [] + if not isinstance(results, list): + raise ValueError(f"Line {line_no}: results must be a list.") + if not query: + raise ValueError(f"Line {line_no}: missing query/query_text.") + items.append({**item, "query": str(query), "results": results}) + if max_queries and len(items) >= max_queries: + break + return items + + +def get_title(result: dict[str, Any]) -> str: + return str(result.get("title") or result.get("display_name") or "") + + +def rank_discounted_mean(values: list[float]) -> float: + if not values: + return 0.0 + weights = [1.0 / math.log2(rank + 2) for rank in range(len(values))] + return sum(value * weight for value, weight in zip(values, weights)) / sum(weights) + + +def summarize(values: list[float]) -> dict[str, float | int]: + if not values: + return {"count": 0, "mean": 0.0, "median": 0.0, "min": 0.0, "max": 0.0} + return { + "count": len(values), + "mean": round(statistics.mean(values), 5), + "median": round(statistics.median(values), 5), + "min": round(min(values), 5), + "max": round(max(values), 5), + } + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--input-jsonl", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, default=Path("outputs/search_result_eval")) + parser.add_argument("--top-k", type=int, default=10) + parser.add_argument("--max-queries", type=int, default=None) + parser.add_argument("--save-good", action="store_true", help="Save good query-level records under good/.") + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if not rows: + path.write_text("", encoding="utf-8-sig") + return + fieldnames: list[str] = [] + for row in rows: + for key in row: + if key not in fieldnames: + fieldnames.append(key) + with path.open("w", encoding="utf-8-sig", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def write_classified_jsonl( + output_dir: Path, + records: list[dict[str, Any]], + *, + save_good: bool = False, + level: str | None = None, +) -> None: + for record in records: + status = "bad" if record.get("eval_status") else "good" + if status == "good" and not save_good: + continue + labels = record.get("labels") or [] + if not labels: + labels = ["QUALITY_GOOD.PASS"] if status == "good" else ["QUALITY_BAD.UNKNOWN"] + for label in labels: + parts = str(label).split(".") + status_dir = output_dir / status + if level: + status_dir = status_dir / level + if len(parts) > 1: + path = status_dir / Path(*parts[:-1]) / f"{parts[-1]}.jsonl" + else: + path = status_dir / f"{parts[0]}.jsonl" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py new file mode 100644 index 00000000..9e6c61eb --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -0,0 +1,51 @@ +from dingo.model.llm.llm_search_result_effectiveness import ( + LLMSearchResultEffectiveness, + _filter_llm_field_issues, + _issues_to_labels, + _looks_like_utf8_latin1_mojibake, + _rule_abnormal_char_issues, +) + + +def _mojibake(value: str) -> str: + return value.encode("utf-8").decode("latin-1") + + +def test_detects_utf8_cyrillic_decoded_as_latin1(): + broken = _mojibake("Развитие научных исследований") + + assert _looks_like_utf8_latin1_mojibake(broken) + assert "RuleMojibake" in _rule_abnormal_char_issues(broken) + assert _filter_llm_field_issues("title", broken, ["title:mojibake"]) == ["title:mojibake"] + assert _issues_to_labels(["RuleMojibake", "title:mojibake"]) == ["Effectiveness.Error_Mojibake"] + + +def test_does_not_flag_valid_latin_or_cyrillic_text(): + assert not _looks_like_utf8_latin1_mojibake("Ð is a valid Icelandic letter") + assert not _looks_like_utf8_latin1_mojibake("Развитие научных исследований") + + +def test_detects_mojibake_fragment_in_mixed_language_text(): + broken = "中文标题 | " + _mojibake("Научные исследования") + + assert _looks_like_utf8_latin1_mojibake(broken) + + +def test_rule_only_grade_penalizes_mojibake_fields(): + broken_title = _mojibake("Развитие научных исследований") + broken_abstract = _mojibake( + "В этой статье рассматриваются современные научные исследования и методы анализа данных. " * 4 + ) + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + + grade = grader.grade( + title=broken_title, + abstract=broken_abstract, + keywords=["research", "analysis", "data"], + venue="Science Journal", + ) + + assert "RuleMojibake" in grade.issues + assert grade.title_score == 0.1 + assert grade.abstract_score == 0.1 + assert grade.score < 0.5 diff --git a/test/scripts/model/llm/test_llm_search_result_relevance.py b/test/scripts/model/llm/test_llm_search_result_relevance.py new file mode 100644 index 00000000..ffab09e8 --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_relevance.py @@ -0,0 +1,47 @@ +from dingo.model.llm.llm_search_result_relevance import ( + _extract_result_dois, + _grade_doi_result, + _normalize_doi, + is_doi_query, +) + + +def test_normalize_doi_variants(): + assert _normalize_doi("10.1016/j.ijbiomac.2025.143529") == "10.1016/j.ijbiomac.2025.143529" + assert _normalize_doi("https://doi.org/10.1038/NCOMMS7112") == "10.1038/ncomms7112" + assert _normalize_doi(":10.1111/jipb.70096") == "10.1111/jipb.70096" + assert _normalize_doi("PBPK review") == "" + assert is_doi_query("10.1016/j.ijbiomac.2025.143529") + assert not is_doi_query("PBPK review") + + +def test_extract_result_dois_from_supported_fields(): + result = { + "doi": "https://doi.org/10.1000/ABC", + "unique_id": "paper:10.2000/xyz", + "locations": [{"url": "https://doi.org/10.3000/location"}], + } + assert _extract_result_dois(result) == [ + "10.1000/abc", + "10.2000/xyz", + "10.3000/location", + ] + + +def test_doi_query_uses_exact_match(): + matched = _grade_doi_result( + "10.1016/j.ijbiomac.2025.143529", + {"doi": "https://doi.org/10.1016/j.ijbiomac.2025.143529"}, + ) + mismatched = _grade_doi_result( + "10.1016/j.ijbiomac.2025.143529", + {"doi": "https://doi.org/10.3390/plants14152362"}, + ) + + assert matched is not None and matched.score == 1.0 + assert mismatched is not None and mismatched.score == 0.0 + assert "DOI mismatch" in mismatched.reasoning + + +def test_non_doi_query_falls_back_to_llm(): + assert _grade_doi_result("PBPK相关综述", {"doi": "10.1000/test"}) is None diff --git a/test/scripts/retrieval/test_open_eval.py b/test/scripts/retrieval/test_open_eval.py index 45bd507f..987e9264 100644 --- a/test/scripts/retrieval/test_open_eval.py +++ b/test/scripts/retrieval/test_open_eval.py @@ -82,6 +82,26 @@ def test_invalid_json(self): assert grade.error assert "JSON parse failed" in grade.error + def test_json_embedded_in_text(self): + response = 'Here is the grade:\n{"score": 0.6, "query_relevance": 0.7, "result_quality": 0.5, "content_issues": false, "confidence": 0.8, "reasoning": "ok"}' + grade = _parse_grade_response(response) + assert grade.score == 0.6 + assert grade.error == "" + + def test_lenient_parse_unescaped_quotes_in_reasoning(self): + response = ( + '{"reasoning": "The query "resilience" directly matches the result", ' + '"query_relevance": 0.95, "result_quality": 0.9, ' + '"content_issues": false, "confidence": 0.85, "score": 0.92}' + ) + grade = _parse_grade_response(response) + assert grade.error == "" + assert grade.score == 0.92 + assert grade.query_relevance == 0.95 + assert grade.result_quality == 0.9 + assert grade.content_issues is False + assert grade.confidence == 0.85 + def test_missing_fields_default_to_zero(self): grade = _parse_grade_response('{"score": 0.5}') assert grade.score == 0.5 From da45cae0b235d137a694d9abb4a3be4807fe4c64 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 14 Jul 2026 03:34:00 +0000 Subject: [PATCH 09/80] =?UTF-8?q?=F0=9F=8E=A8=20Auto-format=20code=20with?= =?UTF-8?q?=20pre-commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/llm/llm_search_result_authority.py | 2 -- .../model/llm/llm_search_result_effectiveness.py | 2 -- docs/search_result_effectiveness_executor.md | 1 - docs/search_result_quality_metrics.md | 1 - examples/retrieval/sdk_eval_authority.py | 6 ++---- examples/retrieval/sdk_eval_effectiveness.py | 6 ++---- examples/retrieval/sdk_eval_relevancy.py | 6 ++---- examples/retrieval/sdk_eval_search_result.py | 15 ++------------- examples/retrieval/search_result_eval_utils.py | 1 - .../llm/test_llm_search_result_effectiveness.py | 8 +------- .../model/llm/test_llm_search_result_relevance.py | 7 +------ 11 files changed, 10 insertions(+), 45 deletions(-) diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py index cfd42779..87650b49 100644 --- a/dingo/model/llm/llm_search_result_authority.py +++ b/dingo/model/llm/llm_search_result_authority.py @@ -1,7 +1,6 @@ """Rule-based search result authority grader.""" from __future__ import annotations - import json import math import statistics @@ -13,7 +12,6 @@ from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model - HIGH_AUTHORITY_VENUE_HINTS = ( "nature", "science", diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index 310f6eb3..3b1c78bf 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -11,7 +11,6 @@ """ from __future__ import annotations - import json import logging import re @@ -24,7 +23,6 @@ from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model - logger = logging.getLogger(__name__) diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md index d4031078..e4dd1091 100644 --- a/docs/search_result_effectiveness_executor.md +++ b/docs/search_result_effectiveness_executor.md @@ -128,4 +128,3 @@ Effectiveness = ``` Field scores are based on missing-field checks, length/information-density checks, and abnormal-character handling. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. - diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index f45418bd..6c9da65f 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -638,4 +638,3 @@ Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | 3. 内容有效性当前不按 `metadata_type` 放宽字段要求，因此 ebook 缺少 abstract、keywords、venue 时会低分。 4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。 5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。 - diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py index 11d6d1ca..d0327565 100644 --- a/examples/retrieval/sdk_eval_authority.py +++ b/examples/retrieval/sdk_eval_authority.py @@ -1,7 +1,6 @@ """Run search result authority evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from dingo.config import InputArgs +from dingo.exec import Executor EVALUATOR_NAME = "LLMSearchResultAuthority" diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py index d7bffe0b..a57c3a15 100644 --- a/examples/retrieval/sdk_eval_effectiveness.py +++ b/examples/retrieval/sdk_eval_effectiveness.py @@ -1,7 +1,6 @@ """Run search result effectiveness evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from dingo.config import InputArgs +from dingo.exec import Executor EVALUATOR_NAME = "LLMSearchResultEffectiveness" diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py index d34f0305..2956945b 100644 --- a/examples/retrieval/sdk_eval_relevancy.py +++ b/examples/retrieval/sdk_eval_relevancy.py @@ -1,7 +1,6 @@ """Run search result relevancy evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from dingo.config import InputArgs +from dingo.exec import Executor EVALUATOR_NAME = "LLMSearchResultRelevance" diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py index dc28745e..f4403a1e 100644 --- a/examples/retrieval/sdk_eval_search_result.py +++ b/examples/retrieval/sdk_eval_search_result.py @@ -5,7 +5,6 @@ """ from __future__ import annotations - import argparse import json import math @@ -20,22 +19,12 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json + from dingo.config import InputArgs from dingo.exec import Executor from dingo.model.llm.llm_search_result_relevance import is_doi_query -from search_result_eval_utils import ( - add_common_args, - get_title, - load_query_result_jsonl, - rank_discounted_mean, - summarize, - write_classified_jsonl, - write_csv, - write_json, -) - - WEIGHTS = { "relevance": 0.7, "effectiveness": 0.2, diff --git a/examples/retrieval/search_result_eval_utils.py b/examples/retrieval/search_result_eval_utils.py index 14cde206..d4189d4f 100644 --- a/examples/retrieval/search_result_eval_utils.py +++ b/examples/retrieval/search_result_eval_utils.py @@ -1,7 +1,6 @@ """Utilities for search result JSONL evaluation examples.""" from __future__ import annotations - import argparse import csv import json diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py index 9e6c61eb..d289b1d6 100644 --- a/test/scripts/model/llm/test_llm_search_result_effectiveness.py +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -1,10 +1,4 @@ -from dingo.model.llm.llm_search_result_effectiveness import ( - LLMSearchResultEffectiveness, - _filter_llm_field_issues, - _issues_to_labels, - _looks_like_utf8_latin1_mojibake, - _rule_abnormal_char_issues, -) +from dingo.model.llm.llm_search_result_effectiveness import LLMSearchResultEffectiveness, _filter_llm_field_issues, _issues_to_labels, _looks_like_utf8_latin1_mojibake, _rule_abnormal_char_issues def _mojibake(value: str) -> str: diff --git a/test/scripts/model/llm/test_llm_search_result_relevance.py b/test/scripts/model/llm/test_llm_search_result_relevance.py index ffab09e8..7e75ec10 100644 --- a/test/scripts/model/llm/test_llm_search_result_relevance.py +++ b/test/scripts/model/llm/test_llm_search_result_relevance.py @@ -1,9 +1,4 @@ -from dingo.model.llm.llm_search_result_relevance import ( - _extract_result_dois, - _grade_doi_result, - _normalize_doi, - is_doi_query, -) +from dingo.model.llm.llm_search_result_relevance import _extract_result_dois, _grade_doi_result, _normalize_doi, is_doi_query def test_normalize_doi_variants(): From fd889c1a44c04ca4f756b1bc6c801fa70fcebc12 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Tue, 14 Jul 2026 11:40:32 +0800 Subject: [PATCH 10/80] feat(retrieval): add search result quality evaluation - add relevance, effectiveness, and authority evaluators - add standalone and combined executor-based evaluation scripts - support query-level and result-level classified outputs - improve LLM response parsing and content issue detection - set overall weights to 0.7/0.2/0.1 - add evaluator tests and usage documentation --- docs/search_result_authority_executor.md | 13 +++++++++ docs/search_result_effectiveness_executor.md | 16 +++++++++++ docs/search_result_quality_metrics.md | 28 ++++++++++++++++++++ docs/search_result_relevance_executor.md | 22 +++++++++++++++ test/data/test_search_result.jsonl | 3 +++ 5 files changed, 82 insertions(+) create mode 100644 test/data/test_search_result.jsonl diff --git a/docs/search_result_authority_executor.md b/docs/search_result_authority_executor.md index 17f3a6d0..9b666a95 100644 --- a/docs/search_result_authority_executor.md +++ b/docs/search_result_authority_executor.md @@ -29,6 +29,19 @@ Each flattened input row contains: | `title` | Result title or display name | | `search_result` | Full result payload passed to evaluator | +## Test Data + +The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`). Authority evaluation is rule-based, so this smoke test does not require an LLM API: + +```powershell +python examples/retrieval/sdk_eval_authority.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_authority_smoke ` + --top-k 3 ` + --threshold 0.15 ` + --save-good +``` + ## Commands Run authority evaluation: diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md index d4031078..b3db34b9 100644 --- a/docs/search_result_effectiveness_executor.md +++ b/docs/search_result_effectiveness_executor.md @@ -27,6 +27,22 @@ Each flattened input row contains: | `title` | Result title or display name | | `search_result` | Full result payload passed to evaluator | +## Test Data + +The repository includes `test/data/test_search_result.jsonl` with three queries and nine results. It covers normal papers (`BiMLP`), search-highlight HTML (`海带`), and sparse ebook metadata (`pam`). + +Smoke test with LLM second judgment enabled: + +```powershell +python examples/retrieval/sdk_eval_effectiveness.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_effectiveness_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --threshold 0.15 ` + --save-good +``` + ## Commands Fast rule-only run: diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index f45418bd..0e8828c2 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -428,6 +428,34 @@ overall = 以下命令均在项目根目录执行。 +### 内置测试数据 + +仓库提供了 `test/data/test_search_result.jsonl`，用于在提交代码前快速验证单指标和综合评测流程。该文件包含3条 query、9条 result： + +| Query | 场景 | +|---|---| +| `BiMLP` | 正常学术论文结果 | +| `海带` | 元数据中包含检索高亮 HTML | +| `pam` | 字段稀疏的 ebook 结果 | + +综合 smoke test： + +```powershell +python examples/retrieval/sdk_eval_search_result.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_quality_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --effectiveness-llm-max-tokens 1024 ` + --relevance-threshold 0.15 ` + --effectiveness-threshold 0.15 ` + --authority-threshold 0.15 ` + --overall-threshold 0.15 ` + --save-good +``` + +相关性、有效性和综合 smoke test 需要预先设置 OpenAI-compatible 环境变量；权威性是纯规则评测，不需要 LLM API。 + 综合脚本按评测对象拆分分类目录： ```text diff --git a/docs/search_result_relevance_executor.md b/docs/search_result_relevance_executor.md index 609cc2eb..949b08b4 100644 --- a/docs/search_result_relevance_executor.md +++ b/docs/search_result_relevance_executor.md @@ -30,3 +30,25 @@ In `EvalDetail.reason`, the output keeps both: - `content_issue_evidence`: the matched evidence list This keeps the LLM signal available for analysis while preventing ordinary truncation or sparse metadata from making an otherwise relevant result bad. + +## Test Data + +The repository includes `test/data/test_search_result.jsonl` for local smoke tests. It contains three queries and nine results: + +| Query | Covered scenario | +|---|---| +| `BiMLP` | Normal academic paper results | +| `海带` | Search-highlight HTML in result metadata | +| `pam` | Sparse ebook results | + +With the OpenAI-compatible environment variables configured, run: + +```powershell +python examples/retrieval/sdk_eval_relevancy.py ` + --input-jsonl test/data/test_search_result.jsonl ` + --output-dir outputs/search_result_relevancy_smoke ` + --top-k 3 ` + --llm-max-tokens 1024 ` + --threshold 0.15 ` + --save-good +``` diff --git a/test/data/test_search_result.jsonl b/test/data/test_search_result.jsonl new file mode 100644 index 00000000..c39ad0e3 --- /dev/null +++ b/test/data/test_search_result.jsonl @@ -0,0 +1,3 @@ +{"query":"BiMLP","results":[{"type":["preprint","article"],"publication_venue_type":"repository","access_license":"nonexclusive-distrib","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","cited_by_percentile_year":{},"classifications":{"arxiv_category":["cs.CV"]},"citation_count":9.0,"influential_citation_count":0.0,"abstract":"This paper studies the problem of designing compact binary architectures for vision multi-layer perceptrons (MLPs). We provide extensive analysis on the difficulty of binarizing vision MLPs and find that previous binarization methods perform poorly due to limited capacity of binary MLPs. In contrast with the traditional CNNs that utilizing convolutional operations with large kernel size, fully-connected (FC) layers in MLPs can be treated as convolutional layers with kernel size $1\\times1$. Thus, the representation ability of the FC layers will be limited when being binarized, and places restrictions on the capability of spatial mixing and channel mixing on the intermediate features. To this end, we propose to improve the performance of binary MLP (BiMLP) model by enriching the representation ability of binary FC layers. We design a novel binary block that contains multiple branches to merge a series of outputs from the same stage, and also a universal shortcut connection that encourages the information flow from the previous stage. The downsampling layers are also carefully designed to reduce the computational complexity while maintaining the classification performance. Experimental results on benchmark dataset ImageNet-1k demonstrate the effectiveness of the proposed BiMLP models, which achieve state-of-the-art accuracy compared to prior binary CNNs. The MindSpore code is available at \\url{https://gitee.com/mindspore/models/tree/master/research/cv/BiMLP}.","publication_publisher":[],"access_oa_status":"green","relevance_score":26.327469,"is_content_accessible":true,"publication_venue_name_unified":"arXiv (Cornell University)","locations":[{"type":"","is_oa":"true","url":"http://arxiv.org/abs/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158","license":""},{"type":"","is_oa":"true","url":"https://doi.org/10.48550/arxiv.2212.14158","license":""},{"type":"","is_oa":"unknown","url":"https://arxiv.org/abs/2212.14158","license":""},{"type":"download","is_oa":"true","url":"https://arxiv.org/pdf/2212.14158v1","license":"nonexclusive-distrib"},{"type":"download","is_oa":"true","url":"https://arxiv.org/src/2212.14158v1","license":"nonexclusive-distrib"}],"publication_published_year":2022.0,"keywords":["Computer science","Perceptron","Binary number","Benchmark (surveying)","Binary tree","Layer (electronics)","Artificial intelligence","Convolutional neural network","Kernel (algebra)","Merge (version control)","Pattern recognition (psychology)","Algorithm","Parallel computing","Artificial neural network","Mathematics"],"unique_id":"paper:10.48550/arxiv.2212.14158","doi":"10.48550/arxiv.2212.14158","access_is_oa":"true","doc_id":"bfc49e21253ac0f4278e66c0c59b0bcfe99cc8b30f1b5f1cde36d0e85b957b33","language":"en","citation_normalized_percentile":{},"access_oa_url":["https://arxiv.org/pdf/2212.14158","https://arxiv.org/pdf/2212.14158v1"],"publication_venue_issn":[],"reference_count":55.0,"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","author":[{"orcid":"https://orcid.org/0000-0002-9704-4194","name":"Yixing Xu"},{"orcid":"https://orcid.org/0000-0002-2102-8235","name":"Xinghao Chen"},{"orcid":"https://orcid.org/0000-0002-0013-4530","name":"Yunhe Wang"}]},{"type":["article"],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":[],"access_oa_status":"closed","relevance_score":24.286182,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[{"type":"","is_oa":"false","url":"https://doi.org/10.52202/068431-0367","license":""}],"publication_published_year":2022.0,"keywords":["Perceptron","Binary number","Binary data","Pattern recognition (psychology)","Artificial neural network","Feature (linguistics)"],"unique_id":"paper:10.52202/068431-0367","doi":"10.52202/068431-0367","access_is_oa":"false","language":"","citation_normalized_percentile":{"value":0.33077007,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":[],"publication_venue_issn":[],"title":"BiMLP: Compact Binary Architectures for Vision Multi-Layer Perceptrons","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Xinghao Chen"},{"orcid":"","name":"Yunhe Wang"},{"orcid":"","name":"Yixing Xu"}]},{"type":["article"],"publication_venue_type":"journal","access_license":"cc-by","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"Abstract. Satellite radar altimetry has significantly enhanced inland water monitoring by providing consistent, long-term lake-level observations. However, these datasets often contain substantial gaps caused by sensor malfunctions, orbital limitations, or retrieval errors, complicating hydrological analyses and downstream applications. This study introduces a robust benchmarking framework designed to systematically evaluate gap-filling techniques in satellite-derived lake water-level records through controlled pseudo-gap injection experiments. We investigated three large lakes with distinct hydrological behaviours, the Caspian Sea, Lake Superior, and Lake Tanganyika each representing unique temporal dynamics. Synthetic seven-year gaps (2002–2008) were artificially introduced into complete altimetry datasets, and three sophisticated gap-filling methods were compared: Singular Spectrum Analysis (SSA), Bidirectional Autoregressive (BIAR) models, and Bidirectional Multi-Layer Perceptrons (BiMLP). Method performance was assessed using standard metrics (RMSE, MAE, Bias, and R2), alongside statistical properties including variance, skewness, autocorrelation, and stationarity. BiMLP consistently delivered the highest accuracy across all study lakes, demonstrating exceptional adaptability to both smooth and highly variable signals. SSA performed effectively for lakes exhibiting quasi-periodic behavior, while BIAR showed sensitivity to lag selection and reduced performance under non-stationary conditions. These results emphasize the critical role of bidirectional modeling approaches and rigorous time-series diagnostics in selecting appropriate gap-filling methods for satellite altimetry-based hydrological studies. These findings provide practical guidance for selecting appropriate gap-filling strategies under long data outages in satellite altimetry workflows.","publication_publisher":["Copernicus Publications"],"access_oa_status":"diamond","relevance_score":8.976683,"is_content_accessible":false,"publication_venue_name_unified":"ISPRS annals of the photogrammetry, remote sensing and spatial information sciences","locations":[{"type":"","is_oa":"true","url":"https://doi.org/10.5194/isprs-annals-x-4-w8-2025-557-2026","license":"cc-by"},{"type":"","is_oa":"true","url":"https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf","license":"cc-by"},{"type":"","is_oa":"true","url":"https://doaj.org/article/72149efbb5e740b4a046e48e9a2acb4d","license":"cc-by-sa"}],"publication_published_year":2026.0,"keywords":["Benchmarking","Satellite","Sensitivity (control systems)","Adaptability","Water level","Altimeter","Autoregressive model","Satellite altimetry"],"unique_id":"paper:10.5194/isprs-annals-x-4-w8-2025-557-2026","doi":"10.5194/isprs-annals-x-4-w8-2025-557-2026","access_is_oa":"true","language":"en","citation_normalized_percentile":{"value":0.77162252,"is_in_top_10_percent":"false","is_in_top_1_percent":"false"},"access_oa_url":["https://isprs-annals.copernicus.org/articles/X-4-W8-2025/557/2026/isprs-annals-X-4-W8-2025-557-2026.pdf"],"publication_venue_issn":["2194-9042","2194-9050","2196-6346"],"title":"Benchmarking Gap-Filling Techniques in Satellite Altimetry-Based Lake Water-Level Time Series","reference_count":0.0,"fwci":0.0,"author":[{"orcid":"","name":"Ali MusaHoseini"},{"orcid":"https://orcid.org/0000-0002-0534-0632","name":"Saeed Farzaneh"},{"orcid":"https://orcid.org/0000-0003-3055-041X","name":"Ehsan Forootan"}]}]} +{"query":"海带","results":[{"type":[],"publication_venue_type":"","access_license":"","metadata_type":"paper","isbn13":"","access_xinghe_repository_sha256":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","cited_by_percentile_year":{},"classifications":{},"citation_count":0.0,"abstract":"每次幼儿园吃海带,孩子们都不喜欢,端回的餐盘里都会剩下很多海带.鉴于老师说过“挑食不是好孩子”,所以他们不吃的借口也千奇百怪“老师,我已经吃饱了!”“老师我实在吃不下了!”“老师,我觉得这里面是青色的虫子,所以我……”哎!怎么做才能改善这种情况呢?","publication_publisher":[],"access_oa_status":"","relevance_score":32.074493,"is_content_accessible":true,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e教育\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e实践\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e与\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e研究\u003c/span\u003e | Educational Practice \u0026 Research","locations":[],"publication_published_year":2015.0,"keywords":["凉拌海带丝","《海带的传说》","海带","挑食"],"unique_id":"paper:10.3969/j.issn.1009-010x.2015.01.028","doi":"10.3969/j.issn.1009-010x.2015.01.028","access_is_oa":"unknown","doc_id":"8b1c2021d710f1abf3a83f6dc28fabc6c5b19b57a8447286e232871113a23604","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1009-010X"],"reference_count":0.0,"title":"海带的传说","author":[{"orcid":"","name":"卢瑞云"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"宝宝,今天邀请爸爸妈妈玩\"海带拳\"游戏吧!宝宝当出拳人,妈妈当挑战者,两人一起喊:\"海带,海带!\"同时双手握拳,双臂上下左右挥舞.如果妈妈手臂方向和宝宝不一样,游戏继续;如果一样,挑战失败,换爸爸挑战宝宝.","publication_publisher":[],"access_oa_status":"","relevance_score":29.815878,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e动漫\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e界\u003c/span\u003e | Dong Man Jie","locations":[],"publication_published_year":2020.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1673-8438.2020.43.009","keywords":[],"unique_id":"paper:10.3969/j.issn.1673-8438.2020.43.009","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1673-8438"],"title":"海带拳","reference_count":0.0,"author":[{"orcid":"","name":"深红"}]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"","access_xinghe_repository_sha256":"","metadata_type":"paper","classifications":{},"citation_count":0.0,"abstract":"育种单位:中国科学院海洋研究所、荣成市蜊江水产有限责任公司\r\n品种简介:该品种是以荣成海带栽培群体后代个体的雌配子体和韩国海带自然种群后代个体的雄配子体杂交产生的后代群体为亲本群体,以藻体深褐色、叶片宽大和孢子囊发育良好为选育指标,采用群体选育技术,经连续4代选育而成.\r\n在相同栽培条件下,与普通海带品种相比,在水温6℃左右(4月上旬)可开始收获,收获期可延续至水温达到19℃左右(7月中下旬),产量提高15.0%以上,抗高温、高光能力较强,淡干海带色泽墨绿.","publication_publisher":[],"access_oa_status":"","relevance_score":29.779959,"is_content_accessible":false,"publication_venue_name_unified":"\u003cspan class=\u0027highlight\u0027\u003e中国\u003c/span\u003e\u003cspan class=\u0027highlight\u0027\u003e水产\u003c/span\u003e | China Fisheries","locations":[],"publication_published_year":2015.0,"access_is_oa":"unknown","doi":"10.3969/j.issn.1002-6681.2015.10.029","keywords":[],"unique_id":"paper:10.3969/j.issn.1002-6681.2015.10.029","language":"zh","citation_normalized_percentile":{},"access_oa_url":[],"publication_venue_issn":["1002-6681"],"title":"海带\"205\"","reference_count":0.0,"author":[{"orcid":"","name":"刘峰"},{"orcid":"","name":"王嘉琪"},{"orcid":"","name":"逄少军"},{"orcid":"","name":"刘启顺"},{"orcid":"","name":"孙长彬"}]}]} +{"query":"pam","results":[{"type":["printbook"],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863390","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.211927,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9788362863390","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396","9788362863390"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9788362863396","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Pracownia Wydawnicza \"ElSet\""],"access_oa_status":"","relevance_score":27.09857,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":2013.0,"access_is_oa":"","doi":"","keywords":[],"unique_id":"ebook:9788362863396","language":"pl","access_oa_url":[],"publication_venue_issn":[],"title":"Pam Pam Pam","reference_count":0.0,"author":[{"orcid":"","name":"Jerzy Szczudlik"}],"isbns":["9788362863396"]},{"type":[],"publication_venue_type":"","access_license":"","cited_by_percentile_year":{},"isbn13":"9780439397858","access_xinghe_repository_sha256":"","metadata_type":"ebook","classifications":{},"citation_count":0.0,"abstract":"","publication_publisher":["Scholastic, Inc."],"access_oa_status":"","relevance_score":27.048462,"is_content_accessible":false,"publication_venue_name_unified":"","locations":[],"publication_published_year":1995.0,"access_is_oa":"unknown","doi":"","keywords":[],"unique_id":"ebook:9780439397858","language":"en","access_oa_url":[],"publication_venue_issn":[],"title":"Pam!, Pam!, Pam!","reference_count":0.0,"author":[{"orcid":"","name":"Eve Merriam"}],"isbns":["9780439397858"]}]} From 05f172870b508f2b880dbf0046c821e5371b12f7 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Tue, 14 Jul 2026 14:24:17 +0800 Subject: [PATCH 11/80] feat(retrieval): add search result quality evaluation - add relevance, effectiveness, and authority evaluators - add standalone and combined executor-based evaluation scripts - support query-level and result-level classified outputs - improve LLM response parsing and content issue detection - set overall weights to 0.7/0.2/0.1 - add evaluator tests and usage documentation --- .../model/llm/llm_search_result_authority.py | 20 ++++++++--- .../model/llm/llm_search_result_relevance.py | 35 +++++++++---------- examples/retrieval/sdk_eval_authority.py | 8 ++--- examples/retrieval/sdk_eval_effectiveness.py | 8 ++--- examples/retrieval/sdk_eval_relevancy.py | 8 ++--- examples/retrieval/sdk_eval_search_result.py | 19 +++------- .../llm/test_llm_search_result_authority.py | 24 +++++++++++++ .../llm/test_llm_search_result_relevance.py | 13 +++---- test/scripts/retrieval/test_open_eval.py | 23 ++++++++++++ 9 files changed, 99 insertions(+), 59 deletions(-) create mode 100644 test/scripts/model/llm/test_llm_search_result_authority.py diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py index cfd42779..5d7fe6e9 100644 --- a/dingo/model/llm/llm_search_result_authority.py +++ b/dingo/model/llm/llm_search_result_authority.py @@ -1,8 +1,6 @@ """Rule-based search result authority grader.""" from __future__ import annotations - -import json import math import statistics from dataclasses import dataclass @@ -13,7 +11,6 @@ from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model - HIGH_AUTHORITY_VENUE_HINTS = ( "nature", "science", @@ -63,6 +60,21 @@ def extract_citations(result: dict[str, Any], key: str = "citation_count") -> fl return 0.0 +def _has_doi_in_locations(locations: Any) -> bool: + if not isinstance(locations, list): + return False + for location in locations: + if isinstance(location, dict): + values = location.values() + elif isinstance(location, str): + values = [location] + else: + continue + if any("doi.org" in str(value).lower() for value in values if value): + return True + return False + + @dataclass class AuthorityGrade: """Structured authority score for one search result.""" @@ -135,7 +147,7 @@ def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: venue_score = 0.45 reason = "repository_or_preprint" - doi_score = 1.0 if result.get("doi") or "doi.org" in json.dumps(result.get("locations", [])) else 0.0 + doi_score = 1.0 if result.get("doi") or _has_doi_in_locations(result.get("locations")) else 0.0 score = ( 0.45 * citation_score + 0.20 * influential_score diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index ed43b018..015ee68c 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -35,26 +35,21 @@ DOI_PATTERN = re.compile(r"(?i)(?:https?://(?:dx\.)?doi\.org/)?(10\.\d{4,9}/[^\s]+)") STANDARD_SYSTEM_PROMPT = """\ -You are a helpful assistant that grades the relevance of search results for given queries. -Your task is to assign a relevance score between 0.0 and 1.0 to each result, based on -how good a result is for the query. - -For each search result, carefully read the query and the result. Assign a value for -each criterion as follows: -- Provide a brief explanation of your reasoning in 20 words or fewer. +Grade how useful a search result is for a query and assign a relevance score +from 0.0 to 1.0. +Read the query, title, and available content, then return these fields: +- Brief reasoning in 20 words or fewer. - Assign a query_relevance score between 0.0 and 1.0. - Assign a result_quality score between 0.0 and 1.0. -- Indicate if there are severe content_issues (true/false). +- Set content_issues to true or false. - Assign a confidence score between 0.0 and 1.0. - Assign an overall score between 0.0 and 1.0. -Set content_issues to true only for severe content corruption, such as garbled/mojibake text, -raw HTML/XML or parser residue that materially hurts readability, invisible/control characters, -or unreadable content. Do not mark normal snippets, short abstracts, missing abstracts, or -truncated previews as content_issues when the title/snippet is still readable. +Set content_issues=true only when mojibake, raw HTML/XML, parser residue, invisible/control +characters, or similar corruption makes visible content materially unreadable. Missing or short +abstracts and truncated previews are not issues when the visible title/snippet remains readable. -Return only one valid JSON object. Keep reasoning short and do not use double -quotes inside the reasoning string.""" +Return one valid JSON object only. Do not use double quotes inside reasoning.""" DETAILED_SYSTEM_PROMPT = """\ You are a helpful assistant that grades the relevance of search results for given queries. @@ -242,7 +237,7 @@ def _repair_unescaped_quotes_in_reasoning(text: str) -> str: value_start = start_match.end() next_key = re.search( - r'"\s*,\s*"(query_relevance|result_quality|content_issues|confidence|score)"\s*:', + r'"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"\s*:|})', text[value_start:], flags=re.DOTALL, ) @@ -294,7 +289,7 @@ def _parse_grade_fields_lenient(text: str) -> RelevanceGrade | None: issue_match = re.search(r'"content_issues"\s*:\s*(true|false)', text, flags=re.IGNORECASE) reasoning = "" reasoning_match = re.search( - r'"reasoning"\s*:\s*"(.*?)"\s*,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"', + r'"reasoning"\s*:\s*"(.*?)"\s*(?:,\s*"(?:query_relevance|result_quality|content_issues|confidence|score)"|})', text, flags=re.DOTALL, ) @@ -379,9 +374,11 @@ def is_doi_query(query: str) -> bool: def _extract_result_dois(result: dict[str, Any]) -> list[str]: candidates: list[Any] = [result.get("doi"), result.get("unique_id")] - for location in result.get("locations") or []: - if isinstance(location, dict): - candidates.extend([location.get("doi"), location.get("url"), location.get("landing_page_url")]) + locations = result.get("locations") + if isinstance(locations, list): + for location in locations: + if isinstance(location, dict): + candidates.extend([location.get("doi"), location.get("url"), location.get("landing_page_url")]) dois: list[str] = [] for candidate in candidates: diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py index 11d6d1ca..2c4a7495 100644 --- a/examples/retrieval/sdk_eval_authority.py +++ b/examples/retrieval/sdk_eval_authority.py @@ -1,7 +1,6 @@ """Run search result authority evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultAuthority" diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py index d7bffe0b..8ce3f0d7 100644 --- a/examples/retrieval/sdk_eval_effectiveness.py +++ b/examples/retrieval/sdk_eval_effectiveness.py @@ -1,7 +1,6 @@ """Run search result effectiveness evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultEffectiveness" diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py index d34f0305..6f37306d 100644 --- a/examples/retrieval/sdk_eval_relevancy.py +++ b/examples/retrieval/sdk_eval_relevancy.py @@ -1,7 +1,6 @@ """Run search result relevancy evaluation through Dingo LocalExecutor.""" from __future__ import annotations - import argparse import json import os @@ -13,11 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor - -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultRelevance" diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py index dc28745e..7b309103 100644 --- a/examples/retrieval/sdk_eval_search_result.py +++ b/examples/retrieval/sdk_eval_search_result.py @@ -5,7 +5,6 @@ """ from __future__ import annotations - import argparse import json import math @@ -20,21 +19,11 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from dingo.config import InputArgs -from dingo.exec import Executor -from dingo.model.llm.llm_search_result_relevance import is_doi_query - -from search_result_eval_utils import ( - add_common_args, - get_title, - load_query_result_jsonl, - rank_discounted_mean, - summarize, - write_classified_jsonl, - write_csv, - write_json, -) +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402 +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 +from dingo.model.llm.llm_search_result_relevance import is_doi_query # noqa: E402 WEIGHTS = { "relevance": 0.7, diff --git a/test/scripts/model/llm/test_llm_search_result_authority.py b/test/scripts/model/llm/test_llm_search_result_authority.py new file mode 100644 index 00000000..16cc2e51 --- /dev/null +++ b/test/scripts/model/llm/test_llm_search_result_authority.py @@ -0,0 +1,24 @@ +from dingo.model.llm.llm_search_result_authority import LLMSearchResultAuthority, _has_doi_in_locations + + +class NonSerializableLocation: + pass + + +def test_has_doi_in_location_dict_or_string(): + assert _has_doi_in_locations([{"url": "https://doi.org/10.1000/test"}]) + assert _has_doi_in_locations(["https://DOI.ORG/10.1000/test"]) + + +def test_has_doi_in_locations_ignores_invalid_shapes(): + assert not _has_doi_in_locations(True) + assert not _has_doi_in_locations({"url": "https://doi.org/10.1000/test"}) + assert not _has_doi_in_locations([NonSerializableLocation()]) + + +def test_authority_grade_handles_non_serializable_location(): + grade = LLMSearchResultAuthority().grade( + result={"doi": "", "locations": [NonSerializableLocation()]}, + ) + + assert grade.doi_score == 0.0 diff --git a/test/scripts/model/llm/test_llm_search_result_relevance.py b/test/scripts/model/llm/test_llm_search_result_relevance.py index ffab09e8..210d6d67 100644 --- a/test/scripts/model/llm/test_llm_search_result_relevance.py +++ b/test/scripts/model/llm/test_llm_search_result_relevance.py @@ -1,9 +1,4 @@ -from dingo.model.llm.llm_search_result_relevance import ( - _extract_result_dois, - _grade_doi_result, - _normalize_doi, - is_doi_query, -) +from dingo.model.llm.llm_search_result_relevance import _extract_result_dois, _grade_doi_result, _normalize_doi, is_doi_query def test_normalize_doi_variants(): @@ -28,6 +23,12 @@ def test_extract_result_dois_from_supported_fields(): ] +def test_extract_result_dois_ignores_non_list_locations(): + assert _extract_result_dois({"locations": True}) == [] + assert _extract_result_dois({"locations": 1}) == [] + assert _extract_result_dois({"locations": {"url": "https://doi.org/10.1000/test"}}) == [] + + def test_doi_query_uses_exact_match(): matched = _grade_doi_result( "10.1016/j.ijbiomac.2025.143529", diff --git a/test/scripts/retrieval/test_open_eval.py b/test/scripts/retrieval/test_open_eval.py index 987e9264..dacb0973 100644 --- a/test/scripts/retrieval/test_open_eval.py +++ b/test/scripts/retrieval/test_open_eval.py @@ -102,6 +102,29 @@ def test_lenient_parse_unescaped_quotes_in_reasoning(self): assert grade.content_issues is False assert grade.confidence == 0.85 + def test_repairs_unescaped_quotes_when_reasoning_is_last(self): + response = ( + '{"score": 0.92, "query_relevance": 0.95, "result_quality": 0.9, ' + '"content_issues": false, "confidence": 0.85, ' + '"reasoning": "The query "PBPK" directly matches"}' + ) + + grade = _parse_grade_response(response) + + assert grade.error == "" + assert grade.score == 0.92 + assert grade.reasoning == 'The query "PBPK" directly matches' + + def test_lenient_parse_when_reasoning_is_last(self): + response = '{"score": 0.7, "query_relevance": 0.8, "broken": ???, "reasoning": "final reason"}' + + grade = _parse_grade_response(response) + + assert grade.error == "" + assert grade.score == 0.7 + assert grade.query_relevance == 0.8 + assert grade.reasoning == "final reason" + def test_missing_fields_default_to_zero(self): grade = _parse_grade_response('{"score": 0.5}') assert grade.score == 0.5 From 3f6b94bc3cae26e7a6e75446dc7f41a256d1e82b Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Tue, 14 Jul 2026 14:54:32 +0800 Subject: [PATCH 12/80] feat(retrieval): add search result quality evaluation Fix BUG --- .../model/llm/llm_search_result_authority.py | 25 +++++++++++++++++-- examples/retrieval/sdk_eval_authority.py | 6 ++--- examples/retrieval/sdk_eval_effectiveness.py | 6 ++--- examples/retrieval/sdk_eval_relevancy.py | 6 ++--- examples/retrieval/sdk_eval_search_result.py | 8 +++--- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py index 87650b49..85d8a7c9 100644 --- a/dingo/model/llm/llm_search_result_authority.py +++ b/dingo/model/llm/llm_search_result_authority.py @@ -1,7 +1,6 @@ """Rule-based search result authority grader.""" from __future__ import annotations -import json import math import statistics from dataclasses import dataclass @@ -61,6 +60,28 @@ def extract_citations(result: dict[str, Any], key: str = "citation_count") -> fl return 0.0 +def _has_doi_in_locations(locations: Any) -> bool: + """Return whether a valid locations list contains a DOI URL or value.""" + if not isinstance(locations, list): + return False + + for location in locations: + if isinstance(location, dict): + values = ( + location.get("doi"), + location.get("url"), + location.get("landing_page_url"), + ) + elif isinstance(location, str): + values = (location,) + else: + continue + + if any("doi.org/" in str(value or "").lower() for value in values): + return True + return False + + @dataclass class AuthorityGrade: """Structured authority score for one search result.""" @@ -133,7 +154,7 @@ def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: venue_score = 0.45 reason = "repository_or_preprint" - doi_score = 1.0 if result.get("doi") or "doi.org" in json.dumps(result.get("locations", [])) else 0.0 + doi_score = 1.0 if result.get("doi") or _has_doi_in_locations(result.get("locations")) else 0.0 score = ( 0.45 * citation_score + 0.20 * influential_score diff --git a/examples/retrieval/sdk_eval_authority.py b/examples/retrieval/sdk_eval_authority.py index d0327565..2c4a7495 100644 --- a/examples/retrieval/sdk_eval_authority.py +++ b/examples/retrieval/sdk_eval_authority.py @@ -12,10 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 -from dingo.config import InputArgs -from dingo.exec import Executor +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultAuthority" diff --git a/examples/retrieval/sdk_eval_effectiveness.py b/examples/retrieval/sdk_eval_effectiveness.py index a57c3a15..8ce3f0d7 100644 --- a/examples/retrieval/sdk_eval_effectiveness.py +++ b/examples/retrieval/sdk_eval_effectiveness.py @@ -12,10 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 -from dingo.config import InputArgs -from dingo.exec import Executor +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultEffectiveness" diff --git a/examples/retrieval/sdk_eval_relevancy.py b/examples/retrieval/sdk_eval_relevancy.py index 2956945b..6f37306d 100644 --- a/examples/retrieval/sdk_eval_relevancy.py +++ b/examples/retrieval/sdk_eval_relevancy.py @@ -12,10 +12,10 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl # noqa: E402 -from dingo.config import InputArgs -from dingo.exec import Executor +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 EVALUATOR_NAME = "LLMSearchResultRelevance" diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py index f4403a1e..7b309103 100644 --- a/examples/retrieval/sdk_eval_search_result.py +++ b/examples/retrieval/sdk_eval_search_result.py @@ -19,11 +19,11 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402 -from dingo.config import InputArgs -from dingo.exec import Executor -from dingo.model.llm.llm_search_result_relevance import is_doi_query +from dingo.config import InputArgs # noqa: E402 +from dingo.exec import Executor # noqa: E402 +from dingo.model.llm.llm_search_result_relevance import is_doi_query # noqa: E402 WEIGHTS = { "relevance": 0.7, From 080cf859f860f0b723887fa235aff1dc3b7dc631 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Wed, 15 Jul 2026 18:48:47 +0800 Subject: [PATCH 13/80] =?UTF-8?q?feat(retrieval):=20add=20search=20result?= =?UTF-8?q?=20quality=20evaluation=20=E4=BC=98=E5=8C=96=E4=BA=86=E6=9C=89?= =?UTF-8?q?=E6=95=88=E6=80=A7=E5=92=8C=E7=9B=B8=E5=85=B3=E6=80=A7=E5=8C=B9?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/llm/llm_search_result_authority.py | 153 +++++++++++++--- .../llm/llm_search_result_effectiveness.py | 172 +++++++++++++----- .../model/llm/llm_search_result_relevance.py | 49 +++-- docs/search_result_authority_executor.md | 26 ++- docs/search_result_effectiveness_executor.md | 20 +- docs/search_result_quality_metrics.md | 85 ++++++--- docs/search_result_relevance_executor.md | 5 + examples/retrieval/sdk_eval_search_result.py | 4 +- .../llm/test_llm_search_result_authority.py | 102 +++++++++++ .../test_llm_search_result_effectiveness.py | 107 ++++++++++- 10 files changed, 593 insertions(+), 130 deletions(-) diff --git a/dingo/model/llm/llm_search_result_authority.py b/dingo/model/llm/llm_search_result_authority.py index 85d8a7c9..36f9ec42 100644 --- a/dingo/model/llm/llm_search_result_authority.py +++ b/dingo/model/llm/llm_search_result_authority.py @@ -1,7 +1,9 @@ """Rule-based search result authority grader.""" from __future__ import annotations +import html import math +import re import statistics from dataclasses import dataclass from typing import Any @@ -11,27 +13,86 @@ from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model -HIGH_AUTHORITY_VENUE_HINTS = ( - "nature", - "science", - "cell", - "nejm", - "lancet", - "jama", - "acm", - "ieee", +PRESTIGIOUS_VENUE_PATTERNS = ( + r"^nature(?:$|\s)", + r"^npj(?:$|\s)", + r"^communications (?:biology|chemistry|earth & environment|materials|physics)$", + r"^scientific (?:reports|data)$", + r"^science$", + r"^science (?:advances|immunology|robotics|signaling|translational medicine)$", + r"^cell$", + r"^cell (?:reports|metabolism|systems|stem cell|chemical biology|host & microbe|genomics)$", + r"^(?:cancer|molecular|developmental) cell$", + r"^(?:the )?lancet(?:$|\s)", + r"^(?:the )?new england journal of medicine$", + r"^nejm(?:$|\s)", + r"^jama(?:$|\s)", + r"^(?:the )?bmj$", + r"^(?:proceedings of the national academy of sciences(?: of the united states of america)?|pnas)$", + r"^journal of the american chemical society$", + r"^physical review letters$", + r"^angewandte chemie(?: international edition)?$", + r"^(?:neurips|icml|iclr|cvpr|acl|emnlp|aaai|ijcai|sigir)$", + r"^advances in neural information processing systems$", +) + +RECOGNIZED_VENUE_PATTERNS = ( + r"\bieee\b", + r"\bacm\b", + r"^plos(?:$|\s)", + r"^frontiers in ", + r"^bmj(?:$|\s)", +) + +RECOGNIZED_PUBLISHER_HINTS = ( + "nature portfolio", + "springer nature", "springer", "elsevier", "wiley", - "neurips", - "icml", - "iclr", - "cvpr", - "acl", - "emnlp", - "aaai", - "ijcai", - "sigir", + "taylor & francis", + "routledge", + "sage publishing", + "oxford university press", + "cambridge university press", + "institute of electrical and electronics engineers", + "association for computing machinery", + "american chemical society", + "royal society of chemistry", + "institute of physics", + "iop publishing", + "american physical society", + "american institute of physics", + "frontiers media", + "public library of science", + "bmj publishing", + "wolters kluwer", + "lippincott williams & wilkins", + "american geophysical union", + "royal society", + "american association for the advancement of science", + "de gruyter", + "crc press", + "chapman & hall", + "world scientific", + "academic press", + "humana press", + "emerald publishing", +) + +REPOSITORY_VENUE_HINTS = ( + "arxiv", + "biorxiv", + "medrxiv", + "chemrxiv", + "ssrn", + "zenodo", + "figshare", + "mendeley data", + "open science framework", + "osf preprints", + "research square", + "repository", ) @@ -40,7 +101,31 @@ def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: def _normalize_text(text: Any) -> str: - return " ".join(str(text or "").lower().split()) + value = html.unescape(str(text or "")) + value = re.sub(r"<[^>]+>", " ", value) + return " ".join(value.lower().split()) + + +def _venue_aliases(venue: str) -> list[str]: + aliases = [part.strip(" .,:;-") for part in venue.split("|") if part.strip(" .,:;-")] + return aliases or [venue] + + +def _matches_venue_patterns(venue: str, patterns: tuple[str, ...]) -> bool: + return any(re.search(pattern, alias) for alias in _venue_aliases(venue) for pattern in patterns) + + +def _extract_publishers(result: dict[str, Any]) -> str: + value = result.get("publication_publisher") or result.get("publisher") or "" + if isinstance(value, list): + return _normalize_text(" | ".join(str(item) for item in value if item)) + return _normalize_text(value) + + +def _has_venue_issn(result: dict[str, Any]) -> bool: + value = result.get("publication_venue_issn") or result.get("issn") or [] + values = value if isinstance(value, list) else [value] + return any(re.fullmatch(r"\d{4}-?\d{3}[\dXx]", str(item or "").strip()) for item in values) def extract_venue(result: dict[str, Any]) -> str: @@ -136,6 +221,7 @@ class LLMSearchResultAuthority: def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: venue = _normalize_text(extract_venue(result)) venue_type = _normalize_text(result.get("publication_venue_type") or "") + publishers = _extract_publishers(result) citations = extract_citations(result, "citation_count") influential = extract_citations(result, "influential_citation_count") @@ -144,15 +230,30 @@ def grade(self, *, result: dict[str, Any]) -> AuthorityGrade: venue_score = 0.25 reason = "unknown_or_low_signal_venue" - if any(hint in venue for hint in HIGH_AUTHORITY_VENUE_HINTS): - venue_score = 0.85 - reason = "high_authority_venue_hint" - elif "journal" in venue_type or "conference" in venue_type: - venue_score = 0.65 - reason = "journal_or_conference" - elif "repository" in venue_type or "preprint" in venue: + is_repository = "repository" in venue_type or "preprint" in venue_type or any( + hint in venue for hint in REPOSITORY_VENUE_HINTS + ) + is_academic_book = "book series" in venue_type or "ebook platform" in venue_type or "ebooks" in venue + if is_repository: venue_score = 0.45 reason = "repository_or_preprint" + elif is_academic_book: + venue_score = 0.55 + reason = "academic_book_series" + elif _matches_venue_patterns(venue, PRESTIGIOUS_VENUE_PATTERNS): + venue_score = 0.85 + reason = "prestigious_venue_family" + elif _matches_venue_patterns(venue, RECOGNIZED_VENUE_PATTERNS) or any( + hint in publishers for hint in RECOGNIZED_PUBLISHER_HINTS + ): + venue_score = 0.75 + reason = "recognized_scholarly_publisher_or_venue" + elif "journal" in venue_type or "conference" in venue_type or _has_venue_issn(result): + venue_score = 0.65 + reason = "structured_journal_or_conference" + elif venue: + venue_score = 0.40 + reason = "named_venue" doi_score = 1.0 if result.get("doi") or _has_doi_in_locations(result.get("locations")) else 0.0 score = ( diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index 3b1c78bf..aa9ac9d0 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -15,6 +15,7 @@ import logging import re import statistics +import time from dataclasses import dataclass from typing import Any @@ -40,9 +41,11 @@ MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}" UTF8_LATIN1_SEQUENCE_PATTERN = re.compile(r"[\u00C2\u00C3\u00D0\u00D1][\u0080-\u00BF]") C1_CONTROL_PATTERN = re.compile(r"[\u0080-\u009F]") +UNICODE_REPLACEMENT_CHARACTER = "\ufffd" -LLM_FIELD_QUALITY_SYSTEM_PROMPT = """You are a strict but practical data quality evaluator for academic search result metadata. +LLM_FIELD_QUALITY_SYSTEM_PROMPT = """You are a strict but practical data quality evaluator for +academic search result metadata. Judge whether each supplied metadata field is readable and clean enough to show to users. Focus on real text-quality problems: @@ -58,14 +61,16 @@ - punctuation, pipes used as separators, parentheses, slashes, hyphens - mixed Chinese/English titles, journal names, abbreviations, DOI-like text -Return compact JSON only. Do not use markdown. Keep each reason within 12 words and do not use double quotes inside reasons. +Return compact JSON only. Do not use markdown. Keep each reason within 12 words +and do not use double quotes inside reasons. Schema: { "fields": { "title": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, "abstract": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, "keywords": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, - "venue": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."} + "venue": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."}, + "author": {"score": 0.0-1.0, "issues": ["..."], "reason": "..."} }, "overall_issues": ["..."], "reason": "short overall reason" @@ -147,7 +152,11 @@ def _looks_like_utf8_latin1_mojibake(text: str) -> bool: def _has_mojibake_evidence(text: str) -> bool: value = str(text or "") - return bool(re.search(MOJIBAKE_EVIDENCE_PATTERN, value)) or _looks_like_utf8_latin1_mojibake(value) + return ( + UNICODE_REPLACEMENT_CHARACTER in value + or bool(re.search(MOJIBAKE_EVIDENCE_PATTERN, value)) + or _looks_like_utf8_latin1_mojibake(value) + ) def _rule_abnormal_char_issues(text: str) -> list[str]: @@ -159,15 +168,16 @@ def _rule_abnormal_char_issues(text: str) -> list[str]: special_matches: list[str] = [] for pattern in RULE_SPECIAL_CHARACTER_PATTERNS: special_matches.extend(re.findall(pattern, value)) - if len(special_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + has_html_tag = bool(re.search(HTML_TAG_PATTERN, value)) + if has_html_tag or len(special_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: issues.append("RuleSpecialCharacter") - has_latin1_mojibake = _looks_like_utf8_latin1_mojibake(value) - if has_latin1_mojibake: + has_mojibake = _has_mojibake_evidence(value) + if has_mojibake: issues.append("RuleMojibake") invisible_matches = re.findall(RULE_INVISIBLE_CHAR_PATTERN, value) - if not has_latin1_mojibake and len(invisible_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: + if not has_mojibake and len(invisible_matches) / len(value) >= RULE_ABNORMAL_CHAR_THRESHOLD: issues.append("RuleInvisibleChar") return issues @@ -218,7 +228,7 @@ def _extract_json_object(text: str) -> str: start = value.find("{") end = value.rfind("}") if start >= 0 and end > start: - return value[start : end + 1] + return value[start:end + 1] return value @@ -248,7 +258,7 @@ def _normalize_issues(value: Any) -> list[str]: "missing_title": "Effectiveness.Error_Title_Miss", "missing_abstract": "Effectiveness.Error_Abstract_Miss", "missing_keywords": "Effectiveness.Error_Keywords_Miss", - "missing_venue": "Effectiveness.Error_Venue_Miss", + "missing_author": "Effectiveness.Error_Author_Miss", "html_tag": "Effectiveness.Error_HTML_Tag", "mojibake": "Effectiveness.Error_Mojibake", "invisible_char": "Effectiveness.Error_Invisible_Char", @@ -332,6 +342,38 @@ def extract_venue(result: dict[str, Any]) -> str: ) +def extract_authors(result: dict[str, Any]) -> list[str]: + """Extract author names from common search API response shapes.""" + value = result.get("author") or result.get("authors") or [] + if isinstance(value, str): + return [item.strip() for item in re.split(r"[;|]", value) if item.strip()] + if isinstance(value, dict): + value = [value] + if not isinstance(value, list): + return [] + + authors: list[str] = [] + for item in value: + if isinstance(item, dict): + name = item.get("name") or item.get("display_name") or item.get("author_name") + if name: + authors.append(str(name).strip()) + elif item not in (None, ""): + authors.append(str(item).strip()) + return [author for author in authors if author] + + +def _author_quality(authors: list[str]) -> float: + if not authors: + return 0.0 + valid_count = sum( + 1 + for author in authors + if len(author.strip()) >= 2 and re.search(r"[A-Za-z\u4e00-\u9fff]", author) + ) + return _clamp(valid_count / len(authors)) + + @dataclass class LLMFieldQuality: """LLM readability and corruption judgment for one search result.""" @@ -340,6 +382,7 @@ class LLMFieldQuality: abstract_score: float = 1.0 keywords_score: float = 1.0 venue_score: float = 1.0 + author_score: float = 1.0 issues: list[str] | None = None reason: str = "" error: str = "" @@ -350,6 +393,7 @@ def field_score(self, field: str) -> float: "abstract": self.abstract_score, "keywords": self.keywords_score, "venue": self.venue_score, + "author": self.author_score, }.get(field, 1.0) @@ -367,7 +411,7 @@ def _parse_llm_field_quality_response(text: str) -> LLMFieldQuality: issues: list[str] = [] scores: dict[str, float] = {} reasons: list[str] = [] - for field in ("title", "abstract", "keywords", "venue"): + for field in ("title", "abstract", "keywords", "venue", "author"): field_data = fields.get(field) or {} if not isinstance(field_data, dict): field_data = {} @@ -386,6 +430,7 @@ def _parse_llm_field_quality_response(text: str) -> LLMFieldQuality: abstract_score=scores["abstract"], keywords_score=scores["keywords"], venue_score=scores["venue"], + author_score=scores["author"], issues=issues, reason=str(data.get("reason") or "; ".join(reasons))[:500], ) @@ -400,6 +445,7 @@ class EffectivenessGrade: abstract_score: float = 0.0 keywords_score: float = 0.0 venue_score: float = 0.0 + author_score: float = 0.0 issues: list[str] | None = None llm_quality_reason: str = "" llm_quality_error: str = "" @@ -411,6 +457,7 @@ def to_dict(self) -> dict[str, Any]: "abstract_score": round(self.abstract_score, 5), "keywords_score": round(self.keywords_score, 5), "venue_score": round(self.venue_score, 5), + "author_score": round(self.author_score, 5), "issues": self.issues or [], "llm_quality_reason": self.llm_quality_reason, "llm_quality_error": self.llm_quality_error, @@ -425,6 +472,7 @@ class EffectivenessSummary: mean_abstract_score: float = 0.0 mean_keywords_score: float = 0.0 mean_venue_score: float = 0.0 + mean_author_score: float = 0.0 graded_pairs: int = 0 def to_dict(self) -> dict[str, Any]: @@ -435,13 +483,18 @@ def to_dict(self) -> dict[str, Any]: "effectiveness_mean_abstract_score": round(self.mean_abstract_score, 5), "effectiveness_mean_keywords_score": round(self.mean_keywords_score, 5), "effectiveness_mean_venue_score": round(self.mean_venue_score, 5), + "effectiveness_mean_author_score": round(self.mean_author_score, 5), "effectiveness_graded_pairs": self.graded_pairs, } @Model.llm_register("LLMSearchResultEffectiveness") class LLMSearchResultEffectiveness: - """Effectiveness scorer for title, abstract, keywords, and venue.""" + """Effectiveness scorer for title, abstract, keywords, and authors. + + Venue text is still scanned for corruption, but venue presence and quality + belong to the authority metric and do not affect the effectiveness score. + """ dynamic_config = EvaluatorLLMArgs() default_threshold = 0.15 @@ -485,6 +538,7 @@ def _build_llm_quality_user_message( abstract: str, keywords: list[str], venue: str, + authors: list[str], candidate_fields: set[str] | None = None, ) -> str: all_fields = { @@ -492,6 +546,7 @@ def _build_llm_quality_user_message( "abstract": abstract, "keywords": " | ".join(keywords), "venue": venue, + "author": " | ".join(authors), } selected = candidate_fields or set(all_fields) payload = { @@ -512,36 +567,53 @@ def _judge_llm_field_quality( abstract: str, keywords: list[str], venue: str, + authors: list[str], candidate_fields: set[str] | None = None, ) -> LLMFieldQuality: if not self.enable_llm_quality: return LLMFieldQuality() - try: - client = self._get_client() - completion = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": LLM_FIELD_QUALITY_SYSTEM_PROMPT}, - { - "role": "user", - "content": self._build_llm_quality_user_message( - title=title, - abstract=abstract, - keywords=keywords, - venue=venue, - candidate_fields=candidate_fields, - ), - }, - ], - temperature=self.temperature, - max_tokens=self.max_tokens, - timeout=self.timeout, + client = self._get_client() + last_result = LLMFieldQuality(error="LLM field quality judgment failed") + for attempt in range(3): + try: + completion = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": LLM_FIELD_QUALITY_SYSTEM_PROMPT}, + { + "role": "user", + "content": self._build_llm_quality_user_message( + title=title, + abstract=abstract, + keywords=keywords, + venue=venue, + authors=authors, + candidate_fields=candidate_fields, + ), + }, + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + response_text = completion.choices[0].message.content or "" + last_result = _parse_llm_field_quality_response(response_text) + if not last_result.error: + return last_result + error: Exception | str = last_result.error + except Exception as exc: + error = exc + last_result = LLMFieldQuality(error=str(exc)) + + logger.warning( + "LLM field quality attempt %s/3 failed for title=%r: %s", + attempt + 1, + title, + error, ) - response_text = completion.choices[0].message.content or "" - return _parse_llm_field_quality_response(response_text) - except Exception as e: - logger.warning("LLM field quality judgment failed for title=%r: %s", title, e) - return LLMFieldQuality(error=str(e)) + if attempt < 2: + time.sleep(attempt + 1) + return last_result def grade( self, @@ -550,6 +622,7 @@ def grade( abstract: str = "", keywords: list[str] | str | None = None, venue: str = "", + authors: list[str] | str | None = None, result: dict[str, Any] | None = None, ) -> EffectivenessGrade: if result is not None: @@ -557,12 +630,18 @@ def grade( abstract = str(result.get("abstract") or abstract or "") keywords = extract_keywords(result) if keywords is None else keywords venue = extract_venue(result) or venue + authors = extract_authors(result) if authors is None else authors keyword_items = ( [item.strip() for item in re.split(r"[,;|]", keywords) if item.strip()] if isinstance(keywords, str) else [str(item).strip() for item in (keywords or []) if str(item).strip()] ) + author_items = ( + [item.strip() for item in re.split(r"[;|]", authors) if item.strip()] + if isinstance(authors, str) + else [str(item).strip() for item in (authors or []) if str(item).strip()] + ) title_score = _field_quality(title, min_chars=6, good_chars=35) if _token_count(title) <= 2 and len(str(title).strip()) < 15: @@ -578,6 +657,8 @@ def grade( if venue and not re.search(r"[A-Za-z\u4e00-\u9fff]", venue): venue_score *= 0.4 + author_score = _author_quality(author_items) + issues: list[str] = [] if not str(title or "").strip(): issues.append("missing_title") @@ -585,14 +666,15 @@ def grade( issues.append("missing_abstract") if not keyword_items: issues.append("missing_keywords") - if not str(venue or "").strip(): - issues.append("missing_venue") + if not author_items: + issues.append("missing_author") field_values = { "title": str(title or ""), "abstract": str(abstract or ""), "keywords": " | ".join(keyword_items), "venue": str(venue or ""), + "author": " | ".join(author_items), } rule_candidate_issues = { field: _rule_abnormal_char_issues(value) @@ -612,6 +694,7 @@ def grade( abstract=str(abstract or ""), keywords=keyword_items, venue=str(venue or ""), + authors=author_items, candidate_fields=set(rule_candidate_issues), ) @@ -647,15 +730,16 @@ def apply_confirmed_field_issue(field: str, score: float) -> float: abstract_score = apply_confirmed_field_issue("abstract", abstract_score) keywords_score = apply_confirmed_field_issue("keywords", keywords_score) venue_score = apply_confirmed_field_issue("venue", venue_score) + author_score = apply_confirmed_field_issue("author", author_score) if rule_candidate_issues and self.enable_llm_quality and llm_quality.error: issues.append("llm_quality_parse_error") score = ( - 0.25 * title_score - + 0.45 * abstract_score - + 0.15 * keywords_score - + 0.15 * venue_score + 0.30 * title_score + + 0.50 * abstract_score + + 0.10 * keywords_score + + 0.10 * author_score ) return EffectivenessGrade( score=_clamp(score), @@ -663,6 +747,7 @@ def apply_confirmed_field_issue(field: str, score: float) -> float: abstract_score=_clamp(abstract_score), keywords_score=_clamp(keywords_score), venue_score=_clamp(venue_score), + author_score=_clamp(author_score), issues=issues, llm_quality_reason=llm_quality.reason, llm_quality_error=llm_quality.error, @@ -723,5 +808,6 @@ def aggregate_grades(grades: list[EffectivenessGrade]) -> EffectivenessSummary: mean_abstract_score=statistics.mean(g.abstract_score for g in grades), mean_keywords_score=statistics.mean(g.keywords_score for g in grades), mean_venue_score=statistics.mean(g.venue_score for g in grades), + mean_author_score=statistics.mean(g.author_score for g in grades), graded_pairs=len(grades), ) diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index 015ee68c..febea86f 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -19,6 +19,7 @@ import logging import re import statistics +import time from dataclasses import dataclass from typing import Any @@ -472,23 +473,39 @@ def grade( expected_criteria=expected_criteria or self.expected_criteria, ) - try: - client = self._get_client() - completion = client.chat.completions.create( - model=self.model, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_message}, - ], - temperature=self.temperature, - max_tokens=self.max_tokens, - timeout=self.timeout, + client = self._get_client() + last_grade = RelevanceGrade(error="LLM grading failed") + for attempt in range(3): + try: + completion = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message}, + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + timeout=self.timeout, + ) + response_text = completion.choices[0].message.content or "" + last_grade = _parse_grade_response(response_text) + if not last_grade.error: + return last_grade + error: Exception | str = last_grade.error + except Exception as exc: + error = exc + last_grade = RelevanceGrade(error=str(exc)) + + logger.warning( + "LLM grading attempt %s/3 failed for query=%r title=%r: %s", + attempt + 1, + query, + title, + error, ) - response_text = completion.choices[0].message.content or "" - return _parse_grade_response(response_text) - except Exception as e: - logger.warning("LLM grading failed for query=%r title=%r: %s", query, title, e) - return RelevanceGrade(error=str(e)) + if attempt < 2: + time.sleep(attempt + 1) + return last_grade @classmethod def _config_value(cls, name: str, default: Any = None) -> Any: diff --git a/docs/search_result_authority_executor.md b/docs/search_result_authority_executor.md index 9b666a95..3b171df4 100644 --- a/docs/search_result_authority_executor.md +++ b/docs/search_result_authority_executor.md @@ -147,19 +147,31 @@ Venue scoring: | Condition | `venue_score` | Reason | |---|---:|---| -| Venue name contains a high-authority hint | `0.85` | `high_authority_venue_hint` | -| `publication_venue_type` contains journal or conference | `0.65` | `journal_or_conference` | -| `publication_venue_type` contains repository, or venue contains preprint | `0.45` | `repository_or_preprint` | +| Known repository or preprint source | `0.45` | `repository_or_preprint` | +| Explicit academic book series or ebook platform | `0.55` | `academic_book_series` | +| Venue matches a prestigious journal/conference family | `0.85` | `prestigious_venue_family` | +| Venue or publisher matches a recognized scholarly organization | `0.75` | `recognized_scholarly_publisher_or_venue` | +| Journal/conference type or valid ISSN is present | `0.65` | `structured_journal_or_conference` | +| A venue name is present without stronger structured signals | `0.40` | `named_venue` | | Unknown or low-signal source | `0.25` | `unknown_or_low_signal_venue` | -High-authority venue hints include: +Explicit source types run first. Repository detection prevents names such as `Open Science Framework` from being promoted merely because they contain a prestigious-looking word, while book series and ebook platforms remain at the book-source tier even when their publisher is recognized. + +Prestigious venue families include: ```text -nature, science, cell, nejm, lancet, jama, -acm, ieee, springer, elsevier, wiley, -neurips, icml, iclr, cvpr, acl, emnlp, aaai, ijcai, sigir +Nature and Nature subject journals, Nature Communications, +npj journals, Communications journals, Scientific Reports/Data, +the official Science journal family, selected Cell Press flagships, +NEJM, Lancet, JAMA, The BMJ, PNAS, JACS, PRL, and major AI conferences ``` +The matcher uses anchored family patterns instead of unrestricted substrings. For example, `Science Translational Medicine` matches the Science family, while `Chemical Engineering Science` does not. HTML highlight tags are removed before matching. + +Recognized publisher metadata includes established scholarly publishers and societies such as Springer Nature, Elsevier, Wiley, Oxford University Press, Cambridge University Press, IEEE, ACM, ACS, RSC, IOP, BMJ, PLOS, the Royal Society, De Gruyter, CRC Press, and World Scientific. Publisher recognition is deliberately scored below an explicitly prestigious venue because publisher reputation alone does not make every title a flagship journal. + +The ISSN and venue-type fallback is the main protection against an incomplete whitelist: a journal does not need to appear in a hard-coded title list to receive a structured scholarly venue score. + DOI scoring: ```text diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md index 1918afd9..453b3996 100644 --- a/docs/search_result_effectiveness_executor.md +++ b/docs/search_result_effectiveness_executor.md @@ -59,11 +59,13 @@ python examples/retrieval/sdk_eval_effectiveness.py ` Run with LLM second judgment for abnormal-character candidates: +For full runs, use a low-latency Flash model such as `deepseek-v4-flash`. The LLM is only used to review rule-selected suspicious fields, but a slow Pro model can still substantially increase runtime. Reserve Pro models for small-sample diagnosis, and use temperature `0` for reproducible comparisons. + ```powershell $env:OPENAI_API_KEY="..." $env:OPENAI_BASE_URL="http://35.220.164.252:3888/v1/" $env:OPENAI_MODEL="deepseek-v4-flash" -$env:OPENAI_TEMPERATURE="0.7" +$env:OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_effectiveness.py ` --input-jsonl outputs/meta_search_97_query_results.jsonl ` @@ -121,7 +123,7 @@ Error labels: | `search_result/Effectiveness/Error_Title_Miss.jsonl` | Missing title | | `search_result/Effectiveness/Error_Abstract_Miss.jsonl` | Missing abstract | | `search_result/Effectiveness/Error_Keywords_Miss.jsonl` | Missing keywords | -| `search_result/Effectiveness/Error_Venue_Miss.jsonl` | Missing publication venue/source | +| `search_result/Effectiveness/Error_Author_Miss.jsonl` | Missing author metadata | | `search_result/Effectiveness/Error_HTML_Tag.jsonl` | LLM-confirmed HTML tag pollution | | `search_result/Effectiveness/Error_Mojibake.jsonl` | LLM-confirmed mojibake | | `search_result/Effectiveness/Error_Invisible_Char.jsonl` | LLM-confirmed invisible characters | @@ -133,14 +135,16 @@ Error labels: ## Scoring -The metric score is unchanged: +The metric score is: ```text Effectiveness = - title_score * 0.25 -+ abstract_score * 0.45 -+ keywords_score * 0.15 -+ venue_score * 0.15 + title_score * 0.30 ++ abstract_score * 0.50 ++ keywords_score * 0.10 ++ author_score * 0.10 ``` -Field scores are based on missing-field checks, length/information-density checks, and abnormal-character handling. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. +`author_score` checks whether at least one recognizable author name is present. A valid single-author paper receives full author completeness credit; the metric does not reward a larger author count. + +Title, abstract, keywords, and author use missing-field checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index 5c4f60c0..653ba60a 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -18,7 +18,7 @@ - 内容有效性判断“这条结果记录是否有足够信息可读可用”。 - 权威性判断“这条结果是否有论文影响力、来源、DOI 等可信信号”。 -例如，用户搜索 `Wallace Chafe`，rank1 返回标题也是 `Wallace Chafe`，相关性可能较好；但如果该结果没有 abstract、keywords、publication venue，则内容有效性会较低。 +例如，用户搜索 `Wallace Chafe`，rank1 返回标题也是 `Wallace Chafe`，相关性可能较好；但如果该结果没有 abstract、keywords、author，则内容有效性会较低。Publication venue 的缺失由权威性指标负责。 ## 2. 输入格式 @@ -92,7 +92,7 @@ QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS Effectiveness/Error_Title_Miss.jsonl Effectiveness/Error_Abstract_Miss.jsonl Effectiveness/Error_Keywords_Miss.jsonl -Effectiveness/Error_Venue_Miss.jsonl +Effectiveness/Error_Author_Miss.jsonl Effectiveness/Error_HTML_Tag.jsonl Effectiveness/Error_Mojibake.jsonl Effectiveness/Error_Invisible_Char.jsonl @@ -197,13 +197,28 @@ QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR | `OPENAI_BASE_URL` | 空 | OpenAI compatible endpoint | | `OPENAI_TEMPERATURE` | 0.0 | LLM temperature | +### 5.6 LLM 模型选择建议 + +全量检索评测需要逐条判断 query-result 对，LLM 调用量通常接近 `query 数 × top-k`。建议优先使用低延迟的 Flash 类模型，例如： + +```text +OPENAI_MODEL=deepseek-v4-flash +``` + +- **全量评测和日常回归**：推荐 Flash 类模型，可显著缩短相关性判断时间，并降低长时间批处理中的超时风险。 +- **疑难样本复核**：Pro 类模型更适合抽取少量低分、边界或争议样本进行人工辅助复核，不建议直接用于数百至数千条 result 的常规全量评测。 +- **新旧版本对比**：两次评测必须固定相同模型、prompt、`max_tokens` 和 temperature；建议设置 `OPENAI_TEMPERATURE=0`，减少 LLM 随机波动。 +- **并发设置**：建议从 `--llm-workers 2` 至 `4` 开始，根据模型服务的限流和稳定性逐步调整。并发过高可能增加 5xx、超时或空响应。 + +模型名称由实际 OpenAI-compatible 服务决定，`deepseek-v4-flash` 仅作为当前环境的推荐示例，不是 Dingo 的强制依赖。 + ## 6. 内容有效性 Effectiveness ### 6.1 业务逻辑 内容有效性判断的是一条检索结果记录是否“可读、完整、可用于用户判断”，不判断它是否与 query 相关。 -当前不按 `metadata_type` 做差异化处理。也就是说，`paper`、`ebook`、未来新增类型都用同一套字段完整性标准。这有利于持续观察数据库元数据补全质量：如果 ebook 未来补全 abstract、keywords、venue，有效性分数会自然提升。 +当前不按 `metadata_type` 做差异化处理。也就是说，`paper`、`ebook`、未来新增类型都使用相同的 title、abstract、keywords、author 完整性标准。Venue 是否存在及其来源可信度统一交给权威性指标，避免对 ebook 等类型重复惩罚。 ### 6.2 字段权重 @@ -211,25 +226,25 @@ QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR ```text Effectiveness = - title_score * 0.25 -+ abstract_score * 0.45 -+ keywords_score * 0.15 -+ venue_score * 0.15 + title_score * 0.30 ++ abstract_score * 0.50 ++ keywords_score * 0.10 ++ author_score * 0.10 ``` | 子项 | 权重 | 业务含义 | |---|---:|---| -| `title_score` | 0.25 | 标题是否存在、长度是否合理、是否可读 | -| `abstract_score` | 0.45 | 摘要是否存在、信息量是否充足、是否可读 | -| `keywords_score` | 0.15 | 关键词是否存在、是否提供主题信息 | -| `venue_score` | 0.15 | 期刊/会议/来源名称是否存在、是否可读 | +| `title_score` | 0.30 | 标题是否存在、长度是否合理、是否可读 | +| `abstract_score` | 0.50 | 摘要是否存在、信息量是否充足、是否可读 | +| `keywords_score` | 0.10 | 关键词是否存在、是否提供主题信息 | +| `author_score` | 0.10 | 是否至少存在一个可识别的作者姓名；不按作者数量额外加分 | 字段为空时，该字段直接得 0 分。 示例：如果 result 只有标题，其他字段为空，且 `title_score=0.32948`： ```text -0.32948 * 0.25 + 0 + 0 + 0 = 0.08237 +0.32948 * 0.30 + 0 + 0 + 0 = 0.09884 ``` ### 6.3 字段评分逻辑 @@ -237,7 +252,8 @@ Effectiveness = 每个字段先做基础质量判断： - 为空：0 分。 -- 字段内容异常先由规则筛选，再由 LLM 判断：如果存在 HTML 泄漏、乱码、不可见字符、严重特殊字符噪声等，会按 LLM 字段质量分降低该字段分数。乱码筛选包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 +- title、abstract、keywords、author 四个字段参与有效性评分并进行缺失检查。venue 不参与有效性加权，也不会因为缺失产生问题；其来源可信度由权威性指标负责。 +- title、abstract、keywords、venue、author 五个字段都会进行异常检查。venue 有值时仍检查 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声。HTML 标签和 Unicode 替换字符 `�` 出现即进入 LLM 二次确认；其他异常字符按规则阈值筛选。乱码筛选还包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 - 长度太短：低分。 - 长度和信息量达到要求：接近或等于 1 分。 @@ -252,6 +268,10 @@ venue source ``` +`author` 兼容 `author`、`authors` 字段以及字符串、对象、对象列表等常见结构。存在至少一个含中文或英文字母、长度不少于 2 个字符的作者姓名时，作者基础分为 1；作者数量不会提高分数。 + +对比不同检索后端时，需要把后端原始作者信息统一映射到 `author` 或 `authors`。未映射作者字段会被视为缺失并使单条 result 的有效性总分降低 `0.10`。 + ### 6.4 Issues 类型 | issue | 含义 | @@ -259,8 +279,8 @@ source | `missing_title` | 标题为空 | | `missing_abstract` | 摘要为空 | | `missing_keywords` | 关键词为空 | -| `missing_venue` | 期刊/会议/来源名为空 | -| `title:html_tag` / `abstract:html_tag` / `venue:html_tag` | LLM 判断字段中有 HTML/XML 标签泄漏 | +| `missing_author` | 作者为空 | +| `title:html_tag` / `abstract:html_tag` / `venue:html_tag` / `author:html_tag` | LLM 判断字段中有 HTML/XML 标签泄漏 | | `*:mojibake` | LLM 判断字段存在乱码或编码错误 | | `*:invisible_char` | LLM 判断字段存在不可见/控制字符 | | `*:unreadable_text` | LLM 判断字段整体不可读 | @@ -272,7 +292,7 @@ source - `RuleSpecialCharacter` 和 `RuleInvisibleChar` 只用于快速召回疑似异常字段，不直接作为最终扣分依据。 - LaTeX、化学符号、单位、希腊字母、`|` 分隔符等正常学术表达不应被 LLM 判为问题。 - HTML 高亮标签、明显 mojibake、不可见字符、严重乱码会由 LLM 输出字段级 issue，并降低对应字段分数。 -- 分析 bad 样本时建议结合原始 title、abstract、venue 和 `llm_quality_reason` 进行人工抽查。 +- 分析 bad 样本时建议结合原始 title、abstract、keywords、venue、author 和 `llm_quality_reason` 进行人工抽查。 ### 6.5 可调参数 @@ -350,19 +370,29 @@ source | 条件 | `venue_score` | reason | |---|---:|---| -| venue 名称包含高权威来源提示词 | 0.85 | `high_authority_venue_hint` | -| `publication_venue_type` 是 journal 或 conference | 0.65 | `journal_or_conference` | -| repository 或 preprint | 0.45 | `repository_or_preprint` | +| 已知 repository 或 preprint（优先判断） | 0.45 | `repository_or_preprint` | +| 明确的学术 book series 或 ebook platform（优先判断） | 0.55 | `academic_book_series` | +| 明确命中权威期刊/会议家族 | 0.85 | `prestigious_venue_family` | +| 命中正规学术出版机构或来源组织 | 0.75 | `recognized_scholarly_publisher_or_venue` | +| `publication_venue_type` 是 journal/conference，或存在有效 ISSN | 0.65 | `structured_journal_or_conference` | +| 只有非空来源名称 | 0.40 | `named_venue` | | 未知或低信号来源 | 0.25 | `unknown_or_low_signal_venue` | -高权威来源提示词包括： +权威期刊家族使用带边界的规则匹配，包括： ```text -nature, science, cell, nejm, lancet, jama, -acm, ieee, springer, elsevier, wiley, -neurips, icml, iclr, cvpr, acl, emnlp, aaai, ijcai, sigir +Nature 及 Nature 学科子刊、Nature Communications、npj 系列、 +Communications 系列、Scientific Reports/Data、Science 官方期刊家族、 +部分 Cell Press 旗舰刊、NEJM、Lancet、JAMA、The BMJ、PNAS、 +JACS、PRL 以及主要 AI 会议 ``` +匹配前会移除 HTML 高亮标签，并按 `|` 拆分中英文来源别名。规则不再使用无限制的普通子串：`Science Translational Medicine` 会命中 Science 家族，而 `Chemical Engineering Science` 不会。 + +正规出版机构包括 Springer Nature、Elsevier、Wiley、Oxford University Press、Cambridge University Press、IEEE、ACM、ACS、RSC、IOP、BMJ、PLOS、Royal Society、De Gruyter、CRC Press、World Scientific 等。出版商只能提供正规学术来源信号，因此分数低于明确命中的旗舰期刊。 + +ISSN 和 journal/conference 类型是防止白名单漏判的主要兜底：专业期刊即使不在硬编码刊名列表中，只要具有结构化来源信息，也能获得 0.65，而不是被直接判成低信号来源。 + ### 7.5 DOI 评分 ```text @@ -491,7 +521,7 @@ python examples/retrieval/sdk_eval_search_result.py ` export OPENAI_API_KEY="" export OPENAI_BASE_URL="" export OPENAI_MODEL="deepseek-v4-flash" -export OPENAI_TEMPERATURE="0.7" +export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_relevancy.py \ --input-jsonl outputs/meta_search_97_query_results.jsonl \ @@ -510,7 +540,7 @@ Windows PowerShell 示例： $env:OPENAI_API_KEY="" $env:OPENAI_BASE_URL="" $env:OPENAI_MODEL="deepseek-v4-flash" -$env:OPENAI_TEMPERATURE="0.7" +$env:OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_relevancy.py ` --input-jsonl outputs/meta_search_97_query_results.jsonl ` @@ -529,7 +559,7 @@ python examples/retrieval/sdk_eval_relevancy.py ` export OPENAI_API_KEY="" export OPENAI_BASE_URL="" export OPENAI_MODEL="deepseek-v4-flash" -export OPENAI_TEMPERATURE="0.7" +export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_effectiveness.py \ --input-jsonl outputs/meta_search_97_query_results.jsonl \ @@ -559,6 +589,7 @@ python examples/retrieval/sdk_eval_authority.py \ export OPENAI_API_KEY="" export OPENAI_BASE_URL="" export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_search_result.py \ --input-jsonl outputs/meta_search_97_query_results.jsonl \ @@ -663,6 +694,6 @@ Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | 1. 相关性使用 LLM，temperature 大于 0 时，同一批数据重跑可能有轻微分数波动。 2. LLM 相关性结果可能出现 JSON 解析失败，脚本会记录 `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`。 -3. 内容有效性当前不按 `metadata_type` 放宽字段要求，因此 ebook 缺少 abstract、keywords、venue 时会低分。 +3. 内容有效性当前不按 `metadata_type` 放宽 title、abstract、keywords、author 的字段要求；venue 缺失不再降低有效性分数，由权威性指标统一判断。 4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。 5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。 diff --git a/docs/search_result_relevance_executor.md b/docs/search_result_relevance_executor.md index 949b08b4..12cef9b7 100644 --- a/docs/search_result_relevance_executor.md +++ b/docs/search_result_relevance_executor.md @@ -43,7 +43,12 @@ The repository includes `test/data/test_search_result.jsonl` for local smoke tes With the OpenAI-compatible environment variables configured, run: +For full-dataset evaluation, prefer a low-latency Flash model such as `deepseek-v4-flash`. A Pro model is better reserved for reviewing a small number of ambiguous samples because pointwise relevance evaluation makes approximately `query count x top-k` LLM calls. Keep the model and prompt fixed and use temperature `0` when comparing search versions. + ```powershell +$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_TEMPERATURE="0" + python examples/retrieval/sdk_eval_relevancy.py ` --input-jsonl test/data/test_search_result.jsonl ` --output-dir outputs/search_result_relevancy_smoke ` diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py index 7b309103..b7488b5e 100644 --- a/examples/retrieval/sdk_eval_search_result.py +++ b/examples/retrieval/sdk_eval_search_result.py @@ -19,7 +19,7 @@ if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402 +from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402,E501 from dingo.config import InputArgs # noqa: E402 from dingo.exec import Executor # noqa: E402 @@ -35,7 +35,7 @@ "Effectiveness.Error_Title_Miss": "missing_title", "Effectiveness.Error_Abstract_Miss": "missing_abstract", "Effectiveness.Error_Keywords_Miss": "missing_keywords", - "Effectiveness.Error_Venue_Miss": "missing_venue", + "Effectiveness.Error_Author_Miss": "missing_author", "Effectiveness.Error_HTML_Tag": "html_tag", "Effectiveness.Error_Mojibake": "mojibake", "Effectiveness.Error_Invisible_Char": "invisible_char", diff --git a/test/scripts/model/llm/test_llm_search_result_authority.py b/test/scripts/model/llm/test_llm_search_result_authority.py index 16cc2e51..88d5633b 100644 --- a/test/scripts/model/llm/test_llm_search_result_authority.py +++ b/test/scripts/model/llm/test_llm_search_result_authority.py @@ -22,3 +22,105 @@ def test_authority_grade_handles_non_serializable_location(): ) assert grade.doi_score == 0.0 + + +def test_nature_portfolio_families_are_recognized(): + grader = LLMSearchResultAuthority() + + for venue in ( + "Nature Physics", + "Nature Communications", + "npj Digital Medicine", + "Communications Biology", + "Scientific Reports", + "Scientific Data", + ): + grade = grader.grade(result={"publication_venue_name_unified": venue}) + assert grade.venue_score == 0.85 + assert grade.reason == "prestigious_venue_family" + + +def test_generic_science_names_do_not_match_science_family(): + grader = LLMSearchResultAuthority() + + named_only = grader.grade(result={"publication_venue_name_unified": "Grand Garden of Science"}) + structured = grader.grade( + result={ + "publication_venue_name_unified": "Chemical Engineering Science", + "publication_venue_type": "journal", + } + ) + + assert named_only.venue_score == 0.4 + assert named_only.reason == "named_venue" + assert structured.venue_score == 0.65 + assert structured.reason == "structured_journal_or_conference" + + +def test_repository_takes_priority_over_name_patterns(): + grade = LLMSearchResultAuthority().grade( + result={"publication_venue_name_unified": "Open Science Framework"}, + ) + + assert grade.venue_score == 0.45 + assert grade.reason == "repository_or_preprint" + + +def test_issn_and_recognized_publisher_provide_venue_fallbacks(): + grader = LLMSearchResultAuthority() + issn_grade = grader.grade( + result={ + "publication_venue_name_unified": "Specialist Research Journal", + "publication_venue_issn": ["1234-567X"], + } + ) + publisher_grade = grader.grade( + result={ + "publication_venue_name_unified": "Specialist Research Journal", + "publication_publisher": ["Oxford University Press"], + } + ) + + assert issn_grade.venue_score == 0.65 + assert issn_grade.reason == "structured_journal_or_conference" + assert publisher_grade.venue_score == 0.75 + assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue" + + +def test_html_highlight_is_removed_before_venue_matching(): + grade = LLMSearchResultAuthority().grade( + result={"publication_venue_name_unified": "Nature Medicine"}, + ) + + assert grade.venue_score == 0.85 + + +def test_ieee_is_recognized_inside_full_conference_name(): + grade = LLMSearchResultAuthority().grade( + result={ + "publication_venue_name_unified": ( + "13th IEEE International Workshops on Enabling Technologies" + ) + }, + ) + + assert grade.venue_score == 0.75 + assert grade.reason == "recognized_scholarly_publisher_or_venue" + + +def test_ebook_platform_and_academic_publisher_are_recognized(): + grader = LLMSearchResultAuthority() + ebook_grade = grader.grade( + result={ + "publication_venue_name_unified": "Springer eBooks", + "publication_venue_type": "ebook platform", + } + ) + publisher_grade = grader.grade( + result={"publication_publisher": ["Walter De Gruyter & Co"]}, + ) + + assert ebook_grade.venue_score == 0.55 + assert ebook_grade.reason == "academic_book_series" + assert publisher_grade.venue_score == 0.75 + assert publisher_grade.reason == "recognized_scholarly_publisher_or_venue" diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py index d289b1d6..441e27ec 100644 --- a/test/scripts/model/llm/test_llm_search_result_effectiveness.py +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -1,4 +1,13 @@ -from dingo.model.llm.llm_search_result_effectiveness import LLMSearchResultEffectiveness, _filter_llm_field_issues, _issues_to_labels, _looks_like_utf8_latin1_mojibake, _rule_abnormal_char_issues +import pytest + +from dingo.model.llm.llm_search_result_effectiveness import ( # isort: skip + LLMSearchResultEffectiveness, + _filter_llm_field_issues, + _issues_to_labels, + _looks_like_utf8_latin1_mojibake, + _rule_abnormal_char_issues, + extract_authors, +) def _mojibake(value: str) -> str: @@ -43,3 +52,99 @@ def test_rule_only_grade_penalizes_mojibake_fields(): assert grade.title_score == 0.1 assert grade.abstract_score == 0.1 assert grade.score < 0.5 + + +def test_extract_authors_supports_common_response_shapes(): + assert extract_authors({"author": [{"name": "Alice"}, {"display_name": "张三"}]}) == ["Alice", "张三"] + assert extract_authors({"authors": "Alice | Bob"}) == ["Alice", "Bob"] + assert extract_authors({"author": {"author_name": "Carol"}}) == ["Carol"] + + +def test_author_is_scored_without_rewarding_author_count(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + common = { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "venue": "International Journal of Search Quality Research", + } + + single_author = grader.grade(**common, authors=["Alice"]) + multiple_authors = grader.grade(**common, authors=["Alice", "Bob", "Carol"]) + missing_author = grader.grade(**common) + + assert single_author.author_score == 1.0 + assert multiple_authors.author_score == 1.0 + assert single_author.score == multiple_authors.score + assert missing_author.author_score == 0.0 + assert missing_author.score == pytest.approx(single_author.score - 0.1) + assert "missing_author" in missing_author.issues + assert _issues_to_labels(missing_author.issues) == ["Effectiveness.Error_Author_Miss"] + + +def test_missing_venue_is_diagnostic_only_and_does_not_reduce_score(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + common = { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "authors": ["Alice"], + } + + with_venue = grader.grade(**common, venue="International Journal of Search Quality Research") + without_venue = grader.grade(**common, venue="") + + assert with_venue.score == without_venue.score + assert without_venue.venue_score == 0.0 + assert "missing_venue" not in without_venue.issues + assert "Effectiveness.Error_Venue_Miss" not in _issues_to_labels(without_venue.issues) + + +def _complete_result() -> dict: + return { + "title": "A comprehensive evaluation of academic search result metadata", + "abstract": "academic search metadata provides useful information for readers " * 15, + "keywords": ["search", "metadata", "quality", "evaluation", "academic"], + "publication_venue_name_unified": "International Journal of Search Quality Research", + "author": [{"name": "Alice"}], + } + + +@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"]) +def test_all_effectiveness_fields_scan_html_residue(field: str): + result = _complete_result() + contaminated = "clean text leaked markup" + if field == "keywords": + result["keywords"] = [contaminated] + elif field == "venue": + result["publication_venue_name_unified"] = contaminated + elif field == "author": + result["author"] = [{"name": contaminated}] + elif field == "abstract": + result["abstract"] = "readable abstract text " * 100 + contaminated + else: + result[field] = contaminated + + grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result) + + assert "RuleSpecialCharacter" in grade.issues + assert getattr(grade, f"{field}_score") <= 0.1 + + +@pytest.mark.parametrize("field", ["title", "abstract", "keywords", "venue", "author"]) +def test_all_effectiveness_fields_scan_replacement_character(field: str): + result = _complete_result() + contaminated = "metadata contains \ufffd broken text" + if field == "keywords": + result["keywords"] = [contaminated] + elif field == "venue": + result["publication_venue_name_unified"] = contaminated + elif field == "author": + result["author"] = [{"name": contaminated}] + else: + result[field] = contaminated + + grade = LLMSearchResultEffectiveness(enable_llm_quality=False).grade(result=result) + + assert "RuleMojibake" in grade.issues + assert getattr(grade, f"{field}_score") <= 0.1 From 3f8a8f218b6816bcc94e65efb13c18953b90d461 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Thu, 16 Jul 2026 14:27:55 +0800 Subject: [PATCH 14/80] =?UTF-8?q?feat(retrieval):=20add=20search=20result?= =?UTF-8?q?=20quality=20evaluation=20=E4=BC=98=E5=8C=96=E4=BA=86=E6=9C=89?= =?UTF-8?q?=E6=95=88=E6=80=A7=E8=AF=84=E6=B5=8B=EF=BC=9A=E5=8F=96=E6=B6=88?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E9=95=BF=E5=BA=A6=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../llm/llm_search_result_effectiveness.py | 47 ++++--------------- docs/search_result_effectiveness_executor.md | 6 ++- docs/search_result_quality_metrics.md | 25 +++++----- .../test_llm_search_result_effectiveness.py | 33 +++++++++++++ 4 files changed, 58 insertions(+), 53 deletions(-) diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index aa9ac9d0..e1da7af2 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -98,19 +98,9 @@ def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: return max(low, min(high, value)) -def _token_count(text: str) -> int: - return len(re.findall(r"[\w\u4e00-\u9fff]+", text or "")) - - -def _field_quality(text: Any, *, min_chars: int, good_chars: int) -> float: - value = str(text or "").strip() - if not value: - return 0.0 - if len(value) < min_chars: - return 0.25 - if len(value) >= good_chars: - return 1.0 - return _clamp(0.35 + 0.65 * (len(value) - min_chars) / max(1, good_chars - min_chars)) +def _presence_quality(value: Any) -> float: + """Score field presence without using content length as a quality proxy.""" + return 1.0 if str(value or "").strip() else 0.0 def _suspicious_latin1_mojibake_count(text: str) -> int: @@ -363,17 +353,6 @@ def extract_authors(result: dict[str, Any]) -> list[str]: return [author for author in authors if author] -def _author_quality(authors: list[str]) -> float: - if not authors: - return 0.0 - valid_count = sum( - 1 - for author in authors - if len(author.strip()) >= 2 and re.search(r"[A-Za-z\u4e00-\u9fff]", author) - ) - return _clamp(valid_count / len(authors)) - - @dataclass class LLMFieldQuality: """LLM readability and corruption judgment for one search result.""" @@ -643,21 +622,11 @@ def grade( else [str(item).strip() for item in (authors or []) if str(item).strip()] ) - title_score = _field_quality(title, min_chars=6, good_chars=35) - if _token_count(title) <= 2 and len(str(title).strip()) < 15: - title_score *= 0.65 - - abstract_score = _field_quality(abstract, min_chars=80, good_chars=700) - if abstract and _token_count(abstract) < 25: - abstract_score *= 0.7 - - keywords_score = _clamp(len(keyword_items) / 5.0) if keyword_items else 0.0 - - venue_score = _field_quality(venue, min_chars=3, good_chars=30) - if venue and not re.search(r"[A-Za-z\u4e00-\u9fff]", venue): - venue_score *= 0.4 - - author_score = _author_quality(author_items) + title_score = _presence_quality(title) + abstract_score = _presence_quality(abstract) + keywords_score = 1.0 if keyword_items else 0.0 + venue_score = _presence_quality(venue) + author_score = 1.0 if author_items else 0.0 issues: list[str] = [] if not str(title or "").strip(): diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md index 453b3996..432e0d4f 100644 --- a/docs/search_result_effectiveness_executor.md +++ b/docs/search_result_effectiveness_executor.md @@ -145,6 +145,8 @@ Effectiveness = + author_score * 0.10 ``` -`author_score` checks whether at least one recognizable author name is present. A valid single-author paper receives full author completeness credit; the metric does not reward a larger author count. +Each non-empty title, abstract, keywords list, or author list receives full presence credit before corruption checks. Character length, token count, keyword count, and author count do not affect the base score. A confirmed HTML, mojibake, invisible-character, unreadable-text, or special-character issue can still reduce the affected field score. -Title, abstract, keywords, and author use missing-field checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. +`author_score` checks whether at least one non-empty author value is present. A single author receives full presence credit; the metric does not reward a larger author count or penalize a short name. + +Title, abstract, keywords, and author use presence checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index 653ba60a..86cbf054 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -9,7 +9,7 @@ | 指标 | 业务问题 | 评测方式 | |---|---|---| | 相关性 `relevance` | 检索结果是否回答了用户 query 的真实检索意图 | LLM 逐条判断 query-result 匹配程度 | -| 内容有效性 `effectiveness` | 结果记录本身是否完整、可读、可用于判断论文价值 | 规则检查字段缺失/信息量，RuleSpecialCharacter/RuleInvisibleChar 初筛异常候选，LLM 二次确认 | +| 内容有效性 `effectiveness` | 结果记录本身是否完整、可读、可用于判断论文价值 | 规则检查字段缺失，RuleSpecialCharacter/RuleInvisibleChar 初筛异常候选，LLM 二次确认；不使用字段长度打分 | | 权威性 `authority` | 结果是否具备学术可信度和来源影响力信号 | 规则检查 citation、influential citation、venue、DOI | 三个指标关注点不同： @@ -234,17 +234,17 @@ Effectiveness = | 子项 | 权重 | 业务含义 | |---|---:|---| -| `title_score` | 0.30 | 标题是否存在、长度是否合理、是否可读 | -| `abstract_score` | 0.50 | 摘要是否存在、信息量是否充足、是否可读 | -| `keywords_score` | 0.10 | 关键词是否存在、是否提供主题信息 | -| `author_score` | 0.10 | 是否至少存在一个可识别的作者姓名；不按作者数量额外加分 | +| `title_score` | 0.30 | 标题是否存在、是否可读 | +| `abstract_score` | 0.50 | 摘要是否存在、是否可读 | +| `keywords_score` | 0.10 | 是否至少存在一个非空关键词 | +| `author_score` | 0.10 | 是否至少存在一个非空作者值；不按作者数量额外加分 | 字段为空时，该字段直接得 0 分。 -示例：如果 result 只有标题，其他字段为空，且 `title_score=0.32948`： +示例：如果 result 只有非空且无异常的标题，其他字段为空，则 `title_score=1.0`： ```text -0.32948 * 0.30 + 0 + 0 + 0 = 0.09884 +1.0 * 0.30 + 0 + 0 + 0 = 0.30 ``` ### 6.3 字段评分逻辑 @@ -254,10 +254,11 @@ Effectiveness = - 为空：0 分。 - title、abstract、keywords、author 四个字段参与有效性评分并进行缺失检查。venue 不参与有效性加权，也不会因为缺失产生问题；其来源可信度由权威性指标负责。 - title、abstract、keywords、venue、author 五个字段都会进行异常检查。venue 有值时仍检查 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声。HTML 标签和 Unicode 替换字符 `�` 出现即进入 LLM 二次确认；其他异常字符按规则阈值筛选。乱码筛选还包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 -- 长度太短：低分。 -- 长度和信息量达到要求：接近或等于 1 分。 +- 字段非空：基础分为 1 分，不因字符长度、token 数、关键词数量或作者数量增减。 +- 字段存在 HTML 泄漏、乱码、不可见字符、不可读文本或严重特殊字符噪声：规则先召回候选，LLM 确认后降低相应字段分数。 +- 长文本不会获得额外加分，短文本也不会仅因长度被扣分。 -`keywords` 会把列表中的每个 keyword 视为一个主题信号；空列表计为缺失。 +`keywords` 只判断是否至少有一个非空值；空列表计为缺失，关键词数量不影响分数。 `venue` 读取优先级： @@ -268,7 +269,7 @@ venue source ``` -`author` 兼容 `author`、`authors` 字段以及字符串、对象、对象列表等常见结构。存在至少一个含中文或英文字母、长度不少于 2 个字符的作者姓名时，作者基础分为 1；作者数量不会提高分数。 +`author` 兼容 `author`、`authors` 字段以及字符串、对象、对象列表等常见结构。存在至少一个非空作者值时，作者基础分为 1；作者名称长度和作者数量不会影响基础分。乱码、HTML 或特殊字符噪声仍会进入异常检查。 对比不同检索后端时，需要把后端原始作者信息统一映射到 `author` 或 `authors`。未映射作者字段会被视为缺失并使单条 result 的有效性总分降低 `0.10`。 @@ -303,7 +304,7 @@ source | `--llm-max-tokens` | 1024 | 内容有效性 LLM 字段质量判断的最大 token 数 | | `--llm-workers` | 4 | 内容有效性 LLM 二次复核并发数 | | `--llm-timeout` | 60 | LLM 请求超时秒数 | -| `--disable-llm-quality` | false | 关闭 LLM 字段质量判断，仅保留字段缺失/长度规则 | +| `--disable-llm-quality` | false | 关闭 LLM 字段质量判断；保留字段缺失检查，并由规则候选直接触发异常扣分 | 阈值建议： diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py index 441e27ec..8001d2b8 100644 --- a/test/scripts/model/llm/test_llm_search_result_effectiveness.py +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -82,6 +82,39 @@ def test_author_is_scored_without_rewarding_author_count(): assert _issues_to_labels(missing_author.issues) == ["Effectiveness.Error_Author_Miss"] +def test_nonempty_fields_are_not_penalized_for_length_or_item_count(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + + grade = grader.grade( + title="D", + abstract="短", + keywords=["AI"], + venue="J", + authors=["Q"], + ) + + assert grade.title_score == 1.0 + assert grade.abstract_score == 1.0 + assert grade.keywords_score == 1.0 + assert grade.venue_score == 1.0 + assert grade.author_score == 1.0 + assert grade.score == 1.0 + assert grade.issues == [] + + +def test_longer_content_does_not_receive_more_effectiveness_credit(): + grader = LLMSearchResultEffectiveness(enable_llm_quality=False) + short = grader.grade(title="D", abstract="A", keywords=["K"], authors=["Q"]) + long = grader.grade( + title="A comprehensive academic title", + abstract="A complete and readable abstract. " * 100, + keywords=["one", "two", "three", "four", "five"], + authors=["Alice", "Bob"], + ) + + assert short.score == long.score == 1.0 + + def test_missing_venue_is_diagnostic_only_and_does_not_reduce_score(): grader = LLMSearchResultEffectiveness(enable_llm_quality=False) common = { From 2f8c2713f393b293eac8040ba15e96711349f50e Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Tue, 21 Jul 2026 10:48:01 +0800 Subject: [PATCH 15/80] =?UTF-8?q?feat(retrieval):=20add=20search=20result?= =?UTF-8?q?=20quality=20evaluation=20fix=EF=BC=9A=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E4=BA=86LLM=E7=9A=84jsonl=E8=A7=A3=E6=9E=90=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../llm/llm_search_result_effectiveness.py | 16 ++++---- docs/search_result_effectiveness_executor.md | 2 + docs/search_result_quality_metrics.md | 1 + .../test_llm_search_result_effectiveness.py | 39 +++++++++++++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index e1da7af2..86e76ae2 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -1,10 +1,9 @@ """Search result effectiveness grader. -This grader scores whether a returned search result has enough usable -bibliographic content for a user to judge and consume it. Missing-field and -basic information-density checks are deterministic. Readability and corruption -checks can be delegated to an LLM judge to avoid over-penalizing normal academic -formulas, units, and symbols. +This grader scores whether a returned search result has usable bibliographic +content for a user to judge and consume it. Field-presence checks are +deterministic. Readability and corruption checks can be delegated to an LLM +judge to avoid over-penalizing normal academic formulas, units, and symbols. It intentionally does not judge topical relevance; use ``LLMSearchResultRelevance`` for that. @@ -27,17 +26,20 @@ logger = logging.getLogger(__name__) +# Require a tag name immediately after ``<`` (or ```` as HTML markup. +HTML_TAG_PATTERN = r"]*?)?\s*/?>" + RULE_SPECIAL_CHARACTER_PATTERNS = ( r"u200e", r"÷|\? :", r"[锟解枴閿熻В鏋碷�]|\{\/U\}", r"U\+26[0-F][0-D]|U\+273[3-4]|U\+1F[3-6][0-4][0-F]|U\+1F6[8-F][0-F]", r"<\|.*?\|>", - r"<[^>]+>", + HTML_TAG_PATTERN, ) RULE_INVISIBLE_CHAR_PATTERN = r"[\u0080-\u009F\u2000-\u200F\u202F\u205F\u3000\uFEFF\u00A0\u2060-\u206F\uFEFF\xa0]" RULE_ABNORMAL_CHAR_THRESHOLD = 0.01 -HTML_TAG_PATTERN = r"<[^>]+>" MOJIBAKE_EVIDENCE_PATTERN = r"[閿熻В鏋撮柨鐔恍掗弸纰凤拷�]|\{\/U\}|u[0-9a-fA-F]{4}" UTF8_LATIN1_SEQUENCE_PATTERN = re.compile(r"[\u00C2\u00C3\u00D0\u00D1][\u0080-\u00BF]") C1_CONTROL_PATTERN = re.compile(r"[\u0080-\u009F]") diff --git a/docs/search_result_effectiveness_executor.md b/docs/search_result_effectiveness_executor.md index 432e0d4f..234def47 100644 --- a/docs/search_result_effectiveness_executor.md +++ b/docs/search_result_effectiveness_executor.md @@ -150,3 +150,5 @@ Each non-empty title, abstract, keywords list, or author list receives full pres `author_score` checks whether at least one non-empty author value is present. A single author receives full presence credit; the metric does not reward a larger author count or penalize a short name. Title, abstract, keywords, and author use presence checks and contribute to the score. Venue presence and quality are evaluated by the authority metric, so a missing venue does not reduce effectiveness or emit `Error_Venue_Miss`. All five text fields, including a venue when present, still use abnormal-character handling. HTML tags and the Unicode replacement character (`�`) always trigger LLM review, even when they occupy less than the general abnormal-character threshold. `RuleSpecialCharacter` and `RuleInvisibleChar` are fast candidates. When LLM quality judgment is enabled, those candidates are penalized only after LLM confirmation. + +The HTML candidate pattern requires a valid tag name immediately after `<` or `` is not treated as markup, while real academic formatting tags such as ``, ``, ``, and `` remain detectable. diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index 86cbf054..b24b4e38 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -254,6 +254,7 @@ Effectiveness = - 为空：0 分。 - title、abstract、keywords、author 四个字段参与有效性评分并进行缺失检查。venue 不参与有效性加权，也不会因为缺失产生问题；其来源可信度由权威性指标负责。 - title、abstract、keywords、venue、author 五个字段都会进行异常检查。venue 有值时仍检查 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声。HTML 标签和 Unicode 替换字符 `�` 出现即进入 LLM 二次确认；其他异常字符按规则阈值筛选。乱码筛选还包括 UTF-8 被误按 Latin-1 解码产生的 `Ð...`、`Ñ...` 序列及 C1 控制字符。 +- HTML 初筛要求 `<` 后直接出现合法标签名；`< Previous page | Next page >` 等翻页或比较文本不视为 HTML，``、``、``、`` 等真实标签仍进入 LLM 二次确认。 - 字段非空：基础分为 1 分，不因字符长度、token 数、关键词数量或作者数量增减。 - 字段存在 HTML 泄漏、乱码、不可见字符、不可读文本或严重特殊字符噪声：规则先召回候选，LLM 确认后降低相应字段分数。 - 长文本不会获得额外加分，短文本也不会仅因长度被扣分。 diff --git a/test/scripts/model/llm/test_llm_search_result_effectiveness.py b/test/scripts/model/llm/test_llm_search_result_effectiveness.py index 8001d2b8..9ac95e8c 100644 --- a/test/scripts/model/llm/test_llm_search_result_effectiveness.py +++ b/test/scripts/model/llm/test_llm_search_result_effectiveness.py @@ -115,6 +115,45 @@ def test_longer_content_does_not_receive_more_effectiveness_credit(): assert short.score == long.score == 1.0 +def test_preview_navigation_text_is_not_treated_as_html(): + abstract = ( + "Preview this article: Meaning and the Structure of Language, by Wallace Chafe, " + "Page 1 of 1 < Previous page | Next page > " + "/docserver/preview/fulltext/ce/33/8/collegeenglish18315-1.gif" + ) + + assert "RuleSpecialCharacter" not in _rule_abnormal_char_issues(abstract) + + # LLM quality is enabled deliberately: this text should bypass the LLM + # because it is not an abnormal-character candidate. + grade = LLMSearchResultEffectiveness(enable_llm_quality=True).grade( + title="Meaning and the Structure of Language, by Wallace Chafe", + abstract=abstract, + keywords=["Linguistics"], + venue="College English", + authors=["Frank Heny"], + ) + + assert grade.score == 1.0 + assert grade.issues == [] + + +@pytest.mark.parametrize( + "markup", + [ + "language", + "language", + "H2O", + "AM", + ], +) +def test_real_academic_html_tags_remain_detectable(markup: str): + assert "RuleSpecialCharacter" in _rule_abnormal_char_issues(markup) + assert _filter_llm_field_issues("title", markup, ["title:html_tag"]) == [ + "title:html_tag" + ] + + def test_missing_venue_is_diagnostic_only_and_does_not_reduce_score(): grader = LLMSearchResultEffectiveness(enable_llm_quality=False) common = { From 463e2e24bb7bb4c2b003a18bb1efef4630f85236 Mon Sep 17 00:00:00 2001 From: shijin Date: Wed, 22 Jul 2026 14:44:55 +0800 Subject: [PATCH 16/80] feat: update rule --- docs/rules.md | 112 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/docs/rules.md b/docs/rules.md index e8d5a023..a325657c 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -2,26 +2,94 @@ The specific rules for each quality metric are as follows: | Function Name | Type | Description | Reference | |------------------------------|-------------------|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| RuleAlphaWords | EFFECTIVENESS | check whether the ratio of words that contain at least one alphabetic character > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleCapitalWords | UNDERSTANDABILITY | check whether capital words ratio > 0.2 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | -| RuleCharNumber | EFFECTIVENESS | check whether the number of char > 100 | [MAP-en](https://arxiv.org/abs/2405.19327) | -| RuleColonEnd | COMPLETENESS | check whether the last char is ':' | | -| RuleContentNull | EFFECTIVENESS | check whether content is null | | -| RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) | -| RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) | -| RuleHtmlEntity | RELEVANCE | check whether content has html entity | | -| RuleIDCard | SECURITY | check if the content contains ID card. | | -| RuleLineEndWithEllipsis | COMPLETENESS | check whether the ratio of line ends with ellipsis < 0.3 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleLineEndWithTerminal | COMPLETENESS | check whether the ratio of line ends with terminal punctuation mark > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | +| RuleAbnormalChar | EFFECTIVENESS | Check whether content contains abnormal characters. | | +| RuleAbnormalHtml | EFFECTIVENESS | Check whether content contains abnormal HTML. | | +| RuleAbnormalNumber | FLUENCY | Check PDF content for abnormal page or index numbers. | | +| RuleAgentTraceLatencyAnomaly | AGENT_TRACE_QUALITY | Detect abnormally slow agent steps using statistical outlier analysis. | | +| RuleAgentTraceLoopDetection | AGENT_TRACE_QUALITY | Detect repetitive tool-call patterns that indicate loops. | | +| RuleAgentTraceTokenBudget | AGENT_TRACE_QUALITY | Check whether agent token usage exceeds the configured budget. | | +| RuleAlphaWords | EFFECTIVENESS | check whether the ratio of words that contain at least one alphabetic character > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleAudioDataFormat | EFFECTIVENESS | Check whether audio data has the expected format. | | +| RuleAudioDuration | EFFECTIVENESS | Check whether audio duration meets the configured standard. | | +| RuleAudioSnrQuality | EFFECTIVENESS | Check whether the audio signal-to-noise ratio meets the configured standard. | | +| RuleAuthorFieldValidation | EFFECTIVENESS | Validate scientific metadata author fields. | | +| RuleCapitalWords | UNDERSTANDABILITY | check whether capital words ratio > 0.2 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | +| RuleCharNumber | EFFECTIVENESS | check whether the number of char > 100 | [MAP-en](https://arxiv.org/abs/2405.19327) | +| RuleCharSplit | FLUENCY | Check PDF content for abnormally split characters. | | +| RuleColonEnd | COMPLETENESS | check whether the last char is ':' | | +| RuleContentNull | EFFECTIVENESS | check whether content is null | | +| RuleContentShort | EFFECTIVENESS | Check whether content is too short. | | +| RuleContentShortMultiLan | EFFECTIVENESS | Check whether multilingual content is too short. | | +| RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) | +| RuleDictConsistency | EFFECTIVENESS | Compare two dictionary fields and report mismatched keys. | | +| RuleDocFormulaRepeat | SIMILARITY | Check whether formulas repeat in a document. | | +| RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) | +| RuleDoi | EFFECTIVENESS | Validate DOI metadata. | | +| RuleEnterAndSpace | EFFECTIVENESS | Check abnormal combinations of line breaks and spaces. | | +| RuleEnterMore | EFFECTIVENESS | Check whether content has excessive consecutive line breaks. | | +| RuleEnterRatioMore | EFFECTIVENESS | Check whether the line-break ratio is excessive. | | +| RuleHallucinationHHEM | HALLUCINATION | Detect hallucinations with HHEM-2.1-Open. | | +| RuleHeadWordAr | RELEVANCE | Check Arabic content for irrelevant source information. | | +| RuleHeadWordCs | RELEVANCE | Check Czech content for irrelevant source information. | | +| RuleHeadWordHu | RELEVANCE | Check Hungarian content for irrelevant source information. | | +| RuleHeadWordKo | RELEVANCE | Check Korean content for irrelevant source information. | | +| RuleHeadWordRu | RELEVANCE | Check Russian content for irrelevant source information. | | +| RuleHeadWordSr | RELEVANCE | Check Serbian content for irrelevant source information. | | +| RuleHeadWordTh | RELEVANCE | Check Thai content for irrelevant source information. | | +| RuleHeadWordVi | RELEVANCE | Check Vietnamese content for irrelevant source information. | | +| RuleHtmlEntity | RELEVANCE | check whether content has html entity | | +| RuleHtmlTag | EFFECTIVENESS | Check whether content contains image links or HTML tags. | | +| RuleIDCard | SECURITY | check if the content contains ID card. | | +| RuleImageArtimuse | IMG_ARTIMUSE | Detect inappropriate artificial-image usage. | | +| RuleImageDataFormat | EFFECTIVENESS | Check whether image data has the expected format. | | +| RuleImageLabelOverlap | IMG_LABEL_OVERLAP | Check whether image labels overlap. | | +| RuleImageLabelVisualization | IMG_LABEL_VISUALIZATION | Visualize and validate image labels. | | +| RuleImageQuality | IMG_EFFECTIVENESS | Check whether image quality meets the configured standard. | | +| RuleImageRepeat | IMG_SIMILARITY | Detect duplicate images using perceptual hash or CNN features. | | +| RuleImageSizeValid | IMG_EFFECTIVENESS | Check whether image dimensions and aspect ratio are valid. | | +| RuleImageTextSimilarity | IMG_RELEVANCE | Check similarity between an image and its text content. | | +| RuleImageValid | IMG_EFFECTIVENESS | Check whether an image is valid and not uniformly white or black. | | +| RuleInvisibleChar | EFFECTIVENESS | Check whether content contains invisible characters. | | +| RuleIsbn | EFFECTIVENESS | Validate ISBN metadata. | | +| RuleLatexSpecialChar | EFFECTIVENESS | Check PDF content for abnormal LaTeX characters. | | +| RuleLineEndWithEllipsis | COMPLETENESS | check whether the ratio of line ends with ellipsis < 0.3 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleLineEndWithTerminal | COMPLETENESS | check whether the ratio of line ends with terminal punctuation mark > 0.6 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | +| RuleLineJavascriptCount | EFFECTIVENESS | check whether line with the word Javascript. | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | | RuleLineStartWithBulletpoint | UNDERSTANDABILITY | check whether the ratio of line starts with bullet points < 0.9 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleLineJavascriptCount | EFFECTIVENESS | check whether line with the word Javascript. | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | -| RuleLoremIpsum | EFFECTIVENESS | check whether the ratio of lorem ipsum < 3e-08 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | -| RuleMeanWordLength | EFFECTIVENESS | check whether the mean length of word in [3, 10] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleNoPunc | FLUENCY | check whether paragraph has no punctuation. | | -| RuleSentenceNumber | COMPLETENESS | check whether the number of sentence in [3, 7500] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | -| RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | -| RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | -| RuleWatermark | RELEVANCE | check whether content has watermarks. | | -| RuleWordNumber | EFFECTIVENESS | check whether the number of word in [20, 100000] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleLoremIpsum | EFFECTIVENESS | check whether the ratio of lorem ipsum < 3e-08 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | +| RuleMeanWordLength | EFFECTIVENESS | check whether the mean length of word in [3, 10] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleMetadataSimilarity | EFFECTIVENESS | Compare the similarity of scientific metadata fields. | | +| RuleNlpDataFormat | EFFECTIVENESS | Check whether NLP data has the expected format. | | +| RuleNoPunc | FLUENCY | check whether paragraph has no punctuation. | | +| RuleOnlyUrl | EFFECTIVENESS | Check whether content consists only of a URL. | | +| RulePatternSearch | RELEVANCE | Search content using a user-provided pattern. | | +| RulePIIDetection | SECURITY | Detect personally identifiable information in text. | | +| RuleQuanliangFieldValidation | EFFECTIVENESS | Validate full-volume scientific metadata fields. | | +| RuleResumeDateFormat | RESUME_DATE | Check whether a resume uses inconsistent date formats. | | +| RuleResumeDetailedAddress | RESUME_PRIVACY | Check whether a resume contains a detailed address. | | +| RuleResumeEducationMissing | RESUME_COMPLETENESS | Check whether a resume is missing its education section. | | +| RuleResumeEmailMissing | RESUME_CONTACT | Check whether a resume is missing an email address. | | +| RuleResumeEmoji | RESUME_PROFESSIONALISM | Check whether a resume contains emoji. | | +| RuleResumeExcessiveWhitespace | RESUME_FORMAT | Check whether a resume contains excessive whitespace. | | +| RuleResumeExperienceMissing | RESUME_COMPLETENESS | Check whether a resume is missing work experience. | | +| RuleResumeIDCard | RESUME_PRIVACY | Check whether a resume contains a Chinese ID card number. | | +| RuleResumeInformal | RESUME_PROFESSIONALISM | Check whether a resume contains informal language. | | +| RuleResumeMarkdown | RESUME_FORMAT | Check whether a resume contains Markdown syntax errors. | | +| RuleResumeNameMissing | RESUME_STRUCTURE | Check whether a resume is missing a name in its first section. | | +| RuleResumePhoneFormat | RESUME_CONTACT | Check whether a phone number has an invalid format. | | +| RuleResumePhoneMissing | RESUME_CONTACT | Check whether a resume is missing a phone number. | | +| RuleResumeSectionMissing | RESUME_STRUCTURE | Check whether a resume is missing required sections. | | +| RuleSentenceNumber | COMPLETENESS | check whether the number of sentence in [3, 7500] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [C4](https://arxiv.org/abs/1910.10683) | +| RuleSftDataFormat | EFFECTIVENESS | Check whether supervised fine-tuning data has the expected format. | | +| RuleSourceFieldValidation | EFFECTIVENESS | Validate scientific metadata source fields. | | +| RuleSpaceMore | EFFECTIVENESS | Check whether content contains excessive spaces. | | +| RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | +| RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | +| RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | | +| RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | | +| RuleWatermark | RELEVANCE | check whether content has watermarks. | | +| RuleWordNumber | EFFECTIVENESS | check whether the number of word in [20, 100000] | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleWordSplit | FLUENCY | Check PDF content for abnormally split words. | | +| RuleWordStuck | FLUENCY | Check whether words are abnormally joined together. | | From 4d11f89a5d1959453cc907a79d28eab88fe81401 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 23 Jul 2026 15:41:52 +0800 Subject: [PATCH 17/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E5=9B=B0=E6=83=91=E7=A8=8B=E5=BA=A6=20PPL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_common.py | 188 ++++++++++++++++++++ docs/metrics.md | 2 +- docs/rules.md | 1 + examples/guobiao/text_perplexity.py | 32 ++++ test/scripts/model/rule/test_rule_common.py | 109 +++++++++++- 5 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 examples/guobiao/text_perplexity.py diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 4ab70635..6093a6e3 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -1,3 +1,5 @@ +import importlib.util +import math import re import string from typing import Tuple @@ -1634,6 +1636,192 @@ def eval(cls, input_data: Data) -> EvalDetail: return res +@Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain"]) +class RuleTextPerplexity(BaseRule): + """Check whether text perplexity exceeds the configured threshold. + + Perplexity is calculated with a causal language model. Lower values + indicate that the text is more fluent and predictable under the selected + model. Because perplexity values are model-dependent, callers can override + both ``dynamic_config.model`` and ``dynamic_config.threshold``. + """ + + _metric_info = { + "category": "Rule-Based TEXT Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "RuleTextPerplexity", + "description": ( + "Calculates text perplexity with a causal language model and " + "flags text whose PPL exceeds the configured threshold" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "" + } + + _required_fields = [RequiredField.CONTENT] + dynamic_config = EvaluatorRuleArgs( + threshold=100.0, + model="uer/gpt2-chinese-cluecorpussmall", + stride=512, + ) + + _model_name = None + _tokenizer = None + _model = None + + @classmethod + def _check_dependencies(cls): + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "RuleTextPerplexity requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + @classmethod + def _get_model_components(cls, model_name): + if ( + cls._model_name == model_name + and cls._tokenizer is not None + and cls._model is not None + ): + return cls._tokenizer, cls._model + + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as exc: + raise ImportError( + "RuleTextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + cls._tokenizer = AutoTokenizer.from_pretrained(model_name) + cls._model = AutoModelForCausalLM.from_pretrained(model_name) + cls._model.eval() + cls._model_name = model_name + return cls._tokenizer, cls._model + + @classmethod + def _calculate_perplexity(cls, content, tokenizer, model, stride): + try: + import torch + except ImportError as exc: + raise ImportError( + "RuleTextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + encodings = tokenizer(content, return_tensors="pt") + input_ids = encodings["input_ids"] + sequence_length = input_ids.size(1) + if sequence_length < 2: + raise ValueError( + "RuleTextPerplexity requires at least two model tokens" + ) + + model_config = getattr(model, "config", None) + max_length = getattr(model_config, "n_positions", None) + if max_length is None: + max_length = getattr(model_config, "max_position_embeddings", 1024) + max_length = int(max_length) + stride = max(1, min(int(stride), max_length)) + + try: + device = next(model.parameters()).device + except StopIteration: + device = torch.device("cpu") + + total_negative_log_likelihood = 0.0 + total_loss_tokens = 0 + previous_end = 0 + + for begin in range(0, sequence_length, stride): + end = min(begin + max_length, sequence_length) + target_length = end - previous_end + window = input_ids[:, begin:end].to(device) + targets = window.clone() + targets[:, :-target_length] = -100 + + with torch.no_grad(): + output = model(window, labels=targets) + + # Causal language models shift labels by one token internally. + loss_tokens = int((targets[:, 1:] != -100).sum().item()) + if loss_tokens > 0: + total_negative_log_likelihood += output.loss.item() * loss_tokens + total_loss_tokens += loss_tokens + + previous_end = end + if end == sequence_length: + break + + if total_loss_tokens == 0: + raise ValueError( + "RuleTextPerplexity could not calculate loss for the input" + ) + + mean_loss = total_negative_log_likelihood / total_loss_tokens + try: + return math.exp(mean_loss) + except OverflowError: + return float("inf") + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + cls._check_dependencies() + + res = EvalDetail(metric=cls.__name__) + content = input_data.content + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Text perplexity cannot be calculated for empty content"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or threshold <= 0: + raise ValueError( + "RuleTextPerplexity dynamic_config.threshold must be greater than 0" + ) + + model_name = getattr( + cls.dynamic_config, + "model", + "uer/gpt2-chinese-cluecorpussmall", + ) + stride = getattr(cls.dynamic_config, "stride", 512) + tokenizer, model = cls._get_model_components(model_name) + perplexity = cls._calculate_perplexity( + content, + tokenizer, + model, + stride, + ) + + if perplexity > threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Text perplexity {perplexity:.4f} exceeds threshold " + f"{threshold:.4f} (model: {model_name})" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Text perplexity: {perplexity:.4f} " + f"(threshold: {threshold:.4f}, model: {model_name})" + ] + return res + + @Model.rule_register( "QUALITY_BAD_EFFECTIVENESS", [ diff --git a/docs/metrics.md b/docs/metrics.md index 586d26f0..d662fb09 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -62,7 +62,7 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; Checks PDF content for abnormal ch... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | diff --git a/docs/rules.md b/docs/rules.md index a325657c..2b512bfc 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -86,6 +86,7 @@ The specific rules for each quality metric are as follows: | RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | | RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | | RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| RuleTextPerplexity | FLUENCY | Calculate text perplexity with a configurable causal language model and flag values above the configured threshold. | 2025 High-quality dataset quality evaluation specification | | RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | | RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | | | RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | | diff --git a/examples/guobiao/text_perplexity.py b/examples/guobiao/text_perplexity.py new file mode 100644 index 00000000..9d312bbb --- /dev/null +++ b/examples/guobiao/text_perplexity.py @@ -0,0 +1,32 @@ +"""Evaluate one Chinese text using the national-standard perplexity rule. + +Optional dependencies: + conda run -n dingo pip install "dingo-python[hhem]" + +The first run downloads the configured Hugging Face model unless it is already +available locally. Set ``MODEL_NAME`` to a local model directory to avoid a +download. +""" + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.model.rule.rule_common import RuleTextPerplexity + + +def main(): + data = Data( + data_id="guobiao-ppl-example", + content="人工智能正在推动科学研究和产业应用快速发展。高质量数据集能够为模型训练提供准确、完整且具有代表性的样本，从而提高模型在真实应用场景中的稳定性和可靠性。", + ) + + RuleTextPerplexity.dynamic_config = EvaluatorRuleArgs( + threshold=100.0, + model="uer/gpt2-chinese-cluecorpussmall", + stride=512, + ) + result = RuleTextPerplexity.eval(data) + print(result) + + +if __name__ == "__main__": + main() diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index bae3279d..0ffc8193 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -1,6 +1,12 @@ from dingo.io import Data +from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.output.eval_detail import QualityLabel -from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords +from dingo.model.rule.rule_common import ( + RuleDocFormulaRepeat, + RulePIIDetection, + RuleTextPerplexity, + RuleUnsafeWords, +) class TestRuleDocFormulaRepeat: @@ -24,6 +30,107 @@ def test_rule_unsafe_words(self): assert 'java' in tmp.reason +class TestRuleTextPerplexity: + @staticmethod + def _mock_model(monkeypatch, perplexity): + monkeypatch.setattr( + RuleTextPerplexity, + "_check_dependencies", + classmethod(lambda cls: None), + ) + monkeypatch.setattr( + RuleTextPerplexity, + "_get_model_components", + classmethod(lambda cls, model_name: (object(), object())), + ) + monkeypatch.setattr( + RuleTextPerplexity, + "_calculate_perplexity", + classmethod( + lambda cls, content, tokenizer, model, stride: perplexity + ), + ) + + def test_high_perplexity_is_bad(self, monkeypatch): + self._mock_model(monkeypatch, 125.5) + monkeypatch.setattr( + RuleTextPerplexity, + "dynamic_config", + EvaluatorRuleArgs( + threshold=100.0, + model="test-model", + stride=64, + ), + ) + + res = RuleTextPerplexity.eval( + Data(data_id="ppl-high", content="A valid piece of text.") + ) + + assert res.status is True + assert res.label == ["QUALITY_BAD_FLUENCY.RuleTextPerplexity"] + assert "125.5000" in res.reason[0] + assert "test-model" in res.reason[0] + + def test_low_perplexity_is_good(self, monkeypatch): + self._mock_model(monkeypatch, 42.25) + monkeypatch.setattr( + RuleTextPerplexity, + "dynamic_config", + EvaluatorRuleArgs( + threshold=100.0, + model="test-model", + stride=64, + ), + ) + + res = RuleTextPerplexity.eval( + Data(data_id="ppl-low", content="A fluent piece of text.") + ) + + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert "42.2500" in res.reason[0] + + def test_empty_content_is_bad_without_loading_model(self, monkeypatch): + monkeypatch.setattr( + RuleTextPerplexity, + "_check_dependencies", + classmethod(lambda cls: None), + ) + + def fail_if_called(cls, model_name): + raise AssertionError("model should not be loaded for empty content") + + monkeypatch.setattr( + RuleTextPerplexity, + "_get_model_components", + classmethod(fail_if_called), + ) + + res = RuleTextPerplexity.eval(Data(data_id="ppl-empty", content=" ")) + + assert res.status is True + assert res.label == ["QUALITY_BAD_FLUENCY.RuleTextPerplexity"] + assert "empty content" in res.reason[0] + + def test_missing_dependencies_raise_clear_error(self, monkeypatch): + monkeypatch.setattr( + "dingo.model.rule.rule_common.importlib.util.find_spec", + lambda package: None if package == "transformers" else object(), + ) + + try: + RuleTextPerplexity.eval( + Data(data_id="ppl-dependency", content="A piece of text.") + ) + except ImportError as exc: + assert "transformers" in str(exc) + assert "dingo-python[hhem]" in str(exc) + else: + raise AssertionError("expected ImportError for missing transformers") + + class TestRulePIIDetection: """PII 检测规则测试""" From e55f25600430f59a1d838858b4e4c2293ee2f707 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 24 Jul 2026 10:36:26 +0800 Subject: [PATCH 18/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E4=B8=80=E8=87=B4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + dingo/model/rule/rule_common.py | 188 ------------ dingo/model/rule/rule_guobiao.py | 323 ++++++++++++++++++++ docs/metrics.md | 58 ++-- docs/rules.md | 1 + examples/guobiao/text_perplexity.py | 2 +- examples/guobiao/type_consistency.py | 32 ++ test/scripts/model/rule/test_rule_common.py | 46 +++ 8 files changed, 433 insertions(+), 218 deletions(-) create mode 100644 dingo/model/rule/rule_guobiao.py create mode 100644 examples/guobiao/type_consistency.py diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index bf93d1c3..29a461e8 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -7,3 +7,4 @@ class RequiredField(Enum): CONTEXT = "context" IMAGE = "image" METADATA = "metadata" + TYPE = "type" diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 6093a6e3..4ab70635 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -1,5 +1,3 @@ -import importlib.util -import math import re import string from typing import Tuple @@ -1636,192 +1634,6 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain"]) -class RuleTextPerplexity(BaseRule): - """Check whether text perplexity exceeds the configured threshold. - - Perplexity is calculated with a causal language model. Lower values - indicate that the text is more fluent and predictable under the selected - model. Because perplexity values are model-dependent, callers can override - both ``dynamic_config.model`` and ``dynamic_config.threshold``. - """ - - _metric_info = { - "category": "Rule-Based TEXT Quality Metrics", - "quality_dimension": "FLUENCY", - "metric_name": "RuleTextPerplexity", - "description": ( - "Calculates text perplexity with a causal language model and " - "flags text whose PPL exceeds the configured threshold" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "" - } - - _required_fields = [RequiredField.CONTENT] - dynamic_config = EvaluatorRuleArgs( - threshold=100.0, - model="uer/gpt2-chinese-cluecorpussmall", - stride=512, - ) - - _model_name = None - _tokenizer = None - _model = None - - @classmethod - def _check_dependencies(cls): - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - "RuleTextPerplexity requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - @classmethod - def _get_model_components(cls, model_name): - if ( - cls._model_name == model_name - and cls._tokenizer is not None - and cls._model is not None - ): - return cls._tokenizer, cls._model - - try: - from transformers import AutoModelForCausalLM, AutoTokenizer - except ImportError as exc: - raise ImportError( - "RuleTextPerplexity requires transformers and torch. " - 'Install them with: pip install "dingo-python[hhem]"' - ) from exc - - cls._tokenizer = AutoTokenizer.from_pretrained(model_name) - cls._model = AutoModelForCausalLM.from_pretrained(model_name) - cls._model.eval() - cls._model_name = model_name - return cls._tokenizer, cls._model - - @classmethod - def _calculate_perplexity(cls, content, tokenizer, model, stride): - try: - import torch - except ImportError as exc: - raise ImportError( - "RuleTextPerplexity requires transformers and torch. " - 'Install them with: pip install "dingo-python[hhem]"' - ) from exc - - encodings = tokenizer(content, return_tensors="pt") - input_ids = encodings["input_ids"] - sequence_length = input_ids.size(1) - if sequence_length < 2: - raise ValueError( - "RuleTextPerplexity requires at least two model tokens" - ) - - model_config = getattr(model, "config", None) - max_length = getattr(model_config, "n_positions", None) - if max_length is None: - max_length = getattr(model_config, "max_position_embeddings", 1024) - max_length = int(max_length) - stride = max(1, min(int(stride), max_length)) - - try: - device = next(model.parameters()).device - except StopIteration: - device = torch.device("cpu") - - total_negative_log_likelihood = 0.0 - total_loss_tokens = 0 - previous_end = 0 - - for begin in range(0, sequence_length, stride): - end = min(begin + max_length, sequence_length) - target_length = end - previous_end - window = input_ids[:, begin:end].to(device) - targets = window.clone() - targets[:, :-target_length] = -100 - - with torch.no_grad(): - output = model(window, labels=targets) - - # Causal language models shift labels by one token internally. - loss_tokens = int((targets[:, 1:] != -100).sum().item()) - if loss_tokens > 0: - total_negative_log_likelihood += output.loss.item() * loss_tokens - total_loss_tokens += loss_tokens - - previous_end = end - if end == sequence_length: - break - - if total_loss_tokens == 0: - raise ValueError( - "RuleTextPerplexity could not calculate loss for the input" - ) - - mean_loss = total_negative_log_likelihood / total_loss_tokens - try: - return math.exp(mean_loss) - except OverflowError: - return float("inf") - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - cls._check_dependencies() - - res = EvalDetail(metric=cls.__name__) - content = input_data.content - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Text perplexity cannot be calculated for empty content"] - return res - - threshold = cls.dynamic_config.threshold - if threshold is None or threshold <= 0: - raise ValueError( - "RuleTextPerplexity dynamic_config.threshold must be greater than 0" - ) - - model_name = getattr( - cls.dynamic_config, - "model", - "uer/gpt2-chinese-cluecorpussmall", - ) - stride = getattr(cls.dynamic_config, "stride", 512) - tokenizer, model = cls._get_model_components(model_name) - perplexity = cls._calculate_perplexity( - content, - tokenizer, - model, - stride, - ) - - if perplexity > threshold: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"Text perplexity {perplexity:.4f} exceeds threshold " - f"{threshold:.4f} (model: {model_name})" - ] - else: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"Text perplexity: {perplexity:.4f} " - f"(threshold: {threshold:.4f}, model: {model_name})" - ] - return res - - @Model.rule_register( "QUALITY_BAD_EFFECTIVENESS", [ diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py new file mode 100644 index 00000000..3b2b183f --- /dev/null +++ b/dingo/model/rule/rule_guobiao.py @@ -0,0 +1,323 @@ +import importlib.util +import math + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.base import BaseRule + + +@Model.rule_register("QUALITY_BAD_TYPE_CONSISTENCY", ["guobiao"]) +class RuleDataTypeConsistency(BaseRule): + """Check whether content belongs to the type declared in ``input_data.type``. + + A local zero-shot classifier evaluates the hypothesis ``这段文本属于{type}类型``. + The declared type may be any non-empty string, such as ``医疗`` or ``金融``. + """ + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "TYPE_CONSISTENCY", + "metric_name": "RuleDataTypeConsistency", + "description": ( + "Uses a local zero-shot classifier to check whether content belongs " + "to the type declared in the record" + ), + "paper_title": "High-quality dataset classification guide", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + + _required_fields = [RequiredField.CONTENT, RequiredField.TYPE] + dynamic_config = EvaluatorRuleArgs( + threshold=0.5, + model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + device=-1, + ) + + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "RuleDataTypeConsistency requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_match_score( + cls, content, declared_type, model_name, device + ): + classifier = cls._get_classifier(model_name, device) + result = classifier( + content, + candidate_labels=[declared_type], + hypothesis_template="这段文本属于{}类型。", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != declared_type: + raise RuntimeError("Zero-shot classifier returned an invalid result") + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + declared_type = getattr(input_data, "type", None) + content = getattr(input_data, "content", None) + + if not isinstance(declared_type, str) or not declared_type.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Data type is missing or empty"] + return res + + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Content is missing or empty"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + "RuleDataTypeConsistency dynamic_config.threshold must be in (0, 1]" + ) + + model_name = cls.dynamic_config.model + device = cls.dynamic_config.device + score = cls._calculate_match_score( + content, declared_type, model_name, device + ) + + if score >= threshold: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Content matches declared type {declared_type} " + f"(score: {score:.4f}, threshold: {threshold:.4f})" + ] + else: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Content does not match declared type {declared_type} " + f"(score: {score:.4f}, threshold: {threshold:.4f})" + ] + return res + + +@Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain", "guobiao"]) +class RuleTextPerplexity(BaseRule): + """Check whether text perplexity exceeds the configured threshold.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "RuleTextPerplexity", + "description": ( + "Calculates text perplexity with a causal language model and " + "flags text whose PPL exceeds the configured threshold" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "" + } + + _required_fields = [RequiredField.CONTENT] + dynamic_config = EvaluatorRuleArgs( + threshold=100.0, + model="uer/gpt2-chinese-cluecorpussmall", + stride=512, + ) + + + _model_name = None + _tokenizer = None + _model = None + + @classmethod + def _check_dependencies(cls): + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "RuleTextPerplexity requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + @classmethod + def _get_model_components(cls, model_name): + if ( + cls._model_name == model_name + and cls._tokenizer is not None + and cls._model is not None + ): + return cls._tokenizer, cls._model + + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as exc: + raise ImportError( + "RuleTextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + cls._tokenizer = AutoTokenizer.from_pretrained(model_name) + cls._model = AutoModelForCausalLM.from_pretrained(model_name) + cls._model.eval() + cls._model_name = model_name + return cls._tokenizer, cls._model + + @classmethod + def _calculate_perplexity(cls, content, tokenizer, model, stride): + try: + import torch + except ImportError as exc: + raise ImportError( + "RuleTextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + encodings = tokenizer(content, return_tensors="pt") + input_ids = encodings["input_ids"] + sequence_length = input_ids.size(1) + if sequence_length < 2: + raise ValueError( + "RuleTextPerplexity requires at least two model tokens" + ) + + model_config = getattr(model, "config", None) + max_length = getattr(model_config, "n_positions", None) + if max_length is None: + max_length = getattr(model_config, "max_position_embeddings", 1024) + max_length = int(max_length) + stride = max(1, min(int(stride), max_length)) + + try: + device = next(model.parameters()).device + except StopIteration: + device = torch.device("cpu") + + total_negative_log_likelihood = 0.0 + total_loss_tokens = 0 + previous_end = 0 + + for begin in range(0, sequence_length, stride): + end = min(begin + max_length, sequence_length) + target_length = end - previous_end + window = input_ids[:, begin:end].to(device) + targets = window.clone() + targets[:, :-target_length] = -100 + + with torch.no_grad(): + output = model(window, labels=targets) + + loss_tokens = int((targets[:, 1:] != -100).sum().item()) + if loss_tokens > 0: + total_negative_log_likelihood += output.loss.item() * loss_tokens + total_loss_tokens += loss_tokens + + previous_end = end + if end == sequence_length: + break + + if total_loss_tokens == 0: + raise ValueError( + "RuleTextPerplexity could not calculate loss for the input" + ) + + mean_loss = total_negative_log_likelihood / total_loss_tokens + try: + return math.exp(mean_loss) + except OverflowError: + return float("inf") + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + cls._check_dependencies() + + res = EvalDetail(metric=cls.__name__) + content = input_data.content + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Text perplexity cannot be calculated for empty content"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or threshold <= 0: + raise ValueError( + "RuleTextPerplexity dynamic_config.threshold must be greater than 0" + ) + + model_name = getattr( + cls.dynamic_config, + "model", + "uer/gpt2-chinese-cluecorpussmall", + ) + stride = getattr(cls.dynamic_config, "stride", 512) + tokenizer, model = cls._get_model_components(model_name) + perplexity = cls._calculate_perplexity( + content, + tokenizer, + model, + stride, + ) + + if perplexity > threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Text perplexity {perplexity:.4f} exceeds threshold " + f"{threshold:.4f} (model: {model_name})" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Text perplexity: {perplexity:.4f} " + f"(threshold: {threshold:.4f}, model: {model_name})" + ] + return res diff --git a/docs/metrics.md b/docs/metrics.md index d662fb09..6c436115 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,11 +8,11 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 璇勪及绛旀鏄惁鐩存帴鍥炵瓟闂锛屾娴嬫棤鍏冲拰鍐椾綑淇℃伅 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 璇勪及妫€绱笂涓嬫枃鐨勭簿纭害锛屽寘鎷浉鍏虫€у拰鎺掑簭璐ㄩ噺 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRecall` | LLMRAGContextRecall | 璇勪及妫€绱笂涓嬫枃鐨勫畬鏁存€э紝鍒ゆ柇涓婁笅鏂囨槸鍚﹁兘鏀寔绛旀涓殑鎵€鏈夐檲杩?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 璇勪及妫€绱笂涓嬫枃涓庨棶棰樼殑鐩稿叧鎬э紝妫€娴嬪櫔澹颁俊鎭?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 璇勪及鐢熸垚绛旀鏄惁蹇犲疄浜庣粰瀹氫笂涓嬫枃锛屾娴嬪够瑙夊拰缂栭€犱俊鎭?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | ### Pretrain Text Quality Assessment Metrics @@ -25,10 +25,10 @@ This document provides comprehensive information about all quality metrics used | `LLMMathCompare` | LLMMathCompare | Compares the effectiveness of two tools in extracting mathematical formulas from HTML to Markdown format by evaluatin... | Internal Implementation | N/A | N/A | | `LLMSecurityPolitics` | LLMSecurityPolitics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | N/A | | `LLMTableCompare` | LLMTableCompare | Compares the effectiveness of two tools in extracting tables from HTML to Markdown format by evaluating recognition r... | Internal Implementation | N/A | N/A | -| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | -| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | +| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | ### SFT Data Assessment Metrics @@ -36,37 +36,38 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `LLMFactCheckPublic` | LLMFactCheckPublic | Two-stage factuality evaluation pipeline from GPT-5 | [GPT-5 System Card](https://cdn.openai.com/pdf/8124a3ce-ab78-4f06-96eb-49ea29ffb52f/gpt5-system-card-aug7.pdf) (OpenAI) | N/A | N/A | | `LLMHallucination` | LLMHallucination | Evaluates whether the response contains factual contradictions or hallucinations against provided context information | [TruthfulQA: Measuring How Models Mimic Human Falsehoods](https://arxiv.org/abs/2109.07958) (Lin et al., 2021) | N/A | N/A | -| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [📊 See Results](Returns clarity score (0-10) and detailed analysis) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [📊 See Results](Returns difficulty level (1-10) with detailed breakdown) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [馃搳 See Results](Returns clarity score (0-10) and detailed analysis) | [馃摑 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [馃搳 See Results](Returns difficulty level (1-10) with detailed breakdown) | [馃摑 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A | ### Classification Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [📊 See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | +| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [馃搳 See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | ### Multimodality Assessment Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `LLMClassifyQR` | LLMClassifyQR | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | N/A | -| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用DeepSeek-OCR作为Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [📊 See Results](通过对比VLM输出与OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | +| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 璇勪及澶氭ā鎬佹ā鍨嬪鍥剧墖涓枃瀛楀唴瀹圭殑璇嗗埆鍜岀悊瑙ｈ兘鍔涳紝浣跨敤DeepSeek-OCR浣滀负Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [馃搳 See Results](閫氳繃瀵规瘮VLM杈撳嚭涓嶰CR ground truth锛岃瘑鍒枃瀛楅仐婕忋€侀敊璇€佸够瑙夌瓑闂) | N/A | ### Rule-Based TEXT Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_TYPE_CONSISTENCY` | RuleDataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics @@ -105,14 +106,14 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | -| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | +| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [馃搳 See Results](error_category and error_label) | N/A | +| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [馃搳 See Results](error_category and error_label) | N/A | ### RAG Retrieved Evidence Chunk Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [📝 View Example](../examples/rag/sdk_chunk_eval.py) | +| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [馃摑 View Example](../examples/rag/sdk_chunk_eval.py) | ### Resume Quality Assessment Metrics @@ -126,7 +127,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 妫€鏌ュ厓鏁版嵁瀛楁涓庡熀鍑嗘暟鎹殑鐩镐技搴﹀尮閰嶏紝闃堝€奸粯璁や负0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics @@ -158,5 +159,4 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `AgentFactCheck` | AgentFactCheck | Agent-based hallucination detection with autonomous web search | Internal Implementation | N/A | N/A | | `ArticleFactChecker` | ArticleFactChecker | Article-level fact checking with autonomous claims extraction and verification | Internal Implementation | N/A | N/A | -| `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | - +| `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | \ No newline at end of file diff --git a/docs/rules.md b/docs/rules.md index 2b512bfc..d5e645af 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -21,6 +21,7 @@ The specific rules for each quality metric are as follows: | RuleContentShort | EFFECTIVENESS | Check whether content is too short. | | | RuleContentShortMultiLan | EFFECTIVENESS | Check whether multilingual content is too short. | | | RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) | +| RuleDataTypeConsistency | TYPE_CONSISTENCY | Use a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | 2025 High-quality dataset classification guide | | RuleDictConsistency | EFFECTIVENESS | Compare two dictionary fields and report mismatched keys. | | | RuleDocFormulaRepeat | SIMILARITY | Check whether formulas repeat in a document. | | | RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) | diff --git a/examples/guobiao/text_perplexity.py b/examples/guobiao/text_perplexity.py index 9d312bbb..7920819e 100644 --- a/examples/guobiao/text_perplexity.py +++ b/examples/guobiao/text_perplexity.py @@ -10,7 +10,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_common import RuleTextPerplexity +from dingo.model.rule.rule_guobiao import RuleTextPerplexity def main(): diff --git a/examples/guobiao/type_consistency.py b/examples/guobiao/type_consistency.py new file mode 100644 index 00000000..cfd5a66d --- /dev/null +++ b/examples/guobiao/type_consistency.py @@ -0,0 +1,32 @@ +"""Evaluate one Chinese text using the national-standard type-consistency rule. + +Optional dependencies: + conda run -n dingo pip install "dingo-python[hhem]" + +The first run downloads the configured Hugging Face model unless it is already +available locally. +""" + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.model.rule.rule_guobiao import RuleDataTypeConsistency + + +def main(): + data = Data( + data_id="guobiao-type-example", + type="医疗", + content="高血压患者应在医生指导下规律用药，并定期监测血压变化。", + ) + + RuleDataTypeConsistency.dynamic_config = EvaluatorRuleArgs( + threshold=0.5, + model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + device=-1, + ) + result = RuleDataTypeConsistency.eval(data) + print(result) + + +if __name__ == "__main__": + main() diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 0ffc8193..27f230dc 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -1,6 +1,8 @@ +import pytest from dingo.io import Data from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.output.eval_detail import QualityLabel +from dingo.model.rule.rule_guobiao import RuleDataTypeConsistency from dingo.model.rule.rule_common import ( RuleDocFormulaRepeat, RulePIIDetection, @@ -131,6 +133,50 @@ def test_missing_dependencies_raise_clear_error(self, monkeypatch): raise AssertionError("expected ImportError for missing transformers") +class TestRuleDataTypeConsistency: + @staticmethod + def _mock_match_score(monkeypatch, score): + monkeypatch.setattr( + RuleDataTypeConsistency, + "_calculate_match_score", + classmethod(lambda cls, *args: score), + ) + monkeypatch.setattr( + RuleDataTypeConsistency, + "dynamic_config", + EvaluatorRuleArgs(threshold=0.6, model="test-model", device=-1), + ) + + def test_content_matching_declared_type_is_good(self, monkeypatch): + self._mock_match_score(monkeypatch, 0.85) + result = RuleDataTypeConsistency.eval( + Data(data_id="type-match", type="medical", content="Clinical treatment") + ) + + assert result.status is False + assert result.score == 0.85 + assert result.label == [QualityLabel.QUALITY_GOOD] + + def test_content_not_matching_declared_type_is_bad(self, monkeypatch): + self._mock_match_score(monkeypatch, 0.25) + result = RuleDataTypeConsistency.eval( + Data(data_id="type-mismatch", type="medical", content="Stock prices") + ) + + assert result.status is True + assert result.score == 0.25 + assert result.label == [ + "QUALITY_BAD_TYPE_CONSISTENCY.RuleDataTypeConsistency" + ] + + def test_missing_type_is_bad(self): + result = RuleDataTypeConsistency.eval( + Data(data_id="type-missing", content="Ordinary text") + ) + + assert result.status is True + assert "missing or empty" in result.reason[0] + class TestRulePIIDetection: """PII 检测规则测试""" From 7dd93dc7ce3a45d948b551f66d9029233cb2bcb8 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 24 Jul 2026 14:01:43 +0800 Subject: [PATCH 19/80] =?UTF-8?q?feat:=20ci=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_guobiao.py | 1 - test/scripts/model/rule/test_rule_common.py | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index 3b2b183f..e2c9bdf9 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -168,7 +168,6 @@ class RuleTextPerplexity(BaseRule): stride=512, ) - _model_name = None _tokenizer = None _model = None diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 27f230dc..9cd933f0 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -2,11 +2,13 @@ from dingo.io import Data from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.output.eval_detail import QualityLabel -from dingo.model.rule.rule_guobiao import RuleDataTypeConsistency +from dingo.model.rule.rule_guobiao import ( + RuleDataTypeConsistency, + RuleTextPerplexity, +) from dingo.model.rule.rule_common import ( RuleDocFormulaRepeat, RulePIIDetection, - RuleTextPerplexity, RuleUnsafeWords, ) @@ -177,6 +179,7 @@ def test_missing_type_is_bad(self): assert result.status is True assert "missing or empty" in result.reason[0] + class TestRulePIIDetection: """PII 检测规则测试""" From 8608631f1f0d28544ebd1643a160aa17324ff521 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 24 Jul 2026 06:05:26 +0000 Subject: [PATCH 20/80] =?UTF-8?q?=F0=9F=8E=A8=20Auto-format=20code=20with?= =?UTF-8?q?=20pre-commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/scripts/model/rule/test_rule_common.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 9cd933f0..144d61a7 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -1,16 +1,10 @@ import pytest -from dingo.io import Data + from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel -from dingo.model.rule.rule_guobiao import ( - RuleDataTypeConsistency, - RuleTextPerplexity, -) -from dingo.model.rule.rule_common import ( - RuleDocFormulaRepeat, - RulePIIDetection, - RuleUnsafeWords, -) +from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords +from dingo.model.rule.rule_guobiao import RuleDataTypeConsistency, RuleTextPerplexity class TestRuleDocFormulaRepeat: From 19833749bfb990cf8927390be7a970a2aded0686 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 24 Jul 2026 17:12:23 +0800 Subject: [PATCH 21/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E8=AF=B4?= =?UTF-8?q?=E6=98=8E=E6=96=87=E6=A1=A34=E4=B8=AArule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_guobiao.py | 338 ++++++++++++++++++++ examples/guobiao/doc_completeness.py | 29 ++ test/scripts/model/rule/test_rule_common.py | 72 +++++ 3 files changed, 439 insertions(+) create mode 100644 examples/guobiao/doc_completeness.py diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index e2c9bdf9..33823f70 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -1,5 +1,6 @@ import importlib.util import math +import re from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data, RequiredField @@ -320,3 +321,340 @@ def eval(cls, input_data: Data) -> EvalDetail: f"(threshold: {threshold:.4f}, model: {model_name})" ] return res + + +class _RuleDatasetDocCompletenessBase(BaseRule): + """Shared logic for dataset documentation completeness checks.""" + + _required_fields = [RequiredField.CONTENT] + _default_threshold = 0.8 + _default_semantic_threshold = 0.5 + _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli" + _default_device = -1 + _default_dimension_name = "Dataset documentation" + _default_aspect_keywords = {} + dynamic_config = EvaluatorRuleArgs( + threshold=_default_threshold, + semantic_threshold=_default_semantic_threshold, + model=_default_model, + device=_default_device, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + f"{cls.__name__} requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_aspect_score(cls, content, aspect_text, model_name, device): + classifier = cls._get_classifier(model_name, device) + result = classifier( + content, + candidate_labels=[aspect_text], + hypothesis_template="这段文本包含{}相关说明。", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != aspect_text: + raise RuntimeError("Zero-shot classifier returned an invalid result") + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def _match_aspects( + cls, + content, + normalized_content, + aspect_keywords, + model_name, + device, + semantic_threshold, + ): + matched = {} + missing = [] + for aspect_name, keywords in aspect_keywords.items(): + aspect_text = ( + f"{aspect_name}（关键词示例：{'、'.join(keywords)}）" + if keywords + else aspect_name + ) + score = cls._calculate_aspect_score( + content, aspect_text, model_name, device + ) + evidence_keyword = next( + ( + keyword + for keyword in keywords + if keyword.lower() in normalized_content + ), + None, + ) + if score >= semantic_threshold: + matched[aspect_name] = { + "score": round(score, 4), + "keyword": evidence_keyword, + } + else: + missing.append(aspect_name) + return matched, missing + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + content = getattr(input_data, "content", None) + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Content is missing or empty"] + res.score = 0.0 + return res + + threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold) + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" + ) + + semantic_threshold = getattr( + cls.dynamic_config, + "semantic_threshold", + cls._default_semantic_threshold, + ) + if semantic_threshold is None or not 0 < semantic_threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]" + ) + + aspect_keywords = getattr( + cls.dynamic_config, + "aspect_keywords", + cls._default_aspect_keywords, + ) + if not isinstance(aspect_keywords, dict) or not aspect_keywords: + raise ValueError( + f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" + ) + + model_name = getattr(cls.dynamic_config, "model", cls._default_model) + if not isinstance(model_name, str) or not model_name.strip(): + raise ValueError( + f"{cls.__name__} dynamic_config.model must be a non-empty string" + ) + + device = getattr(cls.dynamic_config, "device", cls._default_device) + + dimension_name = getattr( + cls.dynamic_config, "dimension_name", cls._default_dimension_name + ) + + normalized_content = re.sub(r"\s+", "", content).lower() + matched, missing = cls._match_aspects( + content=content, + normalized_content=normalized_content, + aspect_keywords=aspect_keywords, + model_name=model_name, + device=device, + semantic_threshold=semantic_threshold, + ) + + total = len(aspect_keywords) + matched_count = len(matched) + score = matched_count / total if total else 0.0 + res.score = round(score, 4) + matched_desc = ( + ", ".join( + ( + f"{aspect}(semantic_score={detail['score']:.4f}, " + f"keyword_hit={detail['keyword'] or 'None'})" + ) + for aspect, detail in matched.items() + ) + if matched + else "None" + ) + missing_desc = ", ".join(missing) if missing else "None" + + if score < threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} is below " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} meets " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + return res + + +@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) +class RuleDocBasicInfoCompleteness(_RuleDatasetDocCompletenessBase): + """0101: Basic information completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "RuleDocBasicInfoCompleteness", + "description": ( + "Checks whether dataset documentation covers basic information " + "aspects such as scale, format, structure, access, and support" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + _default_dimension_name = "Basic information" + _default_aspect_keywords = { + "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"], + "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"], + "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"], + "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"], + "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) +class RuleDocContentFeatureCompleteness(_RuleDatasetDocCompletenessBase): + """0102: Content feature completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "RuleDocContentFeatureCompleteness", + "description": ( + "Checks whether dataset documentation covers content-feature aspects " + "such as modality, distribution, labels, examples, and limitations" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + _default_dimension_name = "Content feature" + _default_aspect_keywords = { + "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"], + "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"], + "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"], + "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"], + "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) +class RuleDocConstructionProcessCompleteness(_RuleDatasetDocCompletenessBase): + """0103: Construction-process completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "RuleDocConstructionProcessCompleteness", + "description": ( + "Checks whether dataset documentation covers construction-process " + "aspects such as data source, collection, processing, annotation, " + "and version control" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + _default_dimension_name = "Construction process" + _default_aspect_keywords = { + "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"], + "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"], + "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"], + "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"], + "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) +class RuleDocApplicationCompleteness(_RuleDatasetDocCompletenessBase): + """0104: Application-description completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "RuleDocApplicationCompleteness", + "description": ( + "Checks whether dataset documentation covers application aspects " + "such as license, scenarios, evaluation method, benchmark, and cases" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + _default_dimension_name = "Application description" + _default_aspect_keywords = { + "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"], + "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"], + "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"], + "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"], + "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) diff --git a/examples/guobiao/doc_completeness.py b/examples/guobiao/doc_completeness.py new file mode 100644 index 00000000..c3cb656c --- /dev/null +++ b/examples/guobiao/doc_completeness.py @@ -0,0 +1,29 @@ +"""Evaluate one dataset document using a guobiao completeness rule. + +Optional dependencies: + conda run -n dingo pip install "dingo-python[hhem]" + +The first run downloads the configured Hugging Face model unless it is already +available locally. +""" + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.model.rule.rule_guobiao import RuleDocBasicInfoCompleteness + + +def main(): + data = Data( + data_id="guobiao-doc-basic-info-example", + content="本数据集说明文档包含数据集规模与样本数量说明，提供格式规范、文件结构、访问渠道和技术支持方式。" + ) + + RuleDocBasicInfoCompleteness.dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + ) + result = RuleDocBasicInfoCompleteness.eval(data) + print(result) + + +if __name__ == "__main__": + main() diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 9cd933f0..ab466890 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,6 +4,10 @@ from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_guobiao import ( RuleDataTypeConsistency, + RuleDocApplicationCompleteness, + RuleDocBasicInfoCompleteness, + RuleDocConstructionProcessCompleteness, + RuleDocContentFeatureCompleteness, RuleTextPerplexity, ) from dingo.model.rule.rule_common import ( @@ -355,3 +359,71 @@ def test_severity_levels(self): data_low = Data(data_id="14", content="IP：192.168.1.1") res_low = RulePIIDetection.eval(data_low) assert "Low Risk" in str(res_low.reason) + + +class TestRuleDatasetDocCompleteness: + def test_basic_info_completeness_good(self): + content = ( + "本数据集说明包含数据集规模与样本数量，给出格式规范和文件结构，" + "提供访问渠道，并说明技术支持联系方式。" + ) + res = RuleDocBasicInfoCompleteness.eval( + Data(data_id="doc-basic-good", content=content) + ) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert res.score == 1.0 + + def test_basic_info_completeness_bad(self): + content = "仅提到样本数量和文件结构，未说明访问渠道。" + res = RuleDocBasicInfoCompleteness.eval( + Data(data_id="doc-basic-bad", content=content) + ) + assert res.status is True + assert res.label == [ + "QUALITY_BAD_COMPLETENESS.RuleDocBasicInfoCompleteness" + ] + assert res.score < 0.8 + + def test_content_feature_completeness_good(self): + content = ( + "文档包含模态类型、数据分布情况、标签类别统计、样本示例以及局限性说明。" + ) + res = RuleDocContentFeatureCompleteness.eval( + Data(data_id="doc-content-good", content=content) + ) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert res.score == 1.0 + + def test_construction_process_completeness_good(self): + content = ( + "建设过程包括数据来源、采集方法、加工处理流程、标注规范和版本控制记录。" + ) + res = RuleDocConstructionProcessCompleteness.eval( + Data(data_id="doc-process-good", content=content) + ) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert res.score == 1.0 + + def test_application_completeness_good(self): + content = ( + "应用说明提供使用许可、目标应用场景、评估方法、基准测试结果与典型应用案例。" + ) + res = RuleDocApplicationCompleteness.eval( + Data(data_id="doc-application-good", content=content) + ) + assert res.status is False + assert res.label == [QualityLabel.QUALITY_GOOD] + assert res.score == 1.0 + + def test_empty_content_is_bad(self): + res = RuleDocApplicationCompleteness.eval( + Data(data_id="doc-empty", content=" ") + ) + assert res.status is True + assert res.label == [ + "QUALITY_BAD_COMPLETENESS.RuleDocApplicationCompleteness" + ] + assert "missing or empty" in res.reason[0] From cdf1bb2d92918f8ae257262ae120f4755d0add46 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 24 Jul 2026 17:34:21 +0800 Subject: [PATCH 22/80] =?UTF-8?q?feat:=20md=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 56 ++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 6c436115..6c737d3f 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,11 +8,11 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 璇勪及绛旀鏄惁鐩存帴鍥炵瓟闂锛屾娴嬫棤鍏冲拰鍐椾綑淇℃伅 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 璇勪及妫€绱笂涓嬫枃鐨勭簿纭害锛屽寘鎷浉鍏虫€у拰鎺掑簭璐ㄩ噺 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRecall` | LLMRAGContextRecall | 璇勪及妫€绱笂涓嬫枃鐨勫畬鏁存€э紝鍒ゆ柇涓婁笅鏂囨槸鍚﹁兘鏀寔绛旀涓殑鎵€鏈夐檲杩?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 璇勪及妫€绱笂涓嬫枃涓庨棶棰樼殑鐩稿叧鎬э紝妫€娴嬪櫔澹颁俊鎭?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 璇勪及鐢熸垚绛旀鏄惁蹇犲疄浜庣粰瀹氫笂涓嬫枃锛屾娴嬪够瑙夊拰缂栭€犱俊鎭?| [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [馃摑 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | ### Pretrain Text Quality Assessment Metrics @@ -25,10 +25,10 @@ This document provides comprehensive information about all quality metrics used | `LLMMathCompare` | LLMMathCompare | Compares the effectiveness of two tools in extracting mathematical formulas from HTML to Markdown format by evaluatin... | Internal Implementation | N/A | N/A | | `LLMSecurityPolitics` | LLMSecurityPolitics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | N/A | | `LLMTableCompare` | LLMTableCompare | Compares the effectiveness of two tools in extracting tables from HTML to Markdown format by evaluating recognition r... | Internal Implementation | N/A | N/A | -| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | -| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [馃搳 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [馃摑 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | +| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | ### SFT Data Assessment Metrics @@ -36,38 +36,38 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `LLMFactCheckPublic` | LLMFactCheckPublic | Two-stage factuality evaluation pipeline from GPT-5 | [GPT-5 System Card](https://cdn.openai.com/pdf/8124a3ce-ab78-4f06-96eb-49ea29ffb52f/gpt5-system-card-aug7.pdf) (OpenAI) | N/A | N/A | | `LLMHallucination` | LLMHallucination | Evaluates whether the response contains factual contradictions or hallucinations against provided context information | [TruthfulQA: Measuring How Models Mimic Human Falsehoods](https://arxiv.org/abs/2109.07958) (Lin et al., 2021) | N/A | N/A | -| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [馃搳 See Results](Returns clarity score (0-10) and detailed analysis) | [馃摑 View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [馃搳 See Results](Returns difficulty level (1-10) with detailed breakdown) | [馃摑 View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [馃搳 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [See Results](Returns clarity score (0-10) and detailed analysis) | [View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [See Results](Returns difficulty level (1-10) with detailed breakdown) | [View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A | ### Classification Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [馃搳 See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | +| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | ### Multimodality Assessment Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `LLMClassifyQR` | LLMClassifyQR | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | N/A | -| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 璇勪及澶氭ā鎬佹ā鍨嬪鍥剧墖涓枃瀛楀唴瀹圭殑璇嗗埆鍜岀悊瑙ｈ兘鍔涳紝浣跨敤DeepSeek-OCR浣滀负Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [馃搳 See Results](閫氳繃瀵规瘮VLM杈撳嚭涓嶰CR ground truth锛岃瘑鍒枃瀛楅仐婕忋€侀敊璇€佸够瑙夌瓑闂) | N/A | +| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用 DeepSeek-OCR 作为 Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [See Results](通过对比 VLM 输出与 OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | ### Rule-Based TEXT Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, RuleDocContentFeatureCompleteness, RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks dataset-documentation completeness across basic information, content features, construction process, and application guidance, together with text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_TYPE_CONSISTENCY` | RuleDataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | -| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [馃搳 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics @@ -106,14 +106,14 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [馃搳 See Results](error_category and error_label) | N/A | -| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [馃搳 See Results](error_category and error_label) | N/A | +| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | +| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | ### RAG Retrieved Evidence Chunk Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [馃摑 View Example](../examples/rag/sdk_chunk_eval.py) | +| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [View Example](../examples/rag/sdk_chunk_eval.py) | ### Resume Quality Assessment Metrics @@ -127,7 +127,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 妫€鏌ュ厓鏁版嵁瀛楁涓庡熀鍑嗘暟鎹殑鐩镐技搴﹀尮閰嶏紝闃堝€奸粯璁や负0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为 0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics @@ -159,4 +159,4 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `AgentFactCheck` | AgentFactCheck | Agent-based hallucination detection with autonomous web search | Internal Implementation | N/A | N/A | | `ArticleFactChecker` | ArticleFactChecker | Article-level fact checking with autonomous claims extraction and verification | Internal Implementation | N/A | N/A | -| `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | \ No newline at end of file +| `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | From 04e4563e8d25d5316ca9a0942aa684c636dec716 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 24 Jul 2026 18:39:54 +0800 Subject: [PATCH 23/80] feat: ci test --- dingo/model/rule/rule_guobiao.py | 1 + test/scripts/model/rule/test_rule_common.py | 56 ++++++++++++++++++--- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index 33823f70..a4387079 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -127,6 +127,7 @@ def eval(cls, input_data: Data) -> EvalDetail: score = cls._calculate_match_score( content, declared_type, model_name, device ) + res.score = score if score >= threshold: res.label = [QualityLabel.QUALITY_GOOD] diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 69631930..737573d1 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,6 +4,7 @@ from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_guobiao import ( + _RuleDatasetDocCompletenessBase, RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, @@ -125,7 +126,7 @@ def fail_if_called(cls, model_name): def test_missing_dependencies_raise_clear_error(self, monkeypatch): monkeypatch.setattr( - "dingo.model.rule.rule_common.importlib.util.find_spec", + "dingo.model.rule.rule_guobiao.importlib.util.find_spec", lambda package: None if package == "transformers" else object(), ) @@ -363,7 +364,46 @@ def test_severity_levels(self): class TestRuleDatasetDocCompleteness: - def test_basic_info_completeness_good(self): + @staticmethod + def _mock_aspect_matching(monkeypatch): + def mock_match( + cls, + content, + normalized_content, + aspect_keywords, + model_name, + device, + semantic_threshold, + ): + del cls, content, model_name, device, semantic_threshold + matched = {} + missing = [] + for aspect_name, keywords in aspect_keywords.items(): + evidence_keyword = next( + ( + keyword + for keyword in keywords + if keyword.lower() in normalized_content + ), + None, + ) + if evidence_keyword: + matched[aspect_name] = { + "score": 1.0, + "keyword": evidence_keyword, + } + else: + missing.append(aspect_name) + return matched, missing + + monkeypatch.setattr( + _RuleDatasetDocCompletenessBase, + "_match_aspects", + classmethod(mock_match), + ) + + def test_basic_info_completeness_good(self, monkeypatch): + self._mock_aspect_matching(monkeypatch) content = ( "本数据集说明包含数据集规模与样本数量，给出格式规范和文件结构，" "提供访问渠道，并说明技术支持联系方式。" @@ -375,7 +415,8 @@ def test_basic_info_completeness_good(self): assert res.label == [QualityLabel.QUALITY_GOOD] assert res.score == 1.0 - def test_basic_info_completeness_bad(self): + def test_basic_info_completeness_bad(self, monkeypatch): + self._mock_aspect_matching(monkeypatch) content = "仅提到样本数量和文件结构，未说明访问渠道。" res = RuleDocBasicInfoCompleteness.eval( Data(data_id="doc-basic-bad", content=content) @@ -386,7 +427,8 @@ def test_basic_info_completeness_bad(self): ] assert res.score < 0.8 - def test_content_feature_completeness_good(self): + def test_content_feature_completeness_good(self, monkeypatch): + self._mock_aspect_matching(monkeypatch) content = ( "文档包含模态类型、数据分布情况、标签类别统计、样本示例以及局限性说明。" ) @@ -397,7 +439,8 @@ def test_content_feature_completeness_good(self): assert res.label == [QualityLabel.QUALITY_GOOD] assert res.score == 1.0 - def test_construction_process_completeness_good(self): + def test_construction_process_completeness_good(self, monkeypatch): + self._mock_aspect_matching(monkeypatch) content = ( "建设过程包括数据来源、采集方法、加工处理流程、标注规范和版本控制记录。" ) @@ -408,7 +451,8 @@ def test_construction_process_completeness_good(self): assert res.label == [QualityLabel.QUALITY_GOOD] assert res.score == 1.0 - def test_application_completeness_good(self): + def test_application_completeness_good(self, monkeypatch): + self._mock_aspect_matching(monkeypatch) content = ( "应用说明提供使用许可、目标应用场景、评估方法、基准测试结果与典型应用案例。" ) From 9b213e4d83063b768e226d8fc248b2cbfaf5cd64 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 24 Jul 2026 10:42:36 +0000 Subject: [PATCH 24/80] =?UTF-8?q?=F0=9F=8E=A8=20Auto-format=20code=20with?= =?UTF-8?q?=20pre-commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_guobiao.py | 8 ++++---- test/scripts/model/rule/test_rule_common.py | 17 +++-------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index a4387079..528befd8 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -454,7 +454,7 @@ def eval(cls, input_data: Data) -> EvalDetail: raise ValueError( f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" ) - + semantic_threshold = getattr( cls.dynamic_config, "semantic_threshold", @@ -474,15 +474,15 @@ def eval(cls, input_data: Data) -> EvalDetail: raise ValueError( f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" ) - + model_name = getattr(cls.dynamic_config, "model", cls._default_model) if not isinstance(model_name, str) or not model_name.strip(): raise ValueError( f"{cls.__name__} dynamic_config.model must be a non-empty string" ) - + device = getattr(cls.dynamic_config, "device", cls._default_device) - + dimension_name = getattr( cls.dynamic_config, "dimension_name", cls._default_dimension_name ) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 737573d1..c49f0b4e 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -3,20 +3,9 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel -from dingo.model.rule.rule_guobiao import ( - _RuleDatasetDocCompletenessBase, - RuleDataTypeConsistency, - RuleDocApplicationCompleteness, - RuleDocBasicInfoCompleteness, - RuleDocConstructionProcessCompleteness, - RuleDocContentFeatureCompleteness, - RuleTextPerplexity, -) -from dingo.model.rule.rule_common import ( - RuleDocFormulaRepeat, - RulePIIDetection, - RuleUnsafeWords, -) +from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords +from dingo.model.rule.rule_guobiao import (RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, + RuleDocContentFeatureCompleteness, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: From 636506bfcdf08b79651707224e56bc6e94946160 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 10:37:27 +0800 Subject: [PATCH 25/80] =?UTF-8?q?feat:=20Lint=E4=B8=8D=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E7=9B=B4=E6=8E=A5=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/lint.yml | 35 +++++------------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2fa4491e..c34195d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,6 +2,9 @@ name: Lint on: [push, pull_request] +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -29,33 +32,5 @@ jobs: run: | python .github/scripts/check_imports.py - - name: Run pre-commit (auto-fix) - id: pre_commit_auto_fix - run: | - # 运行 pre-commit，允许自动修复，不因修复而失败 - pre-commit run --all-files || true - - - name: Check for changes - id: check_changes - run: | - if [[ -n $(git status --porcelain) ]]; then - echo "changed=true" >> $GITHUB_OUTPUT - echo "📝 Files were modified by pre-commit auto-fix" - else - echo "changed=false" >> $GITHUB_OUTPUT - echo "✅ No auto-fix changes" - fi - - - name: Commit auto-fix changes - if: steps.check_changes.outputs.changed == 'true' && github.event_name == 'push' - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add -A - git commit -m "🎨 Auto-format code with pre-commit" - git push - - - name: Run pre-commit (final check) - run: | - # 再次运行 pre-commit，这次如果有错误就真的失败 - pre-commit run --all-files + - name: Run pre-commit + run: pre-commit run --all-files From 0581fd14afbb707632fe3e856de7df01b8a2f23a Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 11:22:56 +0800 Subject: [PATCH 26/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E5=BA=94=E7=94=A8-=E5=86=85=E5=AE=B9=E6=97=B6?= =?UTF-8?q?=E6=95=88=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + dingo/model/rule/rule_guobiao.py | 134 ++++++++++++++++++++ examples/guobiao/time_range.py | 24 ++++ test/scripts/model/rule/test_rule_common.py | 78 +++++++++++- 4 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 examples/guobiao/time_range.py diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index 29a461e8..869cba47 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -8,3 +8,4 @@ class RequiredField(Enum): IMAGE = "image" METADATA = "metadata" TYPE = "type" + DT = "dt" diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index 528befd8..8d3f6bda 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -1,6 +1,7 @@ import importlib.util import math import re +from datetime import datetime, timezone from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data, RequiredField @@ -145,6 +146,139 @@ def eval(cls, input_data: Data) -> EvalDetail: return res +@Model.rule_register("QUALITY_BAD_TIMELINESS", ["guobiao"]) +class RuleDataTimeRange(BaseRule): + """Check whether creation/update time fields are within configured ranges.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "TIMELINESS", + "metric_name": "RuleDataTimeRange", + "description": ( + "Checks whether created and updated timestamps are within configured " + "time ranges" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + } + + _required_fields = [RequiredField.DT] + dynamic_config = EvaluatorRuleArgs( + dt_start=None, + dt_end=None, + ) + + @classmethod + def _parse_datetime(cls, value, field_name): + if isinstance(value, datetime): + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc).replace(tzinfo=None) + + if isinstance(value, str): + text = value.strip() + if not text: + raise ValueError(f"{field_name} is empty") + + iso_text = text.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(iso_text) + if parsed.tzinfo is None: + return parsed + return parsed.astimezone(timezone.utc).replace(tzinfo=None) + except ValueError: + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) from None + + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) + + @classmethod + def _validate_time_range( + cls, + dt_value, + start_value, + end_value, + ): + parsed_dt = cls._parse_datetime(dt_value, "time_value") + parsed_dt_start = ( + cls._parse_datetime(start_value, "start_time") + if start_value is not None + else None + ) + parsed_dt_end = ( + cls._parse_datetime(end_value, "end_time") + if end_value is not None + else None + ) + + if ( + parsed_dt_start is not None + and parsed_dt_end is not None + and parsed_dt_start > parsed_dt_end + ): + raise ValueError("time range is invalid: start is later than end") + + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + if parsed_dt_end is not None and parsed_dt > parsed_dt_end: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + return True, parsed_dt, parsed_dt_start, parsed_dt_end + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + + dt_start = getattr(cls.dynamic_config, "dt_start", None) + dt_end = getattr(cls.dynamic_config, "dt_end", None) + + if dt_start is None and dt_end is None: + raise ValueError( + "RuleDataTimeRange requires at least one configured range boundary in dynamic_config" + ) + + dt_value = getattr(input_data, "dt", None) + if dt_value is None: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["dt is missing"] + return res + + try: + is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range( + dt_value=dt_value, + start_value=dt_start, + end_value=dt_end, + ) + except ValueError as exc: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [f"dt: {exc}"] + return res + + if not is_valid: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is earlier than " + f"allowed start {parsed_dt_start.isoformat(sep=' ')}" + ] + else: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is later than " + f"allowed end {parsed_dt_end.isoformat(sep=' ')}" + ] + return res + + res.label = [QualityLabel.QUALITY_GOOD] + return res + + @Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain", "guobiao"]) class RuleTextPerplexity(BaseRule): """Check whether text perplexity exceeds the configured threshold.""" diff --git a/examples/guobiao/time_range.py b/examples/guobiao/time_range.py new file mode 100644 index 00000000..8911b9aa --- /dev/null +++ b/examples/guobiao/time_range.py @@ -0,0 +1,24 @@ +"""Evaluate one sample using the guobiao data-time-range rule.""" + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.model.rule.rule_guobiao import RuleDataTimeRange + + +def main(): + data = Data( + data_id="guobiao-time-range-example", + dt="2025-03-01 10:30:00", + content="示例数据", + ) + + RuleDataTimeRange.dynamic_config = EvaluatorRuleArgs( + dt_start="2025-01-01", + dt_end="2025-12-31 23:59:59", + ) + result = RuleDataTimeRange.eval(data) + print(result) + + +if __name__ == "__main__": + main() diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index c49f0b4e..1d4ce268 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -5,7 +5,7 @@ from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords from dingo.model.rule.rule_guobiao import (RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, - RuleDocContentFeatureCompleteness, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) + RuleDocContentFeatureCompleteness, RuleDataTimeRange, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: @@ -175,6 +175,82 @@ def test_missing_type_is_bad(self): assert "missing or empty" in result.reason[0] +class TestRuleDataTimeRange: + def test_dt_in_range_is_good(self, monkeypatch): + monkeypatch.setattr( + RuleDataTimeRange, + "dynamic_config", + EvaluatorRuleArgs( + dt_start="2025-01-01", + dt_end="2025-12-31 23:59:59", + ), + ) + + result = RuleDataTimeRange.eval( + Data( + data_id="time-good", + dt="2025-03-01 08:30:00", + ) + ) + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + def test_created_time_out_of_range_is_bad(self, monkeypatch): + monkeypatch.setattr( + RuleDataTimeRange, + "dynamic_config", + EvaluatorRuleArgs( + dt_start="2025-01-01", + dt_end="2025-12-31 23:59:59", + ), + ) + + result = RuleDataTimeRange.eval( + Data( + data_id="time-created-out", + dt="2024-12-31 23:59:59", + ) + ) + assert result.status is True + assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert "earlier than allowed start" in result.reason[0] + + def test_dt_invalid_format_is_bad(self, monkeypatch): + monkeypatch.setattr( + RuleDataTimeRange, + "dynamic_config", + EvaluatorRuleArgs( + dt_start="2025-01-01", + dt_end="2025-12-31 23:59:59", + ), + ) + + result = RuleDataTimeRange.eval( + Data( + data_id="time-dt-format", + dt="2025年03月01日", + ) + ) + assert result.status is True + assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert "unsupported datetime format" in result.reason[0] + + def test_missing_time_field_is_bad(self, monkeypatch): + monkeypatch.setattr( + RuleDataTimeRange, + "dynamic_config", + EvaluatorRuleArgs( + dt_start="2025-01-01", + dt_end="2025-12-31 23:59:59", + ), + ) + + result = RuleDataTimeRange.eval(Data(data_id="time-missing")) + assert result.status is True + assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert "dt is missing" in result.reason[0] + + class TestRulePIIDetection: """PII 检测规则测试""" From cdfc8c80097fc377c9e98ccfa4c7c253d73f11f8 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 11:29:25 +0800 Subject: [PATCH 27/80] =?UTF-8?q?feat:=20md=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 1 + docs/rules.md | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/metrics.md b/docs/metrics.md index 6c737d3f..3bf09db4 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -66,6 +66,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_TIMELINESS` | RuleDataTimeRange | Checks whether `data.dt` falls within the configured `dt_start` and `dt_end` range required by the target application scenario. | High-quality dataset quality evaluation specification (SAC/TC609) | N/A | N/A | | `QUALITY_BAD_TYPE_CONSISTENCY` | RuleDataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | | `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | diff --git a/docs/rules.md b/docs/rules.md index d5e645af..667d5e12 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -21,8 +21,13 @@ The specific rules for each quality metric are as follows: | RuleContentShort | EFFECTIVENESS | Check whether content is too short. | | | RuleContentShortMultiLan | EFFECTIVENESS | Check whether multilingual content is too short. | | | RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) | +| RuleDataTimeRange | TIMELINESS | Check whether `data.dt` is within the configured `dt_start` and `dt_end` time range. | 2025 High-quality dataset quality evaluation specification | | RuleDataTypeConsistency | TYPE_CONSISTENCY | Use a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | 2025 High-quality dataset classification guide | | RuleDictConsistency | EFFECTIVENESS | Compare two dictionary fields and report mismatched keys. | | +| RuleDocApplicationCompleteness | COMPLETENESS | Check whether dataset documentation covers licensing, target scenarios, evaluation methods, benchmark results, and typical cases. | 2025 High-quality dataset quality evaluation specification | +| RuleDocBasicInfoCompleteness | COMPLETENESS | Check whether dataset documentation covers dataset scale, format specification, file structure, access channel, and technical support. | 2025 High-quality dataset quality evaluation specification | +| RuleDocConstructionProcessCompleteness | COMPLETENESS | Check whether dataset documentation covers data sources, collection methods, processing pipeline, annotation specification, and version control. | 2025 High-quality dataset quality evaluation specification | +| RuleDocContentFeatureCompleteness | COMPLETENESS | Check whether dataset documentation covers modality type, data distribution, label statistics, sample examples, and limitations. | 2025 High-quality dataset quality evaluation specification | | RuleDocFormulaRepeat | SIMILARITY | Check whether formulas repeat in a document. | | | RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) | | RuleDoi | EFFECTIVENESS | Validate DOI metadata. | | From beb5a11ae3f5524b33625f0d30420868c8a70669 Mon Sep 17 00:00:00 2001 From: pekopoke <1135796875@qq.com> Date: Mon, 27 Jul 2026 11:42:22 +0800 Subject: [PATCH 28/80] feat(retrieval): add end-to-end search result evaluation --- dingo/retrieval/backends/agentic.py | 25 +- dingo/retrieval/backends/openalex.py | 5 +- docs/search_result_quality_metrics.md | 203 ++++---- examples/retrieval/sdk_eval_search_result.py | 443 ++++++++++++------ .../retrieval/search_result_eval_utils.py | 68 ++- test/data/test_search_queries.jsonl | 3 + test/scripts/retrieval/test_search_client.py | 42 +- .../test_search_result_eval_script.py | 272 +++++++++++ 8 files changed, 811 insertions(+), 250 deletions(-) create mode 100644 test/data/test_search_queries.jsonl create mode 100644 test/scripts/retrieval/test_search_result_eval_script.py diff --git a/dingo/retrieval/backends/agentic.py b/dingo/retrieval/backends/agentic.py index c110bddf..515eda00 100644 --- a/dingo/retrieval/backends/agentic.py +++ b/dingo/retrieval/backends/agentic.py @@ -7,8 +7,8 @@ POST {api_url}/v1/search -- direct connection to the Go service, no auth. Public (when api_token is set): - POST {api_url}/agentic-search -- SciVerse public gateway with Bearer auth. - POST {api_url}/meta-search -- SciVerse metadata search gateway. + POST {api_url}/agentic-search -- Sciverse public gateway with Bearer auth. + POST {api_url}/meta-search -- Sciverse metadata search gateway. Rate limit defaults to 1 RPS. Examples: @@ -291,16 +291,33 @@ class MetaSearchClient(AgenticSearchClient): def __init__( self, *args, - search_type: str = "paper", + search_type: str = "all", sort_by: str | None = None, freshness_boost: str | None = None, filters: list[dict[str, Any]] | dict[str, Any] | None = None, **kwargs: Any, ) -> None: - self.search_type = search_type + api_url = kwargs.get("api_url") + if api_url: + normalized_url = str(api_url).rstrip("/") + if normalized_url.endswith("/meta-search"): + kwargs["api_url"] = normalized_url[: -len("/meta-search")] + self.search_type = (search_type or "all").strip().lower() + if self.search_type not in {"all", "paper", "ebook"}: + raise ValueError("meta_search search_type must be 'all', 'paper', or 'ebook'") self.sort_by = sort_by self.freshness_boost = freshness_boost self.filters = self._normalize_filters(filters) + if self.search_type != "all" and not any( + item.get("field") == "metadata_type" for item in self.filters + ): + self.filters.append( + { + "field": "metadata_type", + "operator": "FILTER_OP_EQ", + "value": self.search_type, + } + ) super().__init__(*args, **kwargs) if self._public_mode: self.name = "sciverse-meta-search-api" diff --git a/dingo/retrieval/backends/openalex.py b/dingo/retrieval/backends/openalex.py index b32b7907..52f21627 100644 --- a/dingo/retrieval/backends/openalex.py +++ b/dingo/retrieval/backends/openalex.py @@ -50,7 +50,8 @@ _DEFAULT_SELECT = ( "id,doi,display_name,title,abstract_inverted_index,publication_year," - "relevance_score,cited_by_count" + "relevance_score,cited_by_count,type,language,authorships,keywords," + "primary_location,open_access" ) @@ -70,6 +71,8 @@ def __init__( **_kwargs: Any, ) -> None: self.base_url = api_url.rstrip("/") + if self.base_url.endswith("/works"): + self.base_url = self.base_url[: -len("/works")] self.api_key = api_token or os.environ.get("OPENALEX_API_KEY") self.timeout = timeout self.search_type = (search_type or "search").strip().lower() diff --git a/docs/search_result_quality_metrics.md b/docs/search_result_quality_metrics.md index b24b4e38..a9d8c85e 100644 --- a/docs/search_result_quality_metrics.md +++ b/docs/search_result_quality_metrics.md @@ -1,6 +1,6 @@ # Search Result Quality 三指标评测说明 -本文档说明 meta search 检索结果的三类评测指标：相关性、内容有效性、权威性，以及对应的单项评测脚本和综合评测脚本。该方案面向无人工 GT 的检索结果质量检查，输入为 query 及其 top-k 检索结果，输出 query 级和 result 级分数，并按阈值生成 Dingo 风格的 good/bad 分类目录。 +本文档说明检索结果的三类评测指标：相关性、内容有效性、权威性，以及对应的单项评测脚本和端到端综合评测脚本。该方案面向无人工 GT 的检索结果质量检查，既可读取预计算的 query+results，也可从 query 文件直接请求 SciVerse Meta Search 或 OpenAlex，再通过 Dingo Executor 完成评测和分类。 ## 1. 适用场景 @@ -22,7 +22,11 @@ ## 2. 输入格式 -输入文件为 JSONL，每行一个 query 及其检索结果。 +综合脚本支持两种输入模式。 + +### 2.1 预计算结果 + +JSONL 每行一个 query 及其检索结果： ```json {"query": "PBPK Review", "results": [{"title": "...", "abstract": "..."}]} @@ -47,6 +51,22 @@ outputs/meta_search_97_query_results.jsonl ``` +### 2.2 端到端检索 + +设置 `--retrieval-backend meta_search` 或 `openalex` 后，输入只需要包含 query。支持： + +- TXT：每行一个 query； +- CSV：读取 `query`、`query_text` 或 `q` 列； +- JSON/JSONL：字符串或包含上述 query 字段的对象。 + +```json +{"query": "PBPK相关综述"} +``` + +端到端模式使用后端默认相关度检索，不添加时间排序或业务过滤；`--top-k` 同时控制请求数量和评测数量。 + +仓库提供了三条 query 的端到端测试输入：`test/data/test_search_queries.jsonl`。 + ## 3. 输出文件 综合评测脚本 `sdk_eval_search_result.py` 会在 `--output-dir` 下生成一个时间戳子目录，核心结果都放在该子目录中，例如： @@ -60,10 +80,12 @@ outputs/search_result_eval_97q/20260710_162652_1ac3f3be/ | 文件 | 粒度 | 说明 | |---|---|---| | `summary.json` | 全局 | 指标均值、中位数、最小值、最大值、bad/good 数量、阈值、LLM 配置等 | -| `query_scores.csv` | query 级 | 每个 query 的 rank-discount 汇总分、label、eval_status | +| `query_scores.csv` | query 级 | 每个 query 的 top-k 排名加权平均分、label、eval_status | | `result_scores.csv` | result 级 | 每个 query 的每条 top-k 结果分数和诊断信息 | | `all_results.jsonl` | result 级原始明细 | executor 输出的逐条评测结果，保留三个指标的完整 `eval_details` | -| `bad/` | query 级分类 | 低于阈值或运行异常的 query 记录 | +| `retrieval_results.jsonl` | query 级检索结果 | 仅端到端模式生成，保存可复跑的 query+results | +| `retrieval_request_log.jsonl` | query 级请求日志 | 仅端到端模式生成，记录状态码、耗时、结果数和错误，不含 token | +| `bad/` | query 级分类 | 只包含三种 query 聚合指标低分记录，每行一个 query 及其完整 results | | `good/` | query 级分类 | 使用 `--save-good` 时保存通过的 query 记录 | `detailed_results.json` 默认不生成；需要完整嵌套诊断时加 `--save-detailed`。 @@ -74,14 +96,9 @@ outputs/search_result_eval_97q/20260710_162652_1ac3f3be/ ```text QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW -QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW -QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW -QUALITY_GOOD.SEARCH_RESULT_RELEVANCE_PASS -QUALITY_GOOD.SEARCH_RESULT_EFFECTIVENESS_PASS -QUALITY_GOOD.SEARCH_RESULT_AUTHORITY_PASS -QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS +QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS ``` 单独运行 `sdk_eval_effectiveness.py` 时，`summary.json` 采用类似 `sdk_chunk_eval.py` 的 result 级结构：每个 query 的每条 top-k 检索文献都是一个测试对象，`total`、`num_good`、`num_bad` 和 `score` 都按 result 级计算。`type_ratio.search_result` 中会统计 `Effectiveness.Error_*` 和 `QUALITY_GOOD` 的比例，`metrics_score.search_result` 中会统计 `LLMSearchResultEffectiveness` 的 result 级分数分布。 @@ -106,25 +123,14 @@ QUALITY_GOOD.jsonl ## 4. Query 级汇总逻辑 -三个指标都先对 top-k 中每条 result 打分，然后用 rank-discounted mean 汇总到 query 级。 - -第 `rank` 条结果的权重为： +三个指标都先对 top-k 中每条 result 打分，然后使用排名折扣加权平均汇总到 query 级。第 `rank` 条结果的权重为： ```text weight(rank) = 1 / log2(rank + 1) -``` - -query 级分数为： - -```text query_score = sum(result_score_i * weight_i) / sum(weight_i) ``` -业务含义： - -- rank1 的影响最大。 -- rank 越靠后，对 query 总分影响越小。 -- 适合评估搜索排序质量，因为用户更关注前几条结果。 +rank1 权重最高，越靠后的结果权重越低。综合脚本的 bad/good 判定只比较这三个 query 级加权平均分和统一阈值；result 级分数仍保留在 `result_scores.csv` 与 `all_results.jsonl` 中用于定位问题，但不生成 result 级 bad/good 分类文件。空结果 query 的三个聚合分均为 `0`。 ## 5. 相关性 Relevance @@ -162,23 +168,25 @@ query_score = sum(result_score_i * weight_i) / sum(weight_i) DOI query 不调用 LLM。Result 级完全匹配为 `1.0`，否则为 `0.0`；reason 中记录 expected DOI 和 result DOI。 -DOI 的 query 级得分按精确命中的排名折扣： +DOI query 的 result 级相关性仍按精确匹配得到 `1.0` 或 `0.0`，query 级相关性按精确命中的排名折扣： ```text -doi_relevance = max(exact_match / log2(rank + 1)) +doi_relevance = max(exact_match_i / log2(rank_i + 1)) ``` -因此 rank1 命中为 `1.0`，rank2 命中为 `0.63093`，没有精确命中为 `0.0`。普通 query 继续使用 top-k rank-discount mean。 +因此 rank1 精确命中为 `1.0`，rank2 命中为 `0.63093`，没有精确命中为 `0.0`。 -### 5.4 Query 级异常 +### 5.4 LLM 解析异常 -如果某个 query 的任意 rank 出现 LLM JSON 解析失败，会增加： +单独运行相关性脚本时，如果某个 query 的任意 rank 出现 LLM JSON 解析失败，会增加诊断 label： ```text QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR ``` -这类 label 表示运行/解析质量告警，不一定代表业务相关性低。分析低相关时建议区分： +这类 label 表示运行/解析质量告警，不一定代表业务相关性低。综合端到端脚本仍会记录 `relevance_error_count` 和错误文本，但 bad 目录只使用三个指标低分 label；解析失败导致相关性分数低于阈值时，统一归入 `SEARCH_RESULT_RELEVANCE_LOW`。 + +使用单项评测结果分析低相关时建议区分： - `SEARCH_RESULT_RELEVANCE_LOW`：业务低相关。 - `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`：LLM 输出格式或解析异常。 @@ -193,7 +201,7 @@ QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR | `--llm-workers` | 4 | 并发 LLM 调用数 | | `--llm-timeout` | 60 | 单次 LLM 请求超时秒数 | | `--prompt-mode` | `detailed` | prompt 模式 | -| `OPENAI_MODEL` | `gpt-4o` | LLM 模型名，可通过环境变量覆盖 | +| `OPENAI_MODEL` | `gpt-5.4-mini` | LLM 模型名，可通过环境变量覆盖 | | `OPENAI_BASE_URL` | 空 | OpenAI compatible endpoint | | `OPENAI_TEMPERATURE` | 0.0 | LLM temperature | @@ -202,7 +210,7 @@ QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR 全量检索评测需要逐条判断 query-result 对，LLM 调用量通常接近 `query 数 × top-k`。建议优先使用低延迟的 Flash 类模型，例如： ```text -OPENAI_MODEL=deepseek-v4-flash +OPENAI_MODEL=gpt-5.4-mini ``` - **全量评测和日常回归**：推荐 Flash 类模型，可显著缩短相关性判断时间，并降低长时间批处理中的超时风险。 @@ -210,7 +218,7 @@ OPENAI_MODEL=deepseek-v4-flash - **新旧版本对比**：两次评测必须固定相同模型、prompt、`max_tokens` 和 temperature；建议设置 `OPENAI_TEMPERATURE=0`，减少 LLM 随机波动。 - **并发设置**：建议从 `--llm-workers 2` 至 `4` 开始，根据模型服务的限流和稳定性逐步调整。并发过高可能增加 5xx、超时或空响应。 -模型名称由实际 OpenAI-compatible 服务决定，`deepseek-v4-flash` 仅作为当前环境的推荐示例，不是 Dingo 的强制依赖。 +模型名称由实际 OpenAI-compatible 服务决定，当前推荐使用 `gpt-5.4-mini` 兼顾判断质量与运行速度，但它不是 Dingo 的强制依赖。 ## 6. 内容有效性 Effectiveness @@ -323,7 +331,7 @@ source - 期刊/会议/来源类型。 - DOI。 -该指标不判断 query 相关性，也不判断内容字段是否完整。它适合作为 overall 的辅助指标，不建议单独用于硬判“结果错误”。 +该指标不判断 query 相关性，也不判断内容字段是否完整。它是独立的学术权威信号，不建议单独用于硬判“结果错误”。 ### 7.2 字段权重 @@ -423,43 +431,24 @@ doi_score = 0.0 因此，权威性低不一定表示检索结果不相关，只表示该结果缺少学术权威信号。 -## 8. 综合评分 Overall +## 8. 统一阈值与分类 -综合脚本将三个 query 级指标加权： +综合脚本不计算 overall，也不为三个指标设置权重。默认统一阈值为 `0.15`： ```text -overall = - 0.7 * relevance -+ 0.2 * effectiveness -+ 0.1 * authority +relevance < 0.15 → SEARCH_RESULT_RELEVANCE_LOW +effectiveness < 0.15 → SEARCH_RESULT_EFFECTIVENESS_LOW +authority < 0.15 → SEARCH_RESULT_AUTHORITY_LOW ``` -默认权重： - -| 指标 | 权重 | -|---|---:| -| `relevance` | 0.7 | -| `effectiveness` | 0.2 | -| `authority` | 0.1 | - -业务含义： - -- 相关性是核心，因此权重最高。 -- 内容有效性次之，保证结果有足够元数据可读。 -- 权威性作为辅助，不让 citation/DOI 过度主导搜索体验。 - -综合评估会同时检查： - -- `overall` 是否低于 overall 阈值。 -- `relevance` 是否低于相关性阈值。 -- 是否存在 LLM 解析错误。 -- `effectiveness` 是否低于有效性阈值。 -- `authority` 是否低于权威性阈值。 +三个 query 级聚合指标均不低于阈值时才判为 good。同一 query 可以同时命中多个低分标签。空结果的三个指标均记为 `0`，因此会同时进入三类低分文件，不再产生额外的 `EMPTY` bad 类型。LLM 解析错误保留在诊断字段中，但不会形成第四种 bad 分类。 ## 9. 使用命令 以下命令均在项目根目录执行。 +综合脚本启动时会自动读取项目根目录 `.env`，且不会覆盖已经存在的系统环境变量。可复制 `.env.example` 的字段结构创建本地 `.env`；`.env` 已被 `.gitignore` 忽略，禁止提交真实 token 或 key。 + ### 内置测试数据 仓库提供了 `test/data/test_search_result.jsonl`，用于在提交代码前快速验证单指标和综合评测流程。该文件包含3条 query、9条 result： @@ -479,50 +468,37 @@ python examples/retrieval/sdk_eval_search_result.py ` --top-k 3 ` --llm-max-tokens 1024 ` --effectiveness-llm-max-tokens 1024 ` - --relevance-threshold 0.15 ` - --effectiveness-threshold 0.15 ` - --authority-threshold 0.15 ` - --overall-threshold 0.15 ` + --threshold 0.15 ` --save-good ``` 相关性、有效性和综合 smoke test 需要预先设置 OpenAI-compatible 环境变量；权威性是纯规则评测，不需要 LLM API。 -综合脚本按评测对象拆分分类目录： +综合脚本只生成 query 级分类目录： ```text / ├── bad/ -│ ├── query_level/ -│ │ └── QUALITY_BAD/ -│ │ ├── SEARCH_RESULT_RELEVANCE_LOW.jsonl -│ │ ├── SEARCH_RESULT_EFFECTIVENESS_LOW.jsonl -│ │ └── SEARCH_RESULT_AUTHORITY_LOW.jsonl -│ └── result_level/ -│ ├── Relevance/ -│ │ └── Error_Relevance_Low.jsonl -│ ├── Effectiveness/ -│ │ ├── Error_Effectiveness_Low.jsonl -│ │ └── Error_HTML_Tag.jsonl -│ └── Authority/ -│ ├── Error_Authority_Low.jsonl -│ ├── Error_Citation_Miss.jsonl -│ └── Error_DOI_Miss.jsonl +│ └── QUALITY_BAD/ +│ ├── SEARCH_RESULT_RELEVANCE_LOW.jsonl +│ ├── SEARCH_RESULT_EFFECTIVENESS_LOW.jsonl +│ └── SEARCH_RESULT_AUTHORITY_LOW.jsonl └── good/ # 仅使用 --save-good 时生成 - ├── query_level/ - └── result_level/ + └── QUALITY_GOOD/ + └── SEARCH_RESULT_METRICS_PASS.jsonl ``` -- `query_level`：一个 query 的 top-k 聚合得分及其全部结果。 -- `result_level`：每一篇检索文献的原始数据和三个指标明细。 -- 同一 result 可以写入多个原因 label 文件；例如 Authority Low 可能同时进入 Citation Miss 和 DOI Miss。 +- 每行是一个唯一 query，包含三个排名加权平均分、分类 label 和完整 `results`。 +- 每个 result 的 `_evaluation` 字段记录 rank 和三个单条分数。 +- 同一 query 的多个聚合指标低于阈值时，会分别写入对应的低分文件。 +- result 级细分问题仍保留在 `all_results.jsonl` 的 `eval_details` 中。 ### 9.1 单独跑相关性 ```bash export OPENAI_API_KEY="" export OPENAI_BASE_URL="" -export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_MODEL="gpt-5.4-mini" export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_relevancy.py \ @@ -541,7 +517,7 @@ Windows PowerShell 示例： ```powershell $env:OPENAI_API_KEY="" $env:OPENAI_BASE_URL="" -$env:OPENAI_MODEL="deepseek-v4-flash" +$env:OPENAI_MODEL="gpt-5.4-mini" $env:OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_relevancy.py ` @@ -560,7 +536,7 @@ python examples/retrieval/sdk_eval_relevancy.py ` ```bash export OPENAI_API_KEY="" export OPENAI_BASE_URL="" -export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_MODEL="gpt-5.4-mini" export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_effectiveness.py \ @@ -585,28 +561,65 @@ python examples/retrieval/sdk_eval_authority.py \ --save-good ``` -### 9.4 跑综合评分 +### 9.4 评测预计算结果 ```bash export OPENAI_API_KEY="" export OPENAI_BASE_URL="" -export OPENAI_MODEL="deepseek-v4-flash" +export OPENAI_MODEL="gpt-5.4-mini" export OPENAI_TEMPERATURE="0" python examples/retrieval/sdk_eval_search_result.py \ --input-jsonl outputs/meta_search_97_query_results.jsonl \ --output-dir outputs/search_result_quality_97q \ --top-k 10 \ - --relevance-threshold 0.15 \ - --effectiveness-threshold 0.15 \ - --authority-threshold 0.15 \ - --overall-threshold 0.15 \ + --threshold 0.15 \ --llm-max-tokens 1024 \ --effectiveness-llm-max-tokens 512 \ --llm-timeout 60 \ --save-good ``` +### 9.5 端到端评测 SciVerse Meta Search + +```powershell +$env:SCIVERSE_API_TOKEN="" +$env:SEARCH_API_URL="https://api.sciverse.space/meta-search" +$env:OPENAI_API_KEY="" +$env:OPENAI_BASE_URL="" +$env:OPENAI_MODEL="gpt-5.4-mini" +$env:OPENAI_TEMPERATURE="0" + +python examples/retrieval/sdk_eval_search_result.py ` + --input-queries outputs/query.txt ` + --retrieval-backend meta_search ` + --output-dir outputs/meta_search_end_to_end_97q ` + --top-k 10 ` + --threshold 0.15 ` + --search-workers 4 ` + --llm-workers 4 ` + --batch-size 10 +``` + +### 9.6 端到端评测 OpenAlex + +OpenAlex 默认使用普通 `search`，不需要 API key；如有 key，可设置 `OPENALEX_API_KEY`。 + +```powershell +$env:OPENALEX_API_KEY="" +$env:SEARCH_API_URL="https://api.openalex.org/works" + +python examples/retrieval/sdk_eval_search_result.py ` + --input-queries outputs/query.txt ` + --retrieval-backend openalex ` + --output-dir outputs/openalex_end_to_end_97q ` + --top-k 10 ` + --threshold 0.15 ` + --search-workers 4 ` + --llm-workers 4 ` + --batch-size 10 +``` + ## 10. 阈值解释 当前默认统一阈值为： @@ -695,7 +708,7 @@ Import-Csv outputs/search_result_relevancy_97q/query_scores.csv | ## 13. 已知注意事项 1. 相关性使用 LLM，temperature 大于 0 时，同一批数据重跑可能有轻微分数波动。 -2. LLM 相关性结果可能出现 JSON 解析失败，脚本会记录 `SEARCH_RESULT_RELEVANCE_PARSE_ERROR`。 +2. LLM 相关性结果可能出现 JSON 解析失败，脚本会保留解析错误诊断；该条分数为低分时统一归入 `SEARCH_RESULT_RELEVANCE_LOW`。 3. 内容有效性当前不按 `metadata_type` 放宽 title、abstract、keywords、author 的字段要求；venue 缺失不再降低有效性分数，由权威性指标统一判断。 4. 内容有效性使用 `RuleSpecialCharacter` / `RuleInvisibleChar` / `RuleMojibake` 做快速初筛，再用 LLM 二次确认 HTML 泄漏、乱码、不可见字符和严重特殊字符噪声；正常公式、LaTeX、单位符号不应被扣分。 5. 权威性低不一定表示结果不相关，可能只是 citation、DOI、venue 元数据不足。 diff --git a/examples/retrieval/sdk_eval_search_result.py b/examples/retrieval/sdk_eval_search_result.py index b7488b5e..9a28be92 100644 --- a/examples/retrieval/sdk_eval_search_result.py +++ b/examples/retrieval/sdk_eval_search_result.py @@ -6,30 +6,30 @@ from __future__ import annotations import argparse +import concurrent.futures import json import math import os +import shutil import sys import time from datetime import datetime from pathlib import Path +from types import SimpleNamespace from typing import Any PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_ENV_PATH = PROJECT_ROOT / ".env" if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from search_result_eval_utils import add_common_args, get_title, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, write_json # noqa: E402,E501 +from search_result_eval_utils import (add_common_args, get_title, load_queries, load_query_result_jsonl, rank_discounted_mean, summarize, write_classified_jsonl, write_csv, # noqa: E402,E501 + write_json) from dingo.config import InputArgs # noqa: E402 from dingo.exec import Executor # noqa: E402 from dingo.model.llm.llm_search_result_relevance import is_doi_query # noqa: E402 - -WEIGHTS = { - "relevance": 0.7, - "effectiveness": 0.2, - "authority": 0.1, -} +from dingo.retrieval.search_client import PaperResult, create_client # noqa: E402 EFFECTIVENESS_LABEL_TO_ISSUE = { "Effectiveness.Error_Title_Miss": "missing_title", @@ -46,12 +46,27 @@ } +def load_env_file(env_path: Path = DEFAULT_ENV_PATH) -> None: + """Load an optional local .env without overriding process environment.""" + if not env_path.exists(): + return + for raw_line in env_path.read_text(encoding="utf-8-sig").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip().strip('"').strip("'") + if key and key not in os.environ: + os.environ[key] = value + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Evaluate search result quality with Dingo executor.") add_common_args(parser) parser.add_argument("--openai-api-key", default=os.environ.get("OPENAI_API_KEY")) parser.add_argument("--openai-base-url", default=os.environ.get("OPENAI_BASE_URL")) - parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-4o")) + parser.add_argument("--openai-model", default=os.environ.get("OPENAI_MODEL", "gpt-5.4-mini")) parser.add_argument("--openai-temperature", type=float, default=float(os.environ.get("OPENAI_TEMPERATURE", "0.0"))) parser.add_argument("--prompt-mode", choices=("standard", "detailed"), default="detailed") parser.add_argument("--llm-max-tokens", type=int, default=1024) @@ -64,10 +79,33 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Disable LLM readability/corruption judgment for effectiveness.", ) - parser.add_argument("--relevance-threshold", type=float, default=0.15) - parser.add_argument("--effectiveness-threshold", type=float, default=0.15) - parser.add_argument("--authority-threshold", type=float, default=0.15) - parser.add_argument("--overall-threshold", type=float, default=0.15) + parser.add_argument( + "--threshold", + type=float, + default=0.15, + help="Unified query-level threshold for rank-weighted relevance, effectiveness, and authority.", + ) + parser.add_argument( + "--retrieval-backend", + choices=("precomputed", "meta_search", "openalex"), + default="precomputed", + help="Use precomputed results or retrieve queries before evaluation.", + ) + parser.add_argument( + "--search-api-url", + default=os.environ.get("SEARCH_API_URL"), + help="Search API root or full endpoint; defaults to SEARCH_API_URL.", + ) + parser.add_argument( + "--search-api-token", + default=None, + help="Optional token override; prefer SCIVERSE_API_TOKEN or OPENALEX_API_KEY.", + ) + parser.add_argument("--search-type", default=None) + parser.add_argument("--search-timeout", type=float, default=60.0) + parser.add_argument("--search-workers", type=int, default=4) + parser.add_argument("--search-rate-limit", type=float, default=None) + parser.add_argument("--search-max-retries", type=int, default=3) parser.add_argument( "--save-detailed", action="store_true", @@ -76,6 +114,141 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +def _openalex_authors(raw: dict[str, Any]) -> list[dict[str, str]]: + authors: list[dict[str, str]] = [] + for authorship in raw.get("authorships") or []: + author = authorship.get("author") or {} + name = author.get("display_name") or authorship.get("raw_author_name") or "" + if name: + authors.append({"name": str(name), "orcid": str(author.get("orcid") or "")}) + return authors + + +def _openalex_keywords(raw: dict[str, Any]) -> list[str]: + keywords: list[str] = [] + for item in raw.get("keywords") or []: + value = item.get("display_name") if isinstance(item, dict) else item + if value: + keywords.append(str(value)) + return keywords + + +def normalize_search_result(paper: PaperResult, backend: str) -> dict[str, Any]: + """Map backend-specific output to the fields consumed by all three metrics.""" + raw = dict(paper.raw or {}) + if backend == "meta_search": + raw.setdefault("title", paper.title) + raw.setdefault("abstract", paper.abstract) + raw.setdefault("relevance_score", paper.score) + return raw + + primary_location = raw.get("primary_location") or {} + source = primary_location.get("source") or {} + publisher = source.get("host_organization_name") or "" + return { + "unique_id": raw.get("id") or paper.paper_id, + "title": paper.title, + "abstract": paper.abstract, + "keywords": _openalex_keywords(raw), + "author": _openalex_authors(raw), + "doi": raw.get("doi") or "", + "citation_count": raw.get("cited_by_count") or 0, + "influential_citation_count": 0, + "publication_venue_name_unified": source.get("display_name") or "", + "publication_venue_type": source.get("type") or "", + "publication_venue_issn": source.get("issn") or [], + "publication_publisher": [publisher] if publisher else [], + "publication_published_year": raw.get("publication_year") or paper.year, + "metadata_type": raw.get("type") or "paper", + "language": raw.get("language") or "", + "relevance_score": paper.score, + "access_is_oa": str(bool((raw.get("open_access") or {}).get("is_oa"))).lower(), + "openalex_raw": raw, + } + + +def _build_search_client(args: argparse.Namespace): + kwargs: dict[str, Any] = { + "timeout": args.search_timeout, + "max_retries": args.search_max_retries, + } + if args.search_api_token: + kwargs["api_token"] = args.search_api_token + if args.search_api_url: + kwargs["api_url"] = args.search_api_url + elif args.retrieval_backend == "meta_search": + kwargs["api_url"] = os.environ.get( + "SCIVERSE_API_URL", "https://api.sciverse.space" + ) + elif args.retrieval_backend == "openalex": + kwargs["api_url"] = os.environ.get( + "OPENALEX_API_URL", "https://api.openalex.org" + ) + if args.search_type: + kwargs["search_type"] = args.search_type + if args.search_rate_limit is not None: + kwargs["rate_limit"] = args.search_rate_limit + return create_client(args.retrieval_backend, **kwargs) + + +def retrieve_queries( + args: argparse.Namespace, + output_path: Path, + request_log_path: Path, +) -> dict[str, Any]: + queries = load_queries(args.input_jsonl, args.max_queries) + if not queries: + raise ValueError(f"No queries found in {args.input_jsonl}") + + client = _build_search_client(args) + + def search_one(index_query: tuple[int, str]) -> tuple[int, dict[str, Any], dict[str, Any]]: + index, query = index_query + response = client.search(query, limit=args.top_k) + results = [ + normalize_search_result(paper, args.retrieval_backend) + for paper in response.results[: args.top_k] + ] + item = {"query": query, "results": results} + log = { + "query_index": index, + "query": query, + "status_code": response.status_code, + "response_time_ms": round(response.response_time_ms, 3), + "result_count": len(results), + "error": response.error or "", + } + return index, item, log + + completed: list[tuple[int, dict[str, Any], dict[str, Any]]] = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=args.search_workers) as pool: + futures = [pool.submit(search_one, item) for item in enumerate(queries, start=1)] + for future in concurrent.futures.as_completed(futures): + completed.append(future.result()) + completed.sort(key=lambda item: item[0]) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as result_file, request_log_path.open( + "w", encoding="utf-8" + ) as log_file: + for _, item, log in completed: + result_file.write(json.dumps(item, ensure_ascii=False) + "\n") + log_file.write(json.dumps(log, ensure_ascii=False) + "\n") + + logs = [log for _, _, log in completed] + latencies = [float(log["response_time_ms"]) for log in logs] + return { + "backend": args.retrieval_backend, + "query_count": len(logs), + "result_count": sum(int(log["result_count"]) for log in logs), + "success_count": sum(1 for log in logs if not log["error"]), + "error_count": sum(1 for log in logs if log["error"]), + "empty_count": sum(1 for log in logs if not log["error"] and not log["result_count"]), + "mean_response_time_ms": round(sum(latencies) / len(latencies), 3) if latencies else 0.0, + "max_response_time_ms": round(max(latencies), 3) if latencies else 0.0, + } + + def flatten_query_results( input_path: Path, output_path: Path, @@ -116,7 +289,7 @@ def build_executor_input(args: argparse.Namespace, flattened_path: Path) -> dict "prompt_mode": args.prompt_mode, "max_tokens": args.llm_max_tokens, "timeout": args.llm_timeout, - "threshold": args.relevance_threshold, + "threshold": args.threshold, } effectiveness_config = { "model": args.openai_model, @@ -125,10 +298,10 @@ def build_executor_input(args: argparse.Namespace, flattened_path: Path) -> dict "temperature": args.openai_temperature, "max_tokens": args.effectiveness_llm_max_tokens, "timeout": args.llm_timeout, - "threshold": args.effectiveness_threshold, + "threshold": args.threshold, "enable_llm_quality": not args.disable_effectiveness_llm_quality, } - authority_config = {"threshold": args.authority_threshold} + authority_config = {"threshold": args.threshold} return { "task_name": "search_result_quality", "input_path": str(flattened_path), @@ -198,8 +371,8 @@ def build_reports( records: list[dict[str, Any]], args: argparse.Namespace, executor_summary, - executor_result_summary: dict[str, Any] | None = None, empty_queries: list[str] | None = None, + retrieval_summary: dict[str, Any] | None = None, ) -> tuple[dict, list[dict], list[dict], list[dict], list[dict]]: records = sorted( records, @@ -211,6 +384,7 @@ def build_reports( result_rows = [] by_query: dict[str, list[dict[str, Any]]] = {} + full_results_by_query: dict[str, list[dict[str, Any]]] = {} rank_relevance_error_count = 0 rank_effectiveness_llm_quality_error_count = 0 @@ -227,12 +401,6 @@ def build_reports( relevance = round(float(relevance_detail.get("score") or 0.0), 5) effectiveness = round(float(effectiveness_detail.get("score") or 0.0), 5) authority = round(float(authority_detail.get("score") or 0.0), 5) - overall = round( - WEIGHTS["relevance"] * relevance - + WEIGHTS["effectiveness"] * effectiveness - + WEIGHTS["authority"] * authority, - 5, - ) relevance_error = str(relevance_reason.get("error") or "") effectiveness_error = str(effectiveness_reason.get("llm_quality_error") or "") @@ -262,10 +430,18 @@ def build_reports( "venue_score": round(float(authority_reason.get("venue_score") or 0.0), 5), "doi_score": round(float(authority_reason.get("doi_score") or 0.0), 5), "authority_reason": authority_reason.get("reason", ""), - "overall": overall, } result_rows.append(row) by_query.setdefault(query, []).append(row) + full_result = dict(raw.get("search_result") or {}) + full_result.pop("_eval_query", None) + full_result["_evaluation"] = { + "rank": raw.get("rank"), + "relevance": relevance, + "effectiveness": effectiveness, + "authority": authority, + } + full_results_by_query.setdefault(query, []).append(full_result) query_rows = [] detailed = [] @@ -291,29 +467,19 @@ def build_reports( relevance_aggregation = "rank_discounted_mean" query_effectiveness = round(rank_discounted_mean(effectiveness_scores), 5) query_authority = round(rank_discounted_mean(authority_scores), 5) - query_overall = round( - WEIGHTS["relevance"] * query_relevance - + WEIGHTS["effectiveness"] * query_effectiveness - + WEIGHTS["authority"] * query_authority, - 5, - ) labels = [] - if query_overall < args.overall_threshold: - labels.append("QUALITY_BAD.SEARCH_RESULT_OVERALL_LOW") - if query_relevance < args.relevance_threshold: + if query_relevance < args.threshold: labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW") relevance_errors = sum(1 for row in rows if row["relevance_error"]) - if relevance_errors: - labels.append("QUALITY_BAD.SEARCH_RESULT_RELEVANCE_PARSE_ERROR") - if query_effectiveness < args.effectiveness_threshold: + if query_effectiveness < args.threshold: labels.append("QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW") - if query_authority < args.authority_threshold: + if query_authority < args.threshold: labels.append("QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW") eval_status = bool(labels) if not labels: - labels = ["QUALITY_GOOD.SEARCH_RESULT_OVERALL_PASS"] + labels = ["QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS"] query_row = { "query": query, @@ -322,9 +488,10 @@ def build_reports( "relevance_error_count": relevance_errors, "relevance": query_relevance, "relevance_aggregation": relevance_aggregation, + "effectiveness_aggregation": "rank_discounted_mean", + "authority_aggregation": "rank_discounted_mean", "effectiveness": query_effectiveness, "authority": query_authority, - "overall": query_overall, "eval_status": eval_status, "label": "|".join(labels), } @@ -334,26 +501,30 @@ def build_reports( classified_records.append({ "query": query, "metric": "search_result_quality", - "score": query_overall, - "threshold": args.overall_threshold, + "threshold": args.threshold, "eval_status": eval_status, "labels": labels, "relevance": query_relevance, "relevance_aggregation": relevance_aggregation, + "effectiveness_aggregation": "rank_discounted_mean", + "authority_aggregation": "rank_discounted_mean", "effectiveness": query_effectiveness, "authority": query_authority, "relevance_error_count": relevance_errors, "thresholds": { - "relevance": args.relevance_threshold, - "effectiveness": args.effectiveness_threshold, - "authority": args.authority_threshold, - "overall": args.overall_threshold, + "relevance": args.threshold, + "effectiveness": args.threshold, + "authority": args.threshold, }, - "results": rows, + "results": full_results_by_query.get(query, []), }) for query in empty_queries or []: - labels = ["QUALITY_BAD.SEARCH_RESULT_EMPTY"] + labels = [ + "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW", + "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW", + "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW", + ] query_row = { "query": query, "result_count": 0, @@ -363,33 +534,34 @@ def build_reports( "relevance_aggregation": ( "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" ), + "effectiveness_aggregation": "rank_discounted_mean", + "authority_aggregation": "rank_discounted_mean", "effectiveness": 0.0, "authority": 0.0, - "overall": 0.0, "eval_status": True, - "label": labels[0], + "label": "|".join(labels), } query_rows.append(query_row) detailed.append({**query_row, "results": []}) classified_records.append({ "query": query, "metric": "search_result_quality", - "score": 0.0, - "threshold": args.overall_threshold, + "threshold": args.threshold, "eval_status": True, "labels": labels, "relevance": 0.0, "relevance_aggregation": ( "doi_exact_match_rank_discount" if is_doi_query(query) else "rank_discounted_mean" ), + "effectiveness_aggregation": "rank_discounted_mean", + "authority_aggregation": "rank_discounted_mean", "effectiveness": 0.0, "authority": 0.0, "relevance_error_count": 0, "thresholds": { - "relevance": args.relevance_threshold, - "effectiveness": args.effectiveness_threshold, - "authority": args.authority_threshold, - "overall": args.overall_threshold, + "relevance": args.threshold, + "effectiveness": args.threshold, + "authority": args.threshold, }, "results": [], }) @@ -398,13 +570,8 @@ def build_reports( "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "metric": "search_result_quality", "top_k": args.top_k, - "weights": WEIGHTS, - "thresholds": { - "relevance": args.relevance_threshold, - "effectiveness": args.effectiveness_threshold, - "authority": args.authority_threshold, - "overall": args.overall_threshold, - }, + "threshold": args.threshold, + "query_aggregation": "rank_discounted_mean", "llm": { "model": args.openai_model, "prompt_mode": args.prompt_mode, @@ -420,7 +587,6 @@ def build_reports( "relevance": summarize([float(row["relevance"]) for row in query_rows]), "effectiveness": summarize([float(row["effectiveness"]) for row in query_rows]), "authority": summarize([float(row["authority"]) for row in query_rows]), - "overall": summarize([float(row["overall"]) for row in query_rows]), }, "query_count": len(query_rows), "result_count": len(result_rows), @@ -429,102 +595,88 @@ def build_reports( "num_bad": sum(1 for row in query_rows if row["eval_status"]), "num_good": sum(1 for row in query_rows if not row["eval_status"]), } - if executor_result_summary: - summary["result_level"] = { - "score": executor_result_summary.get("score"), - "num_good": executor_result_summary.get("num_good"), - "num_bad": executor_result_summary.get("num_bad"), - "total": executor_result_summary.get("total"), - "type_ratio": executor_result_summary.get("type_ratio", {}), - "metrics_score": executor_result_summary.get("metrics_score", {}), + ratio_labels = ( + "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW", + "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW", + "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW", + "QUALITY_GOOD.SEARCH_RESULT_METRICS_PASS", + ) + if query_rows: + label_counts = {label: 0 for label in ratio_labels} + for row in query_rows: + row_labels = { + label.strip() + for label in str(row.get("label") or "").split("|") + if label.strip() + } + for label in label_counts: + if label in row_labels: + label_counts[label] += 1 + summary["type_ratio"] = { + "search_result_quality": { + label: count / len(query_rows) + for label, count in label_counts.items() + } } + if retrieval_summary: + summary["retrieval"] = retrieval_summary return summary, query_rows, result_rows, detailed, classified_records -def normalize_executor_summary(executor_output_path: str, records: list[dict[str, Any]]) -> dict[str, Any] | None: - """Make executor summary label ratios use record-level quality semantics. - - LocalExecutor counts every EvalDetail label. In this combined script one - result has three metrics, so a bad result can still contain QUALITY_GOOD - from the metrics that passed. This rewrite keeps error labels as - per-result occurrence rates, but makes QUALITY_GOOD mutually exclusive. - """ - total = len(records) - if total == 0: - return None - - field_key = "search_result" - counts: dict[str, int] = {} - for record in records: - labels = set() - for detail in record.get("eval_details", {}).get(field_key, []): - for label in detail.get("label") or []: - labels.add(label) - - if record.get("eval_status"): - labels.discard("QUALITY_GOOD") - else: - labels = {"QUALITY_GOOD"} - - for label in labels: - counts[label] = counts.get(label, 0) + 1 - - summary_path = Path(executor_output_path) / "summary.json" - if not summary_path.exists(): - return None - - summary = json.loads(summary_path.read_text(encoding="utf-8-sig")) - summary.setdefault("type_ratio", {})[field_key] = { - label: round(count / total, 6) - for label, count in sorted(counts.items()) - } - return summary - - -def build_result_classified_records(records: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Expose executor result labels for result-level directory output.""" - classified: list[dict[str, Any]] = [] - for record in records: - labels: list[str] = [] - for details in record.get("eval_details", {}).values(): - for detail in details: - for label in detail.get("label") or []: - if label != "QUALITY_GOOD" and label not in labels: - labels.append(label) - - eval_status = bool(labels) - if not labels: - labels = ["QUALITY_GOOD"] - classified.append({**record, "eval_status": eval_status, "labels": labels}) - return classified +def clear_executor_classification_dirs(run_dir: Path) -> None: + """Remove result-level classifications before writing query-level output.""" + for name in ("bad", "good"): + path = run_dir / name + if path.exists(): + shutil.rmtree(path) def main() -> None: + load_env_file() args = parse_args() os.environ.setdefault("LOCAL_DEPLOYMENT_MODE", "true") args.output_dir.mkdir(parents=True, exist_ok=True) - flattened_path = args.output_dir / f".search_result_quality_input_{int(time.time())}_{os.getpid()}.jsonl" + temp_suffix = f"{int(time.time())}_{os.getpid()}" + flattened_path = args.output_dir / f".search_result_quality_input_{temp_suffix}.jsonl" + retrieval_results_path = args.output_dir / f".search_result_retrieval_{temp_suffix}.jsonl" + retrieval_log_path = args.output_dir / f".search_result_retrieval_log_{temp_suffix}.jsonl" + retrieval_summary: dict[str, Any] = {"backend": args.retrieval_backend} try: + evaluation_input_path = args.input_jsonl + flatten_max_queries = args.max_queries + if args.retrieval_backend != "precomputed": + retrieval_summary = retrieve_queries( + args, + retrieval_results_path, + retrieval_log_path, + ) + evaluation_input_path = retrieval_results_path + flatten_max_queries = None + total, empty_queries = flatten_query_results( - args.input_jsonl, + evaluation_input_path, flattened_path, top_k=args.top_k, - max_queries=args.max_queries, + max_queries=flatten_max_queries, ) - if total == 0: - raise ValueError("No search results found to evaluate.") - - input_data = build_executor_input(args, flattened_path) - executor_summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() - run_dir = Path(executor_summary.output_path) - executor_records = load_executor_records(executor_summary.output_path) - executor_result_summary = normalize_executor_summary(executor_summary.output_path, executor_records) + if total: + input_data = build_executor_input(args, flattened_path) + executor_summary = Executor.exec_map["local"](InputArgs(**input_data)).execute() + run_dir = Path(executor_summary.output_path) + executor_records = load_executor_records(executor_summary.output_path) + clear_executor_classification_dirs(run_dir) + else: + run_dir = args.output_dir / datetime.now().strftime("%Y%m%d_%H%M%S_empty") + run_dir.mkdir(parents=True, exist_ok=True) + executor_summary = SimpleNamespace(output_path=str(run_dir)) + executor_records = [] + summary, query_rows, result_rows, detailed, classified_records = build_reports( executor_records, args, executor_summary, - executor_result_summary, empty_queries, + retrieval_summary, ) write_json(run_dir / "summary.json", summary) @@ -536,18 +688,15 @@ def main() -> None: run_dir, classified_records, save_good=args.save_good, - level="query_level", - ) - write_classified_jsonl( - run_dir, - build_result_classified_records(executor_records), - save_good=args.save_good, - level="result_level", ) + if args.retrieval_backend != "precomputed": + shutil.copyfile(retrieval_results_path, run_dir / "retrieval_results.jsonl") + shutil.copyfile(retrieval_log_path, run_dir / "retrieval_request_log.jsonl") print(f"Saved to {run_dir.resolve()}") finally: - if flattened_path.exists(): - flattened_path.unlink() + for temp_path in (flattened_path, retrieval_results_path, retrieval_log_path): + if temp_path.exists(): + temp_path.unlink() if __name__ == "__main__": diff --git a/examples/retrieval/search_result_eval_utils.py b/examples/retrieval/search_result_eval_utils.py index d4189d4f..227c23c8 100644 --- a/examples/retrieval/search_result_eval_utils.py +++ b/examples/retrieval/search_result_eval_utils.py @@ -37,11 +37,68 @@ def load_query_result_jsonl(path: Path, max_queries: int | None = None) -> list[ return items +def load_queries(path: Path, max_queries: int | None = None) -> list[str]: + """Load unique queries from TXT, CSV, JSON, or JSONL input.""" + suffix = path.suffix.lower() + queries: list[str] = [] + seen: set[str] = set() + + def add(value: Any) -> None: + query = str(value or "").strip() + if query and query not in seen: + seen.add(query) + queries.append(query) + + if suffix == ".txt": + with path.open("r", encoding="utf-8-sig") as f: + for line in f: + add(line) + if max_queries and len(queries) >= max_queries: + break + return queries + + if suffix == ".csv": + with path.open("r", encoding="utf-8-sig", newline="") as f: + for row in csv.DictReader(f): + add(row.get("query") or row.get("query_text") or row.get("q")) + if max_queries and len(queries) >= max_queries: + break + return queries + + if suffix == ".json": + payload = json.loads(path.read_text(encoding="utf-8-sig")) + rows = payload if isinstance(payload, list) else payload.get("queries", []) + for row in rows: + if isinstance(row, str): + add(row) + elif isinstance(row, dict): + add(row.get("query") or row.get("query_text") or row.get("q")) + if max_queries and len(queries) >= max_queries: + break + return queries + + with path.open("r", encoding="utf-8-sig") as f: + for line_no, line in enumerate(f, start=1): + if not line.strip(): + continue + row = json.loads(line) + if isinstance(row, str): + add(row) + elif isinstance(row, dict): + add(row.get("query") or row.get("query_text") or row.get("q")) + else: + raise ValueError(f"Line {line_no}: query row must be a string or object.") + if max_queries and len(queries) >= max_queries: + break + return queries + + def get_title(result: dict[str, Any]) -> str: return str(result.get("title") or result.get("display_name") or "") def rank_discounted_mean(values: list[float]) -> float: + """Return a query-level weighted mean that gives higher ranks more weight.""" if not values: return 0.0 weights = [1.0 / math.log2(rank + 2) for rank in range(len(values))] @@ -61,11 +118,18 @@ def summarize(values: list[float]) -> dict[str, float | int]: def add_common_args(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--input-jsonl", type=Path, required=True) + parser.add_argument( + "--input-jsonl", + "--input-queries", + dest="input_jsonl", + type=Path, + required=True, + help="Precomputed query-result JSONL, or a TXT/CSV/JSON/JSONL query file in live retrieval mode.", + ) parser.add_argument("--output-dir", type=Path, default=Path("outputs/search_result_eval")) parser.add_argument("--top-k", type=int, default=10) parser.add_argument("--max-queries", type=int, default=None) - parser.add_argument("--save-good", action="store_true", help="Save good query-level records under good/.") + parser.add_argument("--save-good", action="store_true", help="Save good classified records under good/.") def write_json(path: Path, payload: dict[str, Any]) -> None: diff --git a/test/data/test_search_queries.jsonl b/test/data/test_search_queries.jsonl new file mode 100644 index 00000000..9b00f397 --- /dev/null +++ b/test/data/test_search_queries.jsonl @@ -0,0 +1,3 @@ +{"query":"BiMLP"} +{"query":"海带"} +{"query":"pam"} diff --git a/test/scripts/retrieval/test_search_client.py b/test/scripts/retrieval/test_search_client.py index 05437276..3bde4619 100644 --- a/test/scripts/retrieval/test_search_client.py +++ b/test/scripts/retrieval/test_search_client.py @@ -2,7 +2,14 @@ import pytest -from dingo.retrieval.search_client import PaperResult, SearchClient, SearchResponse, create_client, list_backends, register_backend +from dingo.retrieval.search_client import ( # isort: skip + PaperResult, + SearchClient, + SearchResponse, + create_client, + list_backends, + register_backend, +) class TestPaperResult: @@ -95,6 +102,23 @@ def test_create_meta_search_client(self): } ] + def test_meta_search_resource_type_becomes_metadata_filter(self): + client = create_client( + "meta_search", + api_url="https://api.sciverse.space", + search_type="ebook", + ) + + payload = client._build_public_payload("test query", 10) + + assert payload["filters"] == [ + { + "field": "metadata_type", + "operator": "FILTER_OP_EQ", + "value": "ebook", + } + ] + def test_create_openalex_client_defaults_to_search(self): client = create_client( "openalex", @@ -189,6 +213,22 @@ def test_parse_serpapi_result(self): class TestOpenAlexClient: + def test_accepts_full_works_endpoint(self): + from dingo.retrieval.backends.openalex import OpenAlexClient + + client = OpenAlexClient(api_url="https://api.openalex.org/works") + + assert client.base_url == "https://api.openalex.org" + + def test_default_select_contains_quality_evaluation_fields(self): + from dingo.retrieval.backends.openalex import OpenAlexClient + + select = OpenAlexClient()._build_params("test", 10)["select"] + + assert "authorships" in select + assert "keywords" in select + assert "primary_location" in select + def test_abstract_from_inverted_index(self): from dingo.retrieval.backends.openalex import OpenAlexClient diff --git a/test/scripts/retrieval/test_search_result_eval_script.py b/test/scripts/retrieval/test_search_result_eval_script.py new file mode 100644 index 00000000..ed300db0 --- /dev/null +++ b/test/scripts/retrieval/test_search_result_eval_script.py @@ -0,0 +1,272 @@ +"""Tests for the end-to-end search result evaluation example.""" + +import json +import os +import sys +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace + +EXAMPLE_DIR = Path(__file__).resolve().parents[3] / "examples" / "retrieval" +if str(EXAMPLE_DIR) not in sys.path: + sys.path.insert(0, str(EXAMPLE_DIR)) + +import sdk_eval_search_result # noqa: E402 +from sdk_eval_search_result import build_reports, clear_executor_classification_dirs, load_env_file, normalize_search_result, retrieve_queries # noqa: E402,E501 +from search_result_eval_utils import load_queries, write_classified_jsonl # noqa: E402 + +from dingo.retrieval.search_client import PaperResult, SearchResponse # noqa: E402 + + +def _record( + query: str, + rank: int, + relevance: float, + effectiveness: float, + authority: float, +) -> dict: + return { + "raw_data": { + "query": query, + "query_index": 1, + "rank": rank, + "title": f"Result {rank}", + "search_result": { + "title": f"Result {rank}", + "abstract": f"Abstract {rank}", + "_eval_query": query, + }, + }, + "eval_details": { + "search_result": [ + {"metric": "LLMSearchResultRelevance", "score": relevance}, + {"metric": "LLMSearchResultEffectiveness", "score": effectiveness}, + {"metric": "LLMSearchResultAuthority", "score": authority}, + ] + } + } + + +def _args() -> Namespace: + return Namespace( + top_k=10, + threshold=0.15, + openai_model="test-model", + prompt_mode="detailed", + llm_max_tokens=1024, + openai_temperature=0.0, + llm_timeout=60.0, + llm_workers=1, + disable_effectiveness_llm_quality=False, + effectiveness_llm_max_tokens=512, + ) + + +def test_load_queries_supports_query_only_jsonl(tmp_path): + path = tmp_path / "queries.jsonl" + path.write_text( + '\n'.join((json.dumps({"query": "first"}), json.dumps({"query_text": "second"}))), + encoding="utf-8", + ) + + assert load_queries(path) == ["first", "second"] + + +def test_load_queries_deduplicates_queries(tmp_path): + path = tmp_path / "queries.jsonl" + path.write_text( + "\n".join((json.dumps({"query": "same"}), json.dumps({"query": "same"}))), + encoding="utf-8", + ) + + assert load_queries(path) == ["same"] + + +def test_load_env_file_does_not_override_process_environment(tmp_path, monkeypatch): + path = tmp_path / ".env" + path.write_text("EXISTING=value-from-file\nNEW_VALUE=loaded\n", encoding="utf-8") + monkeypatch.setenv("EXISTING", "value-from-process") + monkeypatch.delenv("NEW_VALUE", raising=False) + + load_env_file(path) + + assert os.environ["EXISTING"] == "value-from-process" + assert os.environ["NEW_VALUE"] == "loaded" + + +def test_retrieve_queries_writes_reusable_results_and_request_log(tmp_path, monkeypatch): + input_path = tmp_path / "queries.jsonl" + result_path = tmp_path / "retrieval_results.jsonl" + log_path = tmp_path / "request_log.jsonl" + input_path.write_text(json.dumps({"query": "test query"}), encoding="utf-8") + + class FakeClient: + def search(self, query, limit=10): + return SearchResponse( + query=query, + results=[PaperResult(paper_id="p1", title="Test result", raw={"title": "Test result"})], + response_time_ms=12.5, + status_code=200, + ) + + monkeypatch.setattr(sdk_eval_search_result, "_build_search_client", lambda args: FakeClient()) + args = Namespace( + input_jsonl=input_path, + max_queries=None, + top_k=10, + search_workers=1, + retrieval_backend="meta_search", + ) + + summary = retrieve_queries(args, result_path, log_path) + saved = json.loads(result_path.read_text(encoding="utf-8")) + log = json.loads(log_path.read_text(encoding="utf-8")) + + assert saved["query"] == "test query" + assert saved["results"][0]["title"] == "Test result" + assert log["result_count"] == 1 + assert summary["success_count"] == 1 + + +def test_normalize_openalex_result_maps_metric_fields(): + raw = { + "id": "https://openalex.org/W1", + "doi": "https://doi.org/10.1/example", + "cited_by_count": 12, + "publication_year": 2025, + "type": "article", + "language": "en", + "keywords": [{"display_name": "Search"}], + "authorships": [ + {"author": {"display_name": "A. Author", "orcid": "https://orcid.org/1"}} + ], + "primary_location": { + "source": { + "display_name": "Journal of Testing", + "type": "journal", + "issn": ["1234-5678"], + "host_organization_name": "Test Publisher", + } + }, + } + paper = PaperResult( + paper_id=raw["id"], + title="A result", + abstract="An abstract", + score=9.5, + year=2025, + raw=raw, + ) + + result = normalize_search_result(paper, "openalex") + + assert result["citation_count"] == 12 + assert result["keywords"] == ["Search"] + assert result["author"][0]["name"] == "A. Author" + assert result["publication_venue_name_unified"] == "Journal of Testing" + assert result["publication_venue_type"] == "journal" + assert result["publication_publisher"] == ["Test Publisher"] + + +def test_meta_search_accepts_full_endpoint(): + from dingo.retrieval.backends.agentic import MetaSearchClient + + client = MetaSearchClient(api_url="https://api.sciverse.space/meta-search") + + assert client.base_url == "https://api.sciverse.space" + + +def test_query_report_uses_rank_weighted_means_and_embeds_full_results(): + records = [ + _record("query", 1, relevance=0.0, effectiveness=0.1, authority=0.3), + _record("query", 2, relevance=0.2, effectiveness=0.2, authority=0.5), + ] + + summary, query_rows, _, _, classified = build_reports( + records, + _args(), + SimpleNamespace(output_path="output/test"), + ) + + assert summary["query_aggregation"] == "rank_discounted_mean" + assert query_rows[0]["relevance"] == 0.07737 + assert query_rows[0]["effectiveness"] == 0.13869 + assert query_rows[0]["authority"] == 0.37737 + assert query_rows[0]["relevance_aggregation"] == "rank_discounted_mean" + assert classified[0]["labels"] == [ + "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW", + "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW", + ] + assert classified[0]["results"][0]["abstract"] == "Abstract 1" + assert classified[0]["results"][0]["_evaluation"] == { + "rank": 1, + "relevance": 0.0, + "effectiveness": 0.1, + "authority": 0.3, + } + assert "_eval_query" not in classified[0]["results"][0] + + +def test_query_report_has_no_overall_and_empty_query_uses_three_low_labels(): + summary, query_rows, result_rows, _, classified = build_reports( + [], + _args(), + SimpleNamespace(output_path="output/test"), + empty_queries=["empty query"], + ) + + assert "weights" not in summary + assert "overall" not in summary["metrics"] + assert "overall" not in query_rows[0] + assert result_rows == [] + assert classified[0]["labels"] == [ + "QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW", + "QUALITY_BAD.SEARCH_RESULT_EFFECTIVENESS_LOW", + "QUALITY_BAD.SEARCH_RESULT_AUTHORITY_LOW", + ] + + +def test_result_level_executor_summary_is_not_exposed(): + summary, *_ = build_reports( + [], + _args(), + SimpleNamespace(output_path="output/test"), + ) + + assert "result_level" not in summary + + +def test_clear_executor_classification_dirs(tmp_path): + (tmp_path / "bad" / "result_level").mkdir(parents=True) + (tmp_path / "good").mkdir() + + clear_executor_classification_dirs(tmp_path) + + assert not (tmp_path / "bad").exists() + assert not (tmp_path / "good").exists() + + +def test_classified_output_contains_query_records_only(tmp_path): + records = [ + { + "query": "first", + "eval_status": True, + "labels": ["QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW"], + "results": [{"title": "First result"}], + }, + { + "query": "second", + "eval_status": True, + "labels": ["QUALITY_BAD.SEARCH_RESULT_RELEVANCE_LOW"], + "results": [{"title": "Second result"}], + }, + ] + + write_classified_jsonl(tmp_path, records) + + output = tmp_path / "bad" / "QUALITY_BAD" / "SEARCH_RESULT_RELEVANCE_LOW.jsonl" + lines = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + assert [line["query"] for line in lines] == ["first", "second"] + assert all(line["results"] for line in lines) + assert not (tmp_path / "bad" / "query_level").exists() + assert not (tmp_path / "bad" / "result_level").exists() From ffe5b13d6a594de297ada9530d6bf81da0ea6641 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 12:12:52 +0800 Subject: [PATCH 29/80] feat: lint --- test/scripts/model/rule/test_rule_common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 1d4ce268..b0820df9 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,8 +4,8 @@ from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords -from dingo.model.rule.rule_guobiao import (RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, - RuleDocContentFeatureCompleteness, RuleDataTimeRange, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) +from dingo.model.rule.rule_guobiao import (RuleDataTimeRange, RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, + RuleDocContentFeatureCompleteness, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: From 479b956b5a29856264228cf7f788176c82d11cc4 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 14:48:28 +0800 Subject: [PATCH 30/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E7=B1=BB?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_guobiao.py | 46 +++++------ docs/metrics.md | 8 +- docs/rules.md | 14 ++-- examples/guobiao/doc_completeness.py | 6 +- examples/guobiao/text_perplexity.py | 8 +- examples/guobiao/time_range.py | 6 +- examples/guobiao/type_consistency.py | 6 +- test/scripts/model/rule/test_rule_common.py | 86 ++++++++++----------- 8 files changed, 90 insertions(+), 90 deletions(-) diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py index 8d3f6bda..44ccee48 100644 --- a/dingo/model/rule/rule_guobiao.py +++ b/dingo/model/rule/rule_guobiao.py @@ -11,7 +11,7 @@ @Model.rule_register("QUALITY_BAD_TYPE_CONSISTENCY", ["guobiao"]) -class RuleDataTypeConsistency(BaseRule): +class Rule_TC609_0207_DataTypeConsistency(BaseRule): """Check whether content belongs to the type declared in ``input_data.type``. A local zero-shot classifier evaluates the hypothesis ``这段文本属于{type}类型``. @@ -21,7 +21,7 @@ class RuleDataTypeConsistency(BaseRule): _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "TYPE_CONSISTENCY", - "metric_name": "RuleDataTypeConsistency", + "metric_name": "Rule_TC609_0207_DataTypeConsistency", "description": ( "Uses a local zero-shot classifier to check whether content belongs " "to the type declared in the record" @@ -60,7 +60,7 @@ def _get_classifier(cls, model_name, device): ] if missing_packages: raise ImportError( - "RuleDataTypeConsistency requires optional packages: " + "Rule_TC609_0207_DataTypeConsistency requires optional packages: " f"{', '.join(missing_packages)}. " 'Install them with: pip install "dingo-python[hhem]"' ) @@ -120,7 +120,7 @@ def eval(cls, input_data: Data) -> EvalDetail: threshold = cls.dynamic_config.threshold if threshold is None or not 0 < threshold <= 1: raise ValueError( - "RuleDataTypeConsistency dynamic_config.threshold must be in (0, 1]" + "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]" ) model_name = cls.dynamic_config.model @@ -147,13 +147,13 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TIMELINESS", ["guobiao"]) -class RuleDataTimeRange(BaseRule): +class Rule_TC609_0303_DataTimeRange(BaseRule): """Check whether creation/update time fields are within configured ranges.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "TIMELINESS", - "metric_name": "RuleDataTimeRange", + "metric_name": "Rule_TC609_0303_DataTimeRange", "description": ( "Checks whether created and updated timestamps are within configured " "time ranges" @@ -238,7 +238,7 @@ def eval(cls, input_data: Data) -> EvalDetail: if dt_start is None and dt_end is None: raise ValueError( - "RuleDataTimeRange requires at least one configured range boundary in dynamic_config" + "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config" ) dt_value = getattr(input_data, "dt", None) @@ -280,13 +280,13 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain", "guobiao"]) -class RuleTextPerplexity(BaseRule): +class Rule_TC609_02080101_TextPerplexity(BaseRule): """Check whether text perplexity exceeds the configured threshold.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "FLUENCY", - "metric_name": "RuleTextPerplexity", + "metric_name": "Rule_TC609_02080101_TextPerplexity", "description": ( "Calculates text perplexity with a causal language model and " "flags text whose PPL exceeds the configured threshold" @@ -318,7 +318,7 @@ def _check_dependencies(cls): ] if missing_packages: raise ImportError( - "RuleTextPerplexity requires optional packages: " + "Rule_TC609_02080101_TextPerplexity requires optional packages: " f"{', '.join(missing_packages)}. " 'Install them with: pip install "dingo-python[hhem]"' ) @@ -336,7 +336,7 @@ def _get_model_components(cls, model_name): from transformers import AutoModelForCausalLM, AutoTokenizer except ImportError as exc: raise ImportError( - "RuleTextPerplexity requires transformers and torch. " + "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " 'Install them with: pip install "dingo-python[hhem]"' ) from exc @@ -352,7 +352,7 @@ def _calculate_perplexity(cls, content, tokenizer, model, stride): import torch except ImportError as exc: raise ImportError( - "RuleTextPerplexity requires transformers and torch. " + "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " 'Install them with: pip install "dingo-python[hhem]"' ) from exc @@ -361,7 +361,7 @@ def _calculate_perplexity(cls, content, tokenizer, model, stride): sequence_length = input_ids.size(1) if sequence_length < 2: raise ValueError( - "RuleTextPerplexity requires at least two model tokens" + "Rule_TC609_02080101_TextPerplexity requires at least two model tokens" ) model_config = getattr(model, "config", None) @@ -401,7 +401,7 @@ def _calculate_perplexity(cls, content, tokenizer, model, stride): if total_loss_tokens == 0: raise ValueError( - "RuleTextPerplexity could not calculate loss for the input" + "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input" ) mean_loss = total_negative_log_likelihood / total_loss_tokens @@ -425,7 +425,7 @@ def eval(cls, input_data: Data) -> EvalDetail: threshold = cls.dynamic_config.threshold if threshold is None or threshold <= 0: raise ValueError( - "RuleTextPerplexity dynamic_config.threshold must be greater than 0" + "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0" ) model_name = getattr( @@ -667,13 +667,13 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class RuleDocBasicInfoCompleteness(_RuleDatasetDocCompletenessBase): +class Rule_TC609_0101_DocBasicInfoCompleteness(_RuleDatasetDocCompletenessBase): """0101: Basic information completeness in dataset documentation.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "COMPLETENESS", - "metric_name": "RuleDocBasicInfoCompleteness", + "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness", "description": ( "Checks whether dataset documentation covers basic information " "aspects such as scale, format, structure, access, and support" @@ -699,13 +699,13 @@ class RuleDocBasicInfoCompleteness(_RuleDatasetDocCompletenessBase): @Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class RuleDocContentFeatureCompleteness(_RuleDatasetDocCompletenessBase): +class Rule_TC609_0102_DocContentFeatureCompleteness(_RuleDatasetDocCompletenessBase): """0102: Content feature completeness in dataset documentation.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "COMPLETENESS", - "metric_name": "RuleDocContentFeatureCompleteness", + "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness", "description": ( "Checks whether dataset documentation covers content-feature aspects " "such as modality, distribution, labels, examples, and limitations" @@ -731,13 +731,13 @@ class RuleDocContentFeatureCompleteness(_RuleDatasetDocCompletenessBase): @Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class RuleDocConstructionProcessCompleteness(_RuleDatasetDocCompletenessBase): +class Rule_TC609_0103_DocConstructionProcessCompleteness(_RuleDatasetDocCompletenessBase): """0103: Construction-process completeness in dataset documentation.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "COMPLETENESS", - "metric_name": "RuleDocConstructionProcessCompleteness", + "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness", "description": ( "Checks whether dataset documentation covers construction-process " "aspects such as data source, collection, processing, annotation, " @@ -764,13 +764,13 @@ class RuleDocConstructionProcessCompleteness(_RuleDatasetDocCompletenessBase): @Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class RuleDocApplicationCompleteness(_RuleDatasetDocCompletenessBase): +class Rule_TC609_0104_DocApplicationCompleteness(_RuleDatasetDocCompletenessBase): """0104: Application-description completeness in dataset documentation.""" _metric_info = { "category": "National Standard Data Quality Metrics", "quality_dimension": "COMPLETENESS", - "metric_name": "RuleDocApplicationCompleteness", + "metric_name": "Rule_TC609_0104_DocApplicationCompleteness", "description": ( "Checks whether dataset documentation covers application aspects " "such as license, scenarios, evaluation method, benchmark, and cases" diff --git a/docs/metrics.md b/docs/metrics.md index 3bf09db4..eac97126 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -60,14 +60,14 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, RuleDocContentFeatureCompleteness, RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks dataset-documentation completeness across basic information, content features, construction process, and application guidance, together with text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks dataset-documentation completeness across basic information, content features, construction process, and application guidance, together with text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleTextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, Rule_TC609_02080101_TextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_TIMELINESS` | RuleDataTimeRange | Checks whether `data.dt` falls within the configured `dt_start` and `dt_end` range required by the target application scenario. | High-quality dataset quality evaluation specification (SAC/TC609) | N/A | N/A | -| `QUALITY_BAD_TYPE_CONSISTENCY` | RuleDataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | +| `QUALITY_BAD_TIMELINESS` | Rule_TC609_0303_DataTimeRange | Checks whether `data.dt` falls within the configured `dt_start` and `dt_end` range required by the target application scenario. | High-quality dataset quality evaluation specification (SAC/TC609) | N/A | N/A | +| `QUALITY_BAD_TYPE_CONSISTENCY` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | | `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics diff --git a/docs/rules.md b/docs/rules.md index 667d5e12..f2b63d9a 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -2,6 +2,13 @@ The specific rules for each quality metric are as follows: | Function Name | Type | Description | Reference | |------------------------------|-------------------|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Rule_TC609_0101_DocBasicInfoCompleteness | COMPLETENESS | Check whether dataset documentation covers dataset scale, format specification, file structure, access channel, and technical support. | 2025 High-quality dataset quality evaluation specification | +| Rule_TC609_0102_DocContentFeatureCompleteness | COMPLETENESS | Check whether dataset documentation covers modality type, data distribution, label statistics, sample examples, and limitations. | 2025 High-quality dataset quality evaluation specification | +| Rule_TC609_0103_DocConstructionProcessCompleteness | COMPLETENESS | Check whether dataset documentation covers data sources, collection methods, processing pipeline, annotation specification, and version control. | 2025 High-quality dataset quality evaluation specification | +| Rule_TC609_0104_DocApplicationCompleteness | COMPLETENESS | Check whether dataset documentation covers licensing, target scenarios, evaluation methods, benchmark results, and typical cases. | 2025 High-quality dataset quality evaluation specification | +| Rule_TC609_0207_DataTypeConsistency | TYPE_CONSISTENCY | Use a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | 2025 High-quality dataset classification guide | +| Rule_TC609_02080101_TextPerplexity | FLUENCY | Calculate text perplexity with a configurable causal language model and flag values above the configured threshold. | 2025 High-quality dataset quality evaluation specification | +| Rule_TC609_0303_DataTimeRange | TIMELINESS | Check whether `data.dt` is within the configured `dt_start` and `dt_end` time range. | 2025 High-quality dataset quality evaluation specification | | RuleAbnormalChar | EFFECTIVENESS | Check whether content contains abnormal characters. | | | RuleAbnormalHtml | EFFECTIVENESS | Check whether content contains abnormal HTML. | | | RuleAbnormalNumber | FLUENCY | Check PDF content for abnormal page or index numbers. | | @@ -21,13 +28,7 @@ The specific rules for each quality metric are as follows: | RuleContentShort | EFFECTIVENESS | Check whether content is too short. | | | RuleContentShortMultiLan | EFFECTIVENESS | Check whether multilingual content is too short. | | | RuleCurlyBracket | UNDERSTANDABILITY | check whether the ratio of the number of {,} and the number of characters < 0.025 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [C4](https://arxiv.org/abs/1910.10683) | -| RuleDataTimeRange | TIMELINESS | Check whether `data.dt` is within the configured `dt_start` and `dt_end` time range. | 2025 High-quality dataset quality evaluation specification | -| RuleDataTypeConsistency | TYPE_CONSISTENCY | Use a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | 2025 High-quality dataset classification guide | | RuleDictConsistency | EFFECTIVENESS | Compare two dictionary fields and report mismatched keys. | | -| RuleDocApplicationCompleteness | COMPLETENESS | Check whether dataset documentation covers licensing, target scenarios, evaluation methods, benchmark results, and typical cases. | 2025 High-quality dataset quality evaluation specification | -| RuleDocBasicInfoCompleteness | COMPLETENESS | Check whether dataset documentation covers dataset scale, format specification, file structure, access channel, and technical support. | 2025 High-quality dataset quality evaluation specification | -| RuleDocConstructionProcessCompleteness | COMPLETENESS | Check whether dataset documentation covers data sources, collection methods, processing pipeline, annotation specification, and version control. | 2025 High-quality dataset quality evaluation specification | -| RuleDocContentFeatureCompleteness | COMPLETENESS | Check whether dataset documentation covers modality type, data distribution, label statistics, sample examples, and limitations. | 2025 High-quality dataset quality evaluation specification | | RuleDocFormulaRepeat | SIMILARITY | Check whether formulas repeat in a document. | | | RuleDocRepeat | SIMILARITY | check whether content repeats | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) [Gopher](https://arxiv.org/abs/2112.11446) | | RuleDoi | EFFECTIVENESS | Validate DOI metadata. | | @@ -92,7 +93,6 @@ The specific rules for each quality metric are as follows: | RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | | RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | | RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| RuleTextPerplexity | FLUENCY | Calculate text perplexity with a configurable causal language model and flag values above the configured threshold. | 2025 High-quality dataset quality evaluation specification | | RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | | RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | | | RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | | diff --git a/examples/guobiao/doc_completeness.py b/examples/guobiao/doc_completeness.py index c3cb656c..6b9a4dae 100644 --- a/examples/guobiao/doc_completeness.py +++ b/examples/guobiao/doc_completeness.py @@ -9,7 +9,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import RuleDocBasicInfoCompleteness +from dingo.model.rule.rule_guobiao import Rule_TC609_0101_DocBasicInfoCompleteness def main(): @@ -18,10 +18,10 @@ def main(): content="本数据集说明文档包含数据集规模与样本数量说明，提供格式规范、文件结构、访问渠道和技术支持方式。" ) - RuleDocBasicInfoCompleteness.dynamic_config = EvaluatorRuleArgs( + Rule_TC609_0101_DocBasicInfoCompleteness.dynamic_config = EvaluatorRuleArgs( threshold=0.8, ) - result = RuleDocBasicInfoCompleteness.eval(data) + result = Rule_TC609_0101_DocBasicInfoCompleteness.eval(data) print(result) diff --git a/examples/guobiao/text_perplexity.py b/examples/guobiao/text_perplexity.py index 7920819e..40e08d4e 100644 --- a/examples/guobiao/text_perplexity.py +++ b/examples/guobiao/text_perplexity.py @@ -1,4 +1,4 @@ -"""Evaluate one Chinese text using the national-standard perplexity rule. +"""Evaluate one Chinese text using the national-standard perplexity rule. Optional dependencies: conda run -n dingo pip install "dingo-python[hhem]" @@ -10,7 +10,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import RuleTextPerplexity +from dingo.model.rule.rule_guobiao import Rule_TC609_02080101_TextPerplexity def main(): @@ -19,12 +19,12 @@ def main(): content="人工智能正在推动科学研究和产业应用快速发展。高质量数据集能够为模型训练提供准确、完整且具有代表性的样本，从而提高模型在真实应用场景中的稳定性和可靠性。", ) - RuleTextPerplexity.dynamic_config = EvaluatorRuleArgs( + Rule_TC609_02080101_TextPerplexity.dynamic_config = EvaluatorRuleArgs( threshold=100.0, model="uer/gpt2-chinese-cluecorpussmall", stride=512, ) - result = RuleTextPerplexity.eval(data) + result = Rule_TC609_02080101_TextPerplexity.eval(data) print(result) diff --git a/examples/guobiao/time_range.py b/examples/guobiao/time_range.py index 8911b9aa..04002e56 100644 --- a/examples/guobiao/time_range.py +++ b/examples/guobiao/time_range.py @@ -2,7 +2,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import RuleDataTimeRange +from dingo.model.rule.rule_guobiao import Rule_TC609_0303_DataTimeRange def main(): @@ -12,11 +12,11 @@ def main(): content="示例数据", ) - RuleDataTimeRange.dynamic_config = EvaluatorRuleArgs( + Rule_TC609_0303_DataTimeRange.dynamic_config = EvaluatorRuleArgs( dt_start="2025-01-01", dt_end="2025-12-31 23:59:59", ) - result = RuleDataTimeRange.eval(data) + result = Rule_TC609_0303_DataTimeRange.eval(data) print(result) diff --git a/examples/guobiao/type_consistency.py b/examples/guobiao/type_consistency.py index cfd5a66d..e5aa78e6 100644 --- a/examples/guobiao/type_consistency.py +++ b/examples/guobiao/type_consistency.py @@ -9,7 +9,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import RuleDataTypeConsistency +from dingo.model.rule.rule_guobiao import Rule_TC609_0207_DataTypeConsistency def main(): @@ -19,12 +19,12 @@ def main(): content="高血压患者应在医生指导下规律用药，并定期监测血压变化。", ) - RuleDataTypeConsistency.dynamic_config = EvaluatorRuleArgs( + Rule_TC609_0207_DataTypeConsistency.dynamic_config = EvaluatorRuleArgs( threshold=0.5, model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", device=-1, ) - result = RuleDataTypeConsistency.eval(data) + result = Rule_TC609_0207_DataTypeConsistency.eval(data) print(result) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index b0820df9..cd16fa19 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,8 +4,8 @@ from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords -from dingo.model.rule.rule_guobiao import (RuleDataTimeRange, RuleDataTypeConsistency, RuleDocApplicationCompleteness, RuleDocBasicInfoCompleteness, RuleDocConstructionProcessCompleteness, - RuleDocContentFeatureCompleteness, RuleTextPerplexity, _RuleDatasetDocCompletenessBase) +from dingo.model.rule.rule_guobiao import (Rule_TC609_0303_DataTimeRange, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, + Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_02080101_TextPerplexity, _RuleDatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: @@ -29,21 +29,21 @@ def test_rule_unsafe_words(self): assert 'java' in tmp.reason -class TestRuleTextPerplexity: +class TestRule_TC609_02080101_TextPerplexity: @staticmethod def _mock_model(monkeypatch, perplexity): monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "_check_dependencies", classmethod(lambda cls: None), ) monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "_get_model_components", classmethod(lambda cls, model_name: (object(), object())), ) monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "_calculate_perplexity", classmethod( lambda cls, content, tokenizer, model, stride: perplexity @@ -53,7 +53,7 @@ def _mock_model(monkeypatch, perplexity): def test_high_perplexity_is_bad(self, monkeypatch): self._mock_model(monkeypatch, 125.5) monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "dynamic_config", EvaluatorRuleArgs( threshold=100.0, @@ -62,19 +62,19 @@ def test_high_perplexity_is_bad(self, monkeypatch): ), ) - res = RuleTextPerplexity.eval( + res = Rule_TC609_02080101_TextPerplexity.eval( Data(data_id="ppl-high", content="A valid piece of text.") ) assert res.status is True - assert res.label == ["QUALITY_BAD_FLUENCY.RuleTextPerplexity"] + assert res.label == ["QUALITY_BAD_FLUENCY.Rule_TC609_02080101_TextPerplexity"] assert "125.5000" in res.reason[0] assert "test-model" in res.reason[0] def test_low_perplexity_is_good(self, monkeypatch): self._mock_model(monkeypatch, 42.25) monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "dynamic_config", EvaluatorRuleArgs( threshold=100.0, @@ -83,7 +83,7 @@ def test_low_perplexity_is_good(self, monkeypatch): ), ) - res = RuleTextPerplexity.eval( + res = Rule_TC609_02080101_TextPerplexity.eval( Data(data_id="ppl-low", content="A fluent piece of text.") ) @@ -93,7 +93,7 @@ def test_low_perplexity_is_good(self, monkeypatch): def test_empty_content_is_bad_without_loading_model(self, monkeypatch): monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "_check_dependencies", classmethod(lambda cls: None), ) @@ -102,15 +102,15 @@ def fail_if_called(cls, model_name): raise AssertionError("model should not be loaded for empty content") monkeypatch.setattr( - RuleTextPerplexity, + Rule_TC609_02080101_TextPerplexity, "_get_model_components", classmethod(fail_if_called), ) - res = RuleTextPerplexity.eval(Data(data_id="ppl-empty", content=" ")) + res = Rule_TC609_02080101_TextPerplexity.eval(Data(data_id="ppl-empty", content=" ")) assert res.status is True - assert res.label == ["QUALITY_BAD_FLUENCY.RuleTextPerplexity"] + assert res.label == ["QUALITY_BAD_FLUENCY.Rule_TC609_02080101_TextPerplexity"] assert "empty content" in res.reason[0] def test_missing_dependencies_raise_clear_error(self, monkeypatch): @@ -120,7 +120,7 @@ def test_missing_dependencies_raise_clear_error(self, monkeypatch): ) try: - RuleTextPerplexity.eval( + Rule_TC609_02080101_TextPerplexity.eval( Data(data_id="ppl-dependency", content="A piece of text.") ) except ImportError as exc: @@ -130,23 +130,23 @@ def test_missing_dependencies_raise_clear_error(self, monkeypatch): raise AssertionError("expected ImportError for missing transformers") -class TestRuleDataTypeConsistency: +class TestRule_TC609_0207_DataTypeConsistency: @staticmethod def _mock_match_score(monkeypatch, score): monkeypatch.setattr( - RuleDataTypeConsistency, + Rule_TC609_0207_DataTypeConsistency, "_calculate_match_score", classmethod(lambda cls, *args: score), ) monkeypatch.setattr( - RuleDataTypeConsistency, + Rule_TC609_0207_DataTypeConsistency, "dynamic_config", EvaluatorRuleArgs(threshold=0.6, model="test-model", device=-1), ) def test_content_matching_declared_type_is_good(self, monkeypatch): self._mock_match_score(monkeypatch, 0.85) - result = RuleDataTypeConsistency.eval( + result = Rule_TC609_0207_DataTypeConsistency.eval( Data(data_id="type-match", type="medical", content="Clinical treatment") ) @@ -156,18 +156,18 @@ def test_content_matching_declared_type_is_good(self, monkeypatch): def test_content_not_matching_declared_type_is_bad(self, monkeypatch): self._mock_match_score(monkeypatch, 0.25) - result = RuleDataTypeConsistency.eval( + result = Rule_TC609_0207_DataTypeConsistency.eval( Data(data_id="type-mismatch", type="medical", content="Stock prices") ) assert result.status is True assert result.score == 0.25 assert result.label == [ - "QUALITY_BAD_TYPE_CONSISTENCY.RuleDataTypeConsistency" + "QUALITY_BAD_TYPE_CONSISTENCY.Rule_TC609_0207_DataTypeConsistency" ] def test_missing_type_is_bad(self): - result = RuleDataTypeConsistency.eval( + result = Rule_TC609_0207_DataTypeConsistency.eval( Data(data_id="type-missing", content="Ordinary text") ) @@ -175,10 +175,10 @@ def test_missing_type_is_bad(self): assert "missing or empty" in result.reason[0] -class TestRuleDataTimeRange: +class TestRule_TC609_0303_DataTimeRange: def test_dt_in_range_is_good(self, monkeypatch): monkeypatch.setattr( - RuleDataTimeRange, + Rule_TC609_0303_DataTimeRange, "dynamic_config", EvaluatorRuleArgs( dt_start="2025-01-01", @@ -186,7 +186,7 @@ def test_dt_in_range_is_good(self, monkeypatch): ), ) - result = RuleDataTimeRange.eval( + result = Rule_TC609_0303_DataTimeRange.eval( Data( data_id="time-good", dt="2025-03-01 08:30:00", @@ -197,7 +197,7 @@ def test_dt_in_range_is_good(self, monkeypatch): def test_created_time_out_of_range_is_bad(self, monkeypatch): monkeypatch.setattr( - RuleDataTimeRange, + Rule_TC609_0303_DataTimeRange, "dynamic_config", EvaluatorRuleArgs( dt_start="2025-01-01", @@ -205,19 +205,19 @@ def test_created_time_out_of_range_is_bad(self, monkeypatch): ), ) - result = RuleDataTimeRange.eval( + result = Rule_TC609_0303_DataTimeRange.eval( Data( data_id="time-created-out", dt="2024-12-31 23:59:59", ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] assert "earlier than allowed start" in result.reason[0] def test_dt_invalid_format_is_bad(self, monkeypatch): monkeypatch.setattr( - RuleDataTimeRange, + Rule_TC609_0303_DataTimeRange, "dynamic_config", EvaluatorRuleArgs( dt_start="2025-01-01", @@ -225,19 +225,19 @@ def test_dt_invalid_format_is_bad(self, monkeypatch): ), ) - result = RuleDataTimeRange.eval( + result = Rule_TC609_0303_DataTimeRange.eval( Data( data_id="time-dt-format", dt="2025年03月01日", ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] assert "unsupported datetime format" in result.reason[0] def test_missing_time_field_is_bad(self, monkeypatch): monkeypatch.setattr( - RuleDataTimeRange, + Rule_TC609_0303_DataTimeRange, "dynamic_config", EvaluatorRuleArgs( dt_start="2025-01-01", @@ -245,9 +245,9 @@ def test_missing_time_field_is_bad(self, monkeypatch): ), ) - result = RuleDataTimeRange.eval(Data(data_id="time-missing")) + result = Rule_TC609_0303_DataTimeRange.eval(Data(data_id="time-missing")) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.RuleDataTimeRange"] + assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] assert "dt is missing" in result.reason[0] @@ -473,7 +473,7 @@ def test_basic_info_completeness_good(self, monkeypatch): "本数据集说明包含数据集规模与样本数量，给出格式规范和文件结构，" "提供访问渠道，并说明技术支持联系方式。" ) - res = RuleDocBasicInfoCompleteness.eval( + res = Rule_TC609_0101_DocBasicInfoCompleteness.eval( Data(data_id="doc-basic-good", content=content) ) assert res.status is False @@ -483,12 +483,12 @@ def test_basic_info_completeness_good(self, monkeypatch): def test_basic_info_completeness_bad(self, monkeypatch): self._mock_aspect_matching(monkeypatch) content = "仅提到样本数量和文件结构，未说明访问渠道。" - res = RuleDocBasicInfoCompleteness.eval( + res = Rule_TC609_0101_DocBasicInfoCompleteness.eval( Data(data_id="doc-basic-bad", content=content) ) assert res.status is True assert res.label == [ - "QUALITY_BAD_COMPLETENESS.RuleDocBasicInfoCompleteness" + "QUALITY_BAD_COMPLETENESS.Rule_TC609_0101_DocBasicInfoCompleteness" ] assert res.score < 0.8 @@ -497,7 +497,7 @@ def test_content_feature_completeness_good(self, monkeypatch): content = ( "文档包含模态类型、数据分布情况、标签类别统计、样本示例以及局限性说明。" ) - res = RuleDocContentFeatureCompleteness.eval( + res = Rule_TC609_0102_DocContentFeatureCompleteness.eval( Data(data_id="doc-content-good", content=content) ) assert res.status is False @@ -509,7 +509,7 @@ def test_construction_process_completeness_good(self, monkeypatch): content = ( "建设过程包括数据来源、采集方法、加工处理流程、标注规范和版本控制记录。" ) - res = RuleDocConstructionProcessCompleteness.eval( + res = Rule_TC609_0103_DocConstructionProcessCompleteness.eval( Data(data_id="doc-process-good", content=content) ) assert res.status is False @@ -521,7 +521,7 @@ def test_application_completeness_good(self, monkeypatch): content = ( "应用说明提供使用许可、目标应用场景、评估方法、基准测试结果与典型应用案例。" ) - res = RuleDocApplicationCompleteness.eval( + res = Rule_TC609_0104_DocApplicationCompleteness.eval( Data(data_id="doc-application-good", content=content) ) assert res.status is False @@ -529,11 +529,11 @@ def test_application_completeness_good(self, monkeypatch): assert res.score == 1.0 def test_empty_content_is_bad(self): - res = RuleDocApplicationCompleteness.eval( + res = Rule_TC609_0104_DocApplicationCompleteness.eval( Data(data_id="doc-empty", content=" ") ) assert res.status is True assert res.label == [ - "QUALITY_BAD_COMPLETENESS.RuleDocApplicationCompleteness" + "QUALITY_BAD_COMPLETENESS.Rule_TC609_0104_DocApplicationCompleteness" ] assert "missing or empty" in res.reason[0] From 6e797e55fd812292bbba4a7a7d12f66e2b608430 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 14:52:15 +0800 Subject: [PATCH 31/80] feat: lint --- test/scripts/model/rule/test_rule_common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index cd16fa19..25f008f8 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,8 +4,9 @@ from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords -from dingo.model.rule.rule_guobiao import (Rule_TC609_0303_DataTimeRange, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, - Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_02080101_TextPerplexity, _RuleDatasetDocCompletenessBase) +from dingo.model.rule.rule_guobiao import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, + Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity, + _RuleDatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: From 2cabea7a0a747b9add4add7df8b15e84b24efdc9 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 27 Jul 2026 17:15:01 +0800 Subject: [PATCH 32/80] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81md=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/data/converter/base.py | 17 +++++ dingo/data/datasource/local.py | 38 +++++++++- docs/config.md | 3 +- examples/dataset/example_md.py | 69 +++++++++++++++++++ test/scripts/dataset/test_markdown_dataset.py | 65 +++++++++++++++++ 5 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 examples/dataset/example_md.py create mode 100644 test/scripts/dataset/test_markdown_dataset.py diff --git a/dingo/data/converter/base.py b/dingo/data/converter/base.py index 8707bb79..bc88d107 100644 --- a/dingo/data/converter/base.py +++ b/dingo/data/converter/base.py @@ -238,6 +238,23 @@ def _convert(raw: Union[str, Dict]): return _convert +@BaseConverter.register("md") +class MarkdownConverter(BaseConverter): + """Markdown file converter.""" + + def __init__(self): + super().__init__() + + @classmethod + def convertor(cls, input_args: InputArgs) -> Callable: + def _convert(raw: Union[str, Dict]): + if isinstance(raw, Dict): + return Data(**raw) + return Data(**{"id": "", "content": str(raw)}) + + return _convert + + @BaseConverter.register("jsonl") class JsonLineConverter(BaseConverter): """Json line file converter.""" diff --git a/dingo/data/datasource/local.py b/dingo/data/datasource/local.py index fb5a5f97..035daf3a 100644 --- a/dingo/data/datasource/local.py +++ b/dingo/data/datasource/local.py @@ -32,7 +32,7 @@ def to_dict(self) -> Dict[str, Any]: "config_name": self.config_name, } - def load(self, **kwargs) -> Generator[str, None, None]: + def load(self, **kwargs) -> Generator[Any, None, None]: """Load the local file dataset based on `LocalDataSource`. Args: kwargs: Additional keyword arguments used for loading the dataset. @@ -55,6 +55,25 @@ def _find_all_files(self, path: str, file_list: List[str]): if os.path.isdir(f): self._find_all_files(f, file_list) + @staticmethod + def _load_md_file(path: str) -> Dict[str, str]: + try: + with open(path, "r", encoding="utf-8") as md_file: + return { + "id": os.path.basename(path), + "content": md_file.read(), + } + except UnicodeDecodeError as decode_error: + raise RuntimeError( + f'Failed to read markdown file "{path}": Unsupported file encoding. ' + f'Markdown files must be UTF-8 encoded. Original error: {str(decode_error)}' + ) + except Exception as e: + raise RuntimeError( + f'Unexpected error reading markdown file "{path}": {str(e)}. ' + f'Please check if the file exists and is readable.' + ) + def _load_excel_file_xlsx(self, path: str) -> Generator[str, None, None]: """ Load an .xlsx Excel file and return its contents row by row as JSON strings. @@ -377,7 +396,7 @@ def _load_excel_file_xls(self, path: str) -> Generator[str, None, None]: if wb: wb.release_resources() - def _load_local_file(self) -> Generator[str, None, None]: + def _load_local_file(self) -> Generator[Any, None, None]: """ Load a local file and return its contents. @@ -390,14 +409,27 @@ def _load_local_file(self) -> Generator[str, None, None]: raise RuntimeError(f'"{self.path}" is not a valid path') f_list = [] + input_is_file = os.path.isfile(self.path) if os.path.exists(self.path) and os.path.isfile(self.path): f_list = [self.path] elif os.path.exists(self.path) and os.path.isdir(self.path): self._find_all_files(self.path, f_list) - by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2"] + by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2", "md"] for f in f_list: + if self.input_args.dataset.format == "md": + if f.lower().endswith(".md"): + yield self._load_md_file(f) + elif input_is_file: + raise RuntimeError( + f'Input file "{self.path}" is not a markdown file. ' + f'Please provide a ".md" file or a directory containing ".md" files when dataset.format is "md".' + ) + else: + continue + continue + # Check if file is CSV if f.endswith('.csv'): if self.input_args.dataset.format != 'csv': diff --git a/docs/config.md b/docs/config.md index 2e6104f2..ec595fcf 100644 --- a/docs/config.md +++ b/docs/config.md @@ -30,7 +30,7 @@ | Parameter | Type | Default | Required | Description | |-----------|------|---------|----------|-------------| | source | str | "hugging_face" | Yes | 数据源类型，可选值：['hugging_face', 'local'] | -| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'plaintext', 'listjson', 'csv', 'parquet', 'mineru', 'mineru_v2'] | +| format | str | "json" | Yes | 数据格式，可选值：['json', 'jsonl', 'plaintext', 'listjson', 'csv', 'parquet', 'md', 'mineru', 'mineru_v2'] | | field | object | - | Yes | 字段映射配置 | | hf_config | object | - | No | HuggingFace 特定配置 | | mineru_config | object | - | No | MinerU 格式特定配置（仅 mineru / mineru_v2 格式使用） | @@ -46,6 +46,7 @@ MinerU 格式特定配置，用于过滤 block 类型： MinerU 支持的 block 类型包括：`text`, `title`, `image`, `table`, `equation`, `code`, `list`, `header`, `page_footer`, `page_footnote`, `chart` 等。 **格式说明：** +- `md`：读取单个 `.md` 文件或目录下全部 `.md` 文件；每个文件构造成一条 `Data`，包含 `id`（文件名）和 `content`（全文字符串） - `mineru`：对应 MinerU 的 `content_list.json`，顶层为 block 数组 - `mineru_v2`：对应 MinerU 的 `content_list_v2.json`，顶层为页面数组，每页包含 block 数组 diff --git a/examples/dataset/example_md.py b/examples/dataset/example_md.py new file mode 100644 index 00000000..3279e6e5 --- /dev/null +++ b/examples/dataset/example_md.py @@ -0,0 +1,69 @@ +import tempfile +from pathlib import Path + +from dingo.config import InputArgs +from dingo.exec import Executor + + +def run_md_single_file_demo(): + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + md_file = tmp_path / "single.md" + md_file.write_text("# Single File Demo\n\nThis is markdown content.:", encoding="utf-8") + + input_data = { + "input_path": str(md_file), + "dataset": { + "source": "local", + "format": "md", + }, + "evaluator": [ + { + "fields": {"id": "id", "content": "content"}, + "evals": [ + {"name": "RuleColonEnd"}, + ], + } + ], + } + + input_args = InputArgs(**input_data) + executor = Executor.exec_map["local"](input_args) + result = executor.execute() + print("single file demo:") + print(result) + + +def run_md_directory_demo(): + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) + (tmp_path / "a.md").write_text("## A\n\nalpha content:", encoding="utf-8") + (tmp_path / "b.md").write_text("## B\n\nbeta content:", encoding="utf-8") + (tmp_path / "ignore.txt").write_text("this file will be ignored in md format", encoding="utf-8") + + input_data = { + "input_path": str(tmp_path), + "dataset": { + "source": "local", + "format": "md", + }, + "evaluator": [ + { + "fields": {"id": "id", "content": "content"}, + "evals": [ + {"name": "RuleColonEnd"}, + ], + } + ], + } + + input_args = InputArgs(**input_data) + executor = Executor.exec_map["local"](input_args) + result = executor.execute() + print("directory demo:") + print(result) + + +if __name__ == "__main__": + run_md_single_file_demo() + run_md_directory_demo() diff --git a/test/scripts/dataset/test_markdown_dataset.py b/test/scripts/dataset/test_markdown_dataset.py new file mode 100644 index 00000000..6552cfcb --- /dev/null +++ b/test/scripts/dataset/test_markdown_dataset.py @@ -0,0 +1,65 @@ +import pytest + +from dingo.config import InputArgs +from dingo.data.dataset.local import LocalDataset +from dingo.data.datasource.local import LocalDataSource + + +def test_markdown_single_file_to_data(tmp_path): + md_path = tmp_path / "article.md" + content = "# Title\n\nThis is markdown content.\n" + md_path.write_text(content, encoding="utf-8") + + input_args = InputArgs( + input_path=str(md_path), + dataset={"source": "local", "format": "md"}, + evaluator=[], + ) + + dataset = LocalDataset(source=LocalDataSource(input_args=input_args)) + rows = list(dataset.get_data()) + + assert len(rows) == 1 + assert rows[0].id == "article.md" + assert rows[0].content == content + + +def test_markdown_directory_only_reads_md_files(tmp_path): + md1 = tmp_path / "a.md" + txt = tmp_path / "ignore.txt" + csv_file = tmp_path / "table.csv" + subdir = tmp_path / "nested" + subdir.mkdir() + md2 = subdir / "b.md" + + md1.write_text("alpha", encoding="utf-8") + txt.write_text("should be ignored", encoding="utf-8") + csv_file.write_text("c1,c2\n1,2\n", encoding="utf-8") + md2.write_text("beta", encoding="utf-8") + + input_args = InputArgs( + input_path=str(tmp_path), + dataset={"source": "local", "format": "md"}, + evaluator=[], + ) + + dataset = LocalDataset(source=LocalDataSource(input_args=input_args)) + rows = list(dataset.get_data()) + + assert len(rows) == 2 + assert {row.id for row in rows} == {"a.md", "b.md"} + assert {row.content for row in rows} == {"alpha", "beta"} + + +def test_markdown_directory_without_md_files_returns_empty(tmp_path): + (tmp_path / "readme.txt").write_text("plain text", encoding="utf-8") + + input_args = InputArgs( + input_path=str(tmp_path), + dataset={"source": "local", "format": "md"}, + evaluator=[], + ) + + dataset = LocalDataset(source=LocalDataSource(input_args=input_args)) + rows = list(dataset.get_data()) + assert rows == [] From 15e4d2081e548184a4eddb5cd215be982dbe59b1 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 14:32:57 +0800 Subject: [PATCH 33/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E5=85=A8?= =?UTF-8?q?=E9=83=A8rule=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 + dingo/model/rule/rule_guobiao.py | 795 -------------------- docs/metrics.md | 51 +- docs/rules.md | 47 +- examples/guobiao/doc_completeness.py | 2 +- examples/guobiao/text_perplexity.py | 2 +- examples/guobiao/time_range.py | 2 +- examples/guobiao/type_consistency.py | 2 +- test/scripts/model/rule/test_rule_common.py | 24 +- 9 files changed, 105 insertions(+), 822 deletions(-) delete mode 100644 dingo/model/rule/rule_guobiao.py diff --git a/AGENTS.md b/AGENTS.md index 088b0898..aaa0d698 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,8 @@ dingo/ │ │ ├── rule/ ← Rule-based evaluators (80+ built-in) │ │ │ ├── base.py ← BaseRule │ │ │ ├── rule_common.py ← Common rules (text quality, format, PII, etc.) +│ │ │ ├── guobiao/ +│ │ │ │ └── rule_tc609_quality.py ← TC609 quality metrics and placeholders │ │ │ ├── rule_search_ranking.py ← IR ranking metrics (NDCG, MRR, Recall, Precision, MAP, HitRate) │ │ │ └── utils/ ← Shared utilities (normalize, ngrams, etc.) │ │ └── llm/ ← LLM-based evaluators diff --git a/dingo/model/rule/rule_guobiao.py b/dingo/model/rule/rule_guobiao.py deleted file mode 100644 index 44ccee48..00000000 --- a/dingo/model/rule/rule_guobiao.py +++ /dev/null @@ -1,795 +0,0 @@ -import importlib.util -import math -import re -from datetime import datetime, timezone - -from dingo.config.input_args import EvaluatorRuleArgs -from dingo.io.input import Data, RequiredField -from dingo.io.output.eval_detail import EvalDetail, QualityLabel -from dingo.model.model import Model -from dingo.model.rule.base import BaseRule - - -@Model.rule_register("QUALITY_BAD_TYPE_CONSISTENCY", ["guobiao"]) -class Rule_TC609_0207_DataTypeConsistency(BaseRule): - """Check whether content belongs to the type declared in ``input_data.type``. - - A local zero-shot classifier evaluates the hypothesis ``这段文本属于{type}类型``. - The declared type may be any non-empty string, such as ``医疗`` or ``金融``. - """ - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "TYPE_CONSISTENCY", - "metric_name": "Rule_TC609_0207_DataTypeConsistency", - "description": ( - "Uses a local zero-shot classifier to check whether content belongs " - "to the type declared in the record" - ), - "paper_title": "High-quality dataset classification guide", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - - _required_fields = [RequiredField.CONTENT, RequiredField.TYPE] - dynamic_config = EvaluatorRuleArgs( - threshold=0.5, - model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", - device=-1, - ) - - _model_name = None - _model_device = None - _classifier = None - - @classmethod - def _get_classifier(cls, model_name, device): - if ( - cls._model_name == model_name - and cls._model_device == device - and cls._classifier is not None - ): - return cls._classifier - - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - "Rule_TC609_0207_DataTypeConsistency requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - from transformers import pipeline - - cls._classifier = pipeline( - "zero-shot-classification", - model=model_name, - device=device, - ) - cls._model_name = model_name - cls._model_device = device - return cls._classifier - - @classmethod - def _calculate_match_score( - cls, content, declared_type, model_name, device - ): - classifier = cls._get_classifier(model_name, device) - result = classifier( - content, - candidate_labels=[declared_type], - hypothesis_template="这段文本属于{}类型。", - multi_label=True, - truncation=True, - ) - labels = result.get("labels", []) - scores = result.get("scores", []) - if not labels or not scores or labels[0] != declared_type: - raise RuntimeError("Zero-shot classifier returned an invalid result") - score = float(scores[0]) - if not math.isfinite(score) or not 0.0 <= score <= 1.0: - raise RuntimeError( - f"Zero-shot classifier returned an invalid score: {score}" - ) - return score - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - declared_type = getattr(input_data, "type", None) - content = getattr(input_data, "content", None) - - if not isinstance(declared_type, str) or not declared_type.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Data type is missing or empty"] - return res - - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Content is missing or empty"] - return res - - threshold = cls.dynamic_config.threshold - if threshold is None or not 0 < threshold <= 1: - raise ValueError( - "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]" - ) - - model_name = cls.dynamic_config.model - device = cls.dynamic_config.device - score = cls._calculate_match_score( - content, declared_type, model_name, device - ) - res.score = score - - if score >= threshold: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"Content matches declared type {declared_type} " - f"(score: {score:.4f}, threshold: {threshold:.4f})" - ] - else: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"Content does not match declared type {declared_type} " - f"(score: {score:.4f}, threshold: {threshold:.4f})" - ] - return res - - -@Model.rule_register("QUALITY_BAD_TIMELINESS", ["guobiao"]) -class Rule_TC609_0303_DataTimeRange(BaseRule): - """Check whether creation/update time fields are within configured ranges.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "TIMELINESS", - "metric_name": "Rule_TC609_0303_DataTimeRange", - "description": ( - "Checks whether created and updated timestamps are within configured " - "time ranges" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - - _required_fields = [RequiredField.DT] - dynamic_config = EvaluatorRuleArgs( - dt_start=None, - dt_end=None, - ) - - @classmethod - def _parse_datetime(cls, value, field_name): - if isinstance(value, datetime): - if value.tzinfo is None: - return value - return value.astimezone(timezone.utc).replace(tzinfo=None) - - if isinstance(value, str): - text = value.strip() - if not text: - raise ValueError(f"{field_name} is empty") - - iso_text = text.replace("Z", "+00:00") - try: - parsed = datetime.fromisoformat(iso_text) - if parsed.tzinfo is None: - return parsed - return parsed.astimezone(timezone.utc).replace(tzinfo=None) - except ValueError: - raise ValueError( - f"{field_name} has unsupported datetime format: {value!r}" - ) from None - - raise ValueError( - f"{field_name} has unsupported datetime format: {value!r}" - ) - - @classmethod - def _validate_time_range( - cls, - dt_value, - start_value, - end_value, - ): - parsed_dt = cls._parse_datetime(dt_value, "time_value") - parsed_dt_start = ( - cls._parse_datetime(start_value, "start_time") - if start_value is not None - else None - ) - parsed_dt_end = ( - cls._parse_datetime(end_value, "end_time") - if end_value is not None - else None - ) - - if ( - parsed_dt_start is not None - and parsed_dt_end is not None - and parsed_dt_start > parsed_dt_end - ): - raise ValueError("time range is invalid: start is later than end") - - if parsed_dt_start is not None and parsed_dt < parsed_dt_start: - return False, parsed_dt, parsed_dt_start, parsed_dt_end - if parsed_dt_end is not None and parsed_dt > parsed_dt_end: - return False, parsed_dt, parsed_dt_start, parsed_dt_end - return True, parsed_dt, parsed_dt_start, parsed_dt_end - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - - dt_start = getattr(cls.dynamic_config, "dt_start", None) - dt_end = getattr(cls.dynamic_config, "dt_end", None) - - if dt_start is None and dt_end is None: - raise ValueError( - "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config" - ) - - dt_value = getattr(input_data, "dt", None) - if dt_value is None: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["dt is missing"] - return res - - try: - is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range( - dt_value=dt_value, - start_value=dt_start, - end_value=dt_end, - ) - except ValueError as exc: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [f"dt: {exc}"] - return res - - if not is_valid: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - if parsed_dt_start is not None and parsed_dt < parsed_dt_start: - res.reason = [ - f"dt {parsed_dt.isoformat(sep=' ')} is earlier than " - f"allowed start {parsed_dt_start.isoformat(sep=' ')}" - ] - else: - res.reason = [ - f"dt {parsed_dt.isoformat(sep=' ')} is later than " - f"allowed end {parsed_dt_end.isoformat(sep=' ')}" - ] - return res - - res.label = [QualityLabel.QUALITY_GOOD] - return res - - -@Model.rule_register("QUALITY_BAD_FLUENCY", ["pretrain", "guobiao"]) -class Rule_TC609_02080101_TextPerplexity(BaseRule): - """Check whether text perplexity exceeds the configured threshold.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "FLUENCY", - "metric_name": "Rule_TC609_02080101_TextPerplexity", - "description": ( - "Calculates text perplexity with a causal language model and " - "flags text whose PPL exceeds the configured threshold" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "" - } - - _required_fields = [RequiredField.CONTENT] - dynamic_config = EvaluatorRuleArgs( - threshold=100.0, - model="uer/gpt2-chinese-cluecorpussmall", - stride=512, - ) - - _model_name = None - _tokenizer = None - _model = None - - @classmethod - def _check_dependencies(cls): - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - "Rule_TC609_02080101_TextPerplexity requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - @classmethod - def _get_model_components(cls, model_name): - if ( - cls._model_name == model_name - and cls._tokenizer is not None - and cls._model is not None - ): - return cls._tokenizer, cls._model - - try: - from transformers import AutoModelForCausalLM, AutoTokenizer - except ImportError as exc: - raise ImportError( - "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " - 'Install them with: pip install "dingo-python[hhem]"' - ) from exc - - cls._tokenizer = AutoTokenizer.from_pretrained(model_name) - cls._model = AutoModelForCausalLM.from_pretrained(model_name) - cls._model.eval() - cls._model_name = model_name - return cls._tokenizer, cls._model - - @classmethod - def _calculate_perplexity(cls, content, tokenizer, model, stride): - try: - import torch - except ImportError as exc: - raise ImportError( - "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " - 'Install them with: pip install "dingo-python[hhem]"' - ) from exc - - encodings = tokenizer(content, return_tensors="pt") - input_ids = encodings["input_ids"] - sequence_length = input_ids.size(1) - if sequence_length < 2: - raise ValueError( - "Rule_TC609_02080101_TextPerplexity requires at least two model tokens" - ) - - model_config = getattr(model, "config", None) - max_length = getattr(model_config, "n_positions", None) - if max_length is None: - max_length = getattr(model_config, "max_position_embeddings", 1024) - max_length = int(max_length) - stride = max(1, min(int(stride), max_length)) - - try: - device = next(model.parameters()).device - except StopIteration: - device = torch.device("cpu") - - total_negative_log_likelihood = 0.0 - total_loss_tokens = 0 - previous_end = 0 - - for begin in range(0, sequence_length, stride): - end = min(begin + max_length, sequence_length) - target_length = end - previous_end - window = input_ids[:, begin:end].to(device) - targets = window.clone() - targets[:, :-target_length] = -100 - - with torch.no_grad(): - output = model(window, labels=targets) - - loss_tokens = int((targets[:, 1:] != -100).sum().item()) - if loss_tokens > 0: - total_negative_log_likelihood += output.loss.item() * loss_tokens - total_loss_tokens += loss_tokens - - previous_end = end - if end == sequence_length: - break - - if total_loss_tokens == 0: - raise ValueError( - "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input" - ) - - mean_loss = total_negative_log_likelihood / total_loss_tokens - try: - return math.exp(mean_loss) - except OverflowError: - return float("inf") - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - cls._check_dependencies() - - res = EvalDetail(metric=cls.__name__) - content = input_data.content - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Text perplexity cannot be calculated for empty content"] - return res - - threshold = cls.dynamic_config.threshold - if threshold is None or threshold <= 0: - raise ValueError( - "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0" - ) - - model_name = getattr( - cls.dynamic_config, - "model", - "uer/gpt2-chinese-cluecorpussmall", - ) - stride = getattr(cls.dynamic_config, "stride", 512) - tokenizer, model = cls._get_model_components(model_name) - perplexity = cls._calculate_perplexity( - content, - tokenizer, - model, - stride, - ) - - if perplexity > threshold: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"Text perplexity {perplexity:.4f} exceeds threshold " - f"{threshold:.4f} (model: {model_name})" - ] - else: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"Text perplexity: {perplexity:.4f} " - f"(threshold: {threshold:.4f}, model: {model_name})" - ] - return res - - -class _RuleDatasetDocCompletenessBase(BaseRule): - """Shared logic for dataset documentation completeness checks.""" - - _required_fields = [RequiredField.CONTENT] - _default_threshold = 0.8 - _default_semantic_threshold = 0.5 - _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli" - _default_device = -1 - _default_dimension_name = "Dataset documentation" - _default_aspect_keywords = {} - dynamic_config = EvaluatorRuleArgs( - threshold=_default_threshold, - semantic_threshold=_default_semantic_threshold, - model=_default_model, - device=_default_device, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - _model_name = None - _model_device = None - _classifier = None - - @classmethod - def _get_classifier(cls, model_name, device): - if ( - cls._model_name == model_name - and cls._model_device == device - and cls._classifier is not None - ): - return cls._classifier - - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - f"{cls.__name__} requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - from transformers import pipeline - - cls._classifier = pipeline( - "zero-shot-classification", - model=model_name, - device=device, - ) - cls._model_name = model_name - cls._model_device = device - return cls._classifier - - @classmethod - def _calculate_aspect_score(cls, content, aspect_text, model_name, device): - classifier = cls._get_classifier(model_name, device) - result = classifier( - content, - candidate_labels=[aspect_text], - hypothesis_template="这段文本包含{}相关说明。", - multi_label=True, - truncation=True, - ) - labels = result.get("labels", []) - scores = result.get("scores", []) - if not labels or not scores or labels[0] != aspect_text: - raise RuntimeError("Zero-shot classifier returned an invalid result") - score = float(scores[0]) - if not math.isfinite(score) or not 0.0 <= score <= 1.0: - raise RuntimeError( - f"Zero-shot classifier returned an invalid score: {score}" - ) - return score - - @classmethod - def _match_aspects( - cls, - content, - normalized_content, - aspect_keywords, - model_name, - device, - semantic_threshold, - ): - matched = {} - missing = [] - for aspect_name, keywords in aspect_keywords.items(): - aspect_text = ( - f"{aspect_name}（关键词示例：{'、'.join(keywords)}）" - if keywords - else aspect_name - ) - score = cls._calculate_aspect_score( - content, aspect_text, model_name, device - ) - evidence_keyword = next( - ( - keyword - for keyword in keywords - if keyword.lower() in normalized_content - ), - None, - ) - if score >= semantic_threshold: - matched[aspect_name] = { - "score": round(score, 4), - "keyword": evidence_keyword, - } - else: - missing.append(aspect_name) - return matched, missing - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - content = getattr(input_data, "content", None) - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Content is missing or empty"] - res.score = 0.0 - return res - - threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold) - if threshold is None or not 0 < threshold <= 1: - raise ValueError( - f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" - ) - - semantic_threshold = getattr( - cls.dynamic_config, - "semantic_threshold", - cls._default_semantic_threshold, - ) - if semantic_threshold is None or not 0 < semantic_threshold <= 1: - raise ValueError( - f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]" - ) - - aspect_keywords = getattr( - cls.dynamic_config, - "aspect_keywords", - cls._default_aspect_keywords, - ) - if not isinstance(aspect_keywords, dict) or not aspect_keywords: - raise ValueError( - f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" - ) - - model_name = getattr(cls.dynamic_config, "model", cls._default_model) - if not isinstance(model_name, str) or not model_name.strip(): - raise ValueError( - f"{cls.__name__} dynamic_config.model must be a non-empty string" - ) - - device = getattr(cls.dynamic_config, "device", cls._default_device) - - dimension_name = getattr( - cls.dynamic_config, "dimension_name", cls._default_dimension_name - ) - - normalized_content = re.sub(r"\s+", "", content).lower() - matched, missing = cls._match_aspects( - content=content, - normalized_content=normalized_content, - aspect_keywords=aspect_keywords, - model_name=model_name, - device=device, - semantic_threshold=semantic_threshold, - ) - - total = len(aspect_keywords) - matched_count = len(matched) - score = matched_count / total if total else 0.0 - res.score = round(score, 4) - matched_desc = ( - ", ".join( - ( - f"{aspect}(semantic_score={detail['score']:.4f}, " - f"keyword_hit={detail['keyword'] or 'None'})" - ) - for aspect, detail in matched.items() - ) - if matched - else "None" - ) - missing_desc = ", ".join(missing) if missing else "None" - - if score < threshold: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"{dimension_name} completeness score {score:.4f} is below " - f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " - f"matched: {matched_desc}; missing: {missing_desc}" - ] - else: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"{dimension_name} completeness score {score:.4f} meets " - f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " - f"matched: {matched_desc}; missing: {missing_desc}" - ] - return res - - -@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class Rule_TC609_0101_DocBasicInfoCompleteness(_RuleDatasetDocCompletenessBase): - """0101: Basic information completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness", - "description": ( - "Checks whether dataset documentation covers basic information " - "aspects such as scale, format, structure, access, and support" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - _default_dimension_name = "Basic information" - _default_aspect_keywords = { - "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"], - "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"], - "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"], - "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"], - "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class Rule_TC609_0102_DocContentFeatureCompleteness(_RuleDatasetDocCompletenessBase): - """0102: Content feature completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness", - "description": ( - "Checks whether dataset documentation covers content-feature aspects " - "such as modality, distribution, labels, examples, and limitations" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - _default_dimension_name = "Content feature" - _default_aspect_keywords = { - "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"], - "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"], - "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"], - "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"], - "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class Rule_TC609_0103_DocConstructionProcessCompleteness(_RuleDatasetDocCompletenessBase): - """0103: Construction-process completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness", - "description": ( - "Checks whether dataset documentation covers construction-process " - "aspects such as data source, collection, processing, annotation, " - "and version control" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - _default_dimension_name = "Construction process" - _default_aspect_keywords = { - "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"], - "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"], - "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"], - "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"], - "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_COMPLETENESS", ["guobiao"]) -class Rule_TC609_0104_DocApplicationCompleteness(_RuleDatasetDocCompletenessBase): - """0104: Application-description completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0104_DocApplicationCompleteness", - "description": ( - "Checks whether dataset documentation covers application aspects " - "such as license, scenarios, evaluation method, benchmark, and cases" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - } - _default_dimension_name = "Application description" - _default_aspect_keywords = { - "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"], - "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"], - "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"], - "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"], - "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) diff --git a/docs/metrics.md b/docs/metrics.md index eac97126..745262d9 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -56,18 +56,61 @@ This document provides comprehensive information about all quality metrics used | `LLMClassifyQR` | LLMClassifyQR | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | N/A | | `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用 DeepSeek-OCR 作为 Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [See Results](通过对比 VLM 输出与 OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | +### TC609-5-2025-04 Quality Evaluation Metrics + +| Type | Rule | Coverage | Group | Description | +|---|---|---|---|---| +| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | covered | `guobiao` | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | +| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | covered | `guobiao` | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | +| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | covered | `guobiao` | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | +| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | covered | `guobiao` | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | partial | `guobiao` | Combines existing NLP, SFT, image, audio, and video format rules. | +| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | partial | `guobiao` | Combines unsafe-word, PII, and identity-card detection. | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | partial | `guobiao` | Combines image-label overlap and visualization checks. | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | partial | `guobiao` | Combines null-content and short-content checks. | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | partial | `guobiao` | Uses HHEM consistency checking as partial evidence of authenticity. | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | partial | `guobiao` | Combines structured-field and image-text consistency checks. | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | partial | `guobiao` | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | +| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | covered | `pretrain,guobiao` | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | +| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | partial | `guobiao` | Combines alphabetic-word, stop-word, and unique-word ratio checks. | +| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | covered | `guobiao` | Combines document-text and formula repetition checks. | +| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | covered | `guobiao` | Combines null, short, ellipsis-ending, and terminal-ending checks. | +| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | partial | `guobiao` | Uses content length and sentence/word counts as partial missing-information checks. | +| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | partial | `guobiao` | Combines abnormal HTML, character, invisible-content, and watermark checks. | +| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | partial | `guobiao` | Combines punctuation, word-boundary, and line-break fluency checks. | +| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | partial | `guobiao` | Uses image aspect-ratio validation as partial resolution coverage. | +| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | covered | `guobiao` | Uses PHash and CNN duplicate-image detection. | +| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | partial | `guobiao` | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | +| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | partial | `guobiao` | Combines image validity and NIMA quality as partial clarity coverage. | +| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | uncovered | `guobiao_placeholder` | Placeholder: video resolution is not implemented. | +| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | uncovered | `guobiao_placeholder` | Placeholder: duplicate-video detection is not implemented. | +| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | uncovered | `guobiao_placeholder` | Placeholder: video FPS validation is not implemented. | +| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | uncovered | `guobiao_placeholder` | Placeholder: video duration validation is not implemented. | +| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | uncovered | `guobiao_placeholder` | Placeholder: video clarity evaluation is not implemented. | +| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | uncovered | `guobiao_placeholder` | Placeholder: video dynamic-range evaluation is not implemented. | +| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | covered | `guobiao` | Uses the existing Welch power-spectrum SNR implementation. | +| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | uncovered | `guobiao_placeholder` | Placeholder: signal distortion ratio is not implemented. | +| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | uncovered | `guobiao_placeholder` | Placeholder: sample-rate quality validation is not implemented. | +| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | uncovered | `guobiao_placeholder` | Placeholder: audio bit-depth validation is not implemented. | +| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | uncovered | `guobiao_placeholder` | Placeholder: audio bit-rate validation is not implemented. | +| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | covered | `guobiao` | Uses the existing WAV duration implementation. | +| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | partial | `guobiao` | Combines available text cleanliness checks; modality coverage is partial. | +| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | uncovered | `guobiao_placeholder` | Placeholder: target-scenario distribution coverage is not implemented. | +| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | uncovered | `guobiao_placeholder` | Placeholder: dataset scale versus model requirements is not implemented. | +| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | covered | `guobiao` | Checks whether created and updated timestamps are within configured time ranges | +| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | partial | `guobiao` | Uses image annotation checks as partial evidence of annotation accuracy. | +| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | uncovered | `guobiao_placeholder` | Placeholder: before/after model performance comparison is not implemented. | + ### Rule-Based TEXT Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks dataset-documentation completeness across basic information, content features, construction process, and application guidance, together with text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, Rule_TC609_02080101_TextPerplexity, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; calculates model-based text perplexity; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | | `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_TIMELINESS` | Rule_TC609_0303_DataTimeRange | Checks whether `data.dt` falls within the configured `dt_start` and `dt_end` range required by the target application scenario. | High-quality dataset quality evaluation specification (SAC/TC609) | N/A | N/A | -| `QUALITY_BAD_TYPE_CONSISTENCY` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | High-quality dataset classification guide (SAC/TC609) | N/A | N/A | | `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics diff --git a/docs/rules.md b/docs/rules.md index f2b63d9a..ddbdfafa 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -2,13 +2,6 @@ The specific rules for each quality metric are as follows: | Function Name | Type | Description | Reference | |------------------------------|-------------------|---------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Rule_TC609_0101_DocBasicInfoCompleteness | COMPLETENESS | Check whether dataset documentation covers dataset scale, format specification, file structure, access channel, and technical support. | 2025 High-quality dataset quality evaluation specification | -| Rule_TC609_0102_DocContentFeatureCompleteness | COMPLETENESS | Check whether dataset documentation covers modality type, data distribution, label statistics, sample examples, and limitations. | 2025 High-quality dataset quality evaluation specification | -| Rule_TC609_0103_DocConstructionProcessCompleteness | COMPLETENESS | Check whether dataset documentation covers data sources, collection methods, processing pipeline, annotation specification, and version control. | 2025 High-quality dataset quality evaluation specification | -| Rule_TC609_0104_DocApplicationCompleteness | COMPLETENESS | Check whether dataset documentation covers licensing, target scenarios, evaluation methods, benchmark results, and typical cases. | 2025 High-quality dataset quality evaluation specification | -| Rule_TC609_0207_DataTypeConsistency | TYPE_CONSISTENCY | Use a local zero-shot classifier to check whether content belongs to the type declared in `data.type`. | 2025 High-quality dataset classification guide | -| Rule_TC609_02080101_TextPerplexity | FLUENCY | Calculate text perplexity with a configurable causal language model and flag values above the configured threshold. | 2025 High-quality dataset quality evaluation specification | -| Rule_TC609_0303_DataTimeRange | TIMELINESS | Check whether `data.dt` is within the configured `dt_start` and `dt_end` time range. | 2025 High-quality dataset quality evaluation specification | | RuleAbnormalChar | EFFECTIVENESS | Check whether content contains abnormal characters. | | | RuleAbnormalHtml | EFFECTIVENESS | Check whether content contains abnormal HTML. | | | RuleAbnormalNumber | FLUENCY | Check PDF content for abnormal page or index numbers. | | @@ -93,6 +86,46 @@ The specific rules for each quality metric are as follows: | RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | | RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | | RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | +| Rule_TC609_0101_DocBasicInfoCompleteness | TC609_0101 | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0102_DocContentFeatureCompleteness | TC609_0102 | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0103_DocConstructionProcessCompleteness | TC609_0103 | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0104_DocApplicationCompleteness | TC609_0104 | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0201_FormatCompliance | TC609_0201 | Combines existing NLP, SFT, image, audio, and video format rules. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Combines image-label overlap and visualization checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Uses HHEM consistency checking as partial evidence of authenticity. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0206_ContentConsistency | TC609_0206 | Combines structured-field and image-text consistency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080101_TextPerplexity | TC609_02080101 | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080102_KnowledgeInformationDensity | TC609_02080102 | Combines alphabetic-word, stop-word, and unique-word ratio checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080103_RepeatedContent | TC609_02080103 | Combines document-text and formula repetition checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080104_TextCompleteness | TC609_02080104 | Combines null, short, ellipsis-ending, and terminal-ending checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080105_InformationMissing | TC609_02080105 | Uses content length and sentence/word counts as partial missing-information checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080106_TextPurity | TC609_02080106 | Combines abnormal HTML, character, invisible-content, and watermark checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080107_TextCoherence | TC609_02080107 | Combines punctuation, word-boundary, and line-break fluency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080201_ImageResolution | TC609_02080201 | Uses image aspect-ratio validation as partial resolution coverage. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080202_ImageDuplication | TC609_02080202 | Uses PHash and CNN duplicate-image detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080203_ImageSignalNoiseRatio | TC609_02080203 | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080204_ImageClarity | TC609_02080204 | Combines image validity and NIMA quality as partial clarity coverage. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080301_VideoResolution | TC609_02080301 | Placeholder: video resolution is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080302_VideoDuplication | TC609_02080302 | Placeholder: duplicate-video detection is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080303_VideoFrameRate | TC609_02080303 | Placeholder: video FPS validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080304_VideoDuration | TC609_02080304 | Placeholder: video duration validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080305_VideoClarity | TC609_02080305 | Placeholder: video clarity evaluation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080306_VideoDynamicRange | TC609_02080306 | Placeholder: video dynamic-range evaluation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080401_AudioSignalNoiseRatio | TC609_02080401 | Uses the existing Welch power-spectrum SNR implementation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080402_SignalDistortionRatio | TC609_02080402 | Placeholder: signal distortion ratio is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080403_AudioSampleRate | TC609_02080403 | Placeholder: sample-rate quality validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080404_AudioBitDepth | TC609_02080404 | Placeholder: audio bit-depth validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080405_AudioBitRate | TC609_02080405 | Placeholder: audio bit-rate validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_02080406_AudioDuration | TC609_02080406 | Uses the existing WAV duration implementation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0208_ContentCleanliness | TC609_0208 | Combines available text cleanliness checks; modality coverage is partial. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0301_ContentDiversity | TC609_0301 | Placeholder: target-scenario distribution coverage is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0302_ScaleCompleteness | TC609_0302 | Placeholder: dataset scale versus model requirements is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0303_DataTimeRange | TC609_0303 | Checks whether created and updated timestamps are within configured time ranges | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0304_AnnotationAccuracy | TC609_0304 | Uses image annotation checks as partial evidence of annotation accuracy. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0305_ModelAdaptability | TC609_0305 | Placeholder: before/after model performance comparison is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | | RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | | | RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | | diff --git a/examples/guobiao/doc_completeness.py b/examples/guobiao/doc_completeness.py index 6b9a4dae..bb24f4f5 100644 --- a/examples/guobiao/doc_completeness.py +++ b/examples/guobiao/doc_completeness.py @@ -9,7 +9,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import Rule_TC609_0101_DocBasicInfoCompleteness +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0101_DocBasicInfoCompleteness def main(): diff --git a/examples/guobiao/text_perplexity.py b/examples/guobiao/text_perplexity.py index 40e08d4e..75501930 100644 --- a/examples/guobiao/text_perplexity.py +++ b/examples/guobiao/text_perplexity.py @@ -10,7 +10,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import Rule_TC609_02080101_TextPerplexity +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_02080101_TextPerplexity def main(): diff --git a/examples/guobiao/time_range.py b/examples/guobiao/time_range.py index 04002e56..875a61db 100644 --- a/examples/guobiao/time_range.py +++ b/examples/guobiao/time_range.py @@ -2,7 +2,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import Rule_TC609_0303_DataTimeRange +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0303_DataTimeRange def main(): diff --git a/examples/guobiao/type_consistency.py b/examples/guobiao/type_consistency.py index e5aa78e6..86d5dff3 100644 --- a/examples/guobiao/type_consistency.py +++ b/examples/guobiao/type_consistency.py @@ -9,7 +9,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.rule_guobiao import Rule_TC609_0207_DataTypeConsistency +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0207_DataTypeConsistency def main(): diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 25f008f8..3ecad501 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -4,9 +4,9 @@ from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords -from dingo.model.rule.rule_guobiao import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, +from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity, - _RuleDatasetDocCompletenessBase) + _TC609DatasetDocCompletenessBase) class TestRuleDocFormulaRepeat: @@ -68,7 +68,7 @@ def test_high_perplexity_is_bad(self, monkeypatch): ) assert res.status is True - assert res.label == ["QUALITY_BAD_FLUENCY.Rule_TC609_02080101_TextPerplexity"] + assert res.label == ["QUALITY_BAD_TC609_02080101.Rule_TC609_02080101_TextPerplexity"] assert "125.5000" in res.reason[0] assert "test-model" in res.reason[0] @@ -111,12 +111,12 @@ def fail_if_called(cls, model_name): res = Rule_TC609_02080101_TextPerplexity.eval(Data(data_id="ppl-empty", content=" ")) assert res.status is True - assert res.label == ["QUALITY_BAD_FLUENCY.Rule_TC609_02080101_TextPerplexity"] + assert res.label == ["QUALITY_BAD_TC609_02080101.Rule_TC609_02080101_TextPerplexity"] assert "empty content" in res.reason[0] def test_missing_dependencies_raise_clear_error(self, monkeypatch): monkeypatch.setattr( - "dingo.model.rule.rule_guobiao.importlib.util.find_spec", + "dingo.model.rule.guobiao.rule_tc609_quality.importlib.util.find_spec", lambda package: None if package == "transformers" else object(), ) @@ -164,7 +164,7 @@ def test_content_not_matching_declared_type_is_bad(self, monkeypatch): assert result.status is True assert result.score == 0.25 assert result.label == [ - "QUALITY_BAD_TYPE_CONSISTENCY.Rule_TC609_0207_DataTypeConsistency" + "QUALITY_BAD_TC609_0207.Rule_TC609_0207_DataTypeConsistency" ] def test_missing_type_is_bad(self): @@ -213,7 +213,7 @@ def test_created_time_out_of_range_is_bad(self, monkeypatch): ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] + assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "earlier than allowed start" in result.reason[0] def test_dt_invalid_format_is_bad(self, monkeypatch): @@ -233,7 +233,7 @@ def test_dt_invalid_format_is_bad(self, monkeypatch): ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] + assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "unsupported datetime format" in result.reason[0] def test_missing_time_field_is_bad(self, monkeypatch): @@ -248,7 +248,7 @@ def test_missing_time_field_is_bad(self, monkeypatch): result = Rule_TC609_0303_DataTimeRange.eval(Data(data_id="time-missing")) assert result.status is True - assert result.label == ["QUALITY_BAD_TIMELINESS.Rule_TC609_0303_DataTimeRange"] + assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "dt is missing" in result.reason[0] @@ -463,7 +463,7 @@ def mock_match( return matched, missing monkeypatch.setattr( - _RuleDatasetDocCompletenessBase, + _TC609DatasetDocCompletenessBase, "_match_aspects", classmethod(mock_match), ) @@ -489,7 +489,7 @@ def test_basic_info_completeness_bad(self, monkeypatch): ) assert res.status is True assert res.label == [ - "QUALITY_BAD_COMPLETENESS.Rule_TC609_0101_DocBasicInfoCompleteness" + "QUALITY_BAD_TC609_0101.Rule_TC609_0101_DocBasicInfoCompleteness" ] assert res.score < 0.8 @@ -535,6 +535,6 @@ def test_empty_content_is_bad(self): ) assert res.status is True assert res.label == [ - "QUALITY_BAD_COMPLETENESS.Rule_TC609_0104_DocApplicationCompleteness" + "QUALITY_BAD_TC609_0104.Rule_TC609_0104_DocApplicationCompleteness" ] assert "missing or empty" in res.reason[0] From f9e41fea8e4d524026e7adb9ca8284537ad8d65d Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 14:33:23 +0800 Subject: [PATCH 34/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=E7=9B=AE?= =?UTF-8?q?=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/guobiao/__init__.py | 1 + .../model/rule/guobiao/rule_tc609_quality.py | 1340 +++++++++++++++++ 2 files changed, 1341 insertions(+) create mode 100644 dingo/model/rule/guobiao/__init__.py create mode 100644 dingo/model/rule/guobiao/rule_tc609_quality.py diff --git a/dingo/model/rule/guobiao/__init__.py b/dingo/model/rule/guobiao/__init__.py new file mode 100644 index 00000000..9a8d56ca --- /dev/null +++ b/dingo/model/rule/guobiao/__init__.py @@ -0,0 +1 @@ +"""Rules implementing SAC/TC609 high-quality dataset standards.""" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py new file mode 100644 index 00000000..8c59fd51 --- /dev/null +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -0,0 +1,1340 @@ +import importlib +import importlib.util +import math +import re +from datetime import datetime, timezone + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.base import BaseRule + + +def _tc609_metric_info(code, name, description, coverage): + """Build consistent documentation metadata for TC609 rules.""" + return { + "category": "SAC/TC609 High-quality Dataset Metrics", + "quality_dimension": code, + "metric_name": name, + "description": description, + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": code, + "coverage": coverage, + } + + +class _TC609CompositeBase(BaseRule): + """Base class for a TC609 metric composed from existing Dingo rules.""" + + component_rules = () + composition_mode = "all" + + @classmethod + def _resolve_rule(cls, dotted_path): + module_name, class_name = dotted_path.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + reasons = [] + scores = [] + passed_count = 0 + for dotted_path in cls.component_rules: + rule = cls._resolve_rule(dotted_path) + try: + component_res = rule.eval(input_data) + except (ImportError, ModuleNotFoundError): + raise + except Exception as exc: + reasons.append(f"{rule.__name__}: {type(exc).__name__}: {exc}") + continue + if component_res.score is not None: + scores.append(component_res.score) + if component_res.status: + component_reasons = component_res.reason or ["quality check failed"] + reasons.extend( + f"{rule.__name__}: {reason}" for reason in component_reasons + ) + else: + passed_count += 1 + + if scores: + res.score = sum(scores) / len(scores) + if cls.composition_mode == "any": + res.status = passed_count == 0 + else: + res.status = passed_count != len(cls.component_rules) + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res + + +class _TC609PlaceholderBase(BaseRule): + """Base class for registered TC609 metrics that are not implemented yet.""" + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + raise NotImplementedError( + f"{cls.__name__} is a TC609 placeholder and is not implemented yet" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao"]) +class Rule_TC609_0207_DataTypeConsistency(BaseRule): + """Check whether content belongs to the type declared in ``input_data.type``. + + A local zero-shot classifier evaluates the hypothesis ``这段文本属于{type}类型``. + The declared type may be any non-empty string, such as ``医疗`` or ``金融``. + """ + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "TYPE_CONSISTENCY", + "metric_name": "Rule_TC609_0207_DataTypeConsistency", + "description": ( + "Uses a local zero-shot classifier to check whether content belongs " + "to the type declared in the record" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0207", + "coverage": "partial", + } + + _required_fields = [RequiredField.CONTENT, RequiredField.TYPE] + dynamic_config = EvaluatorRuleArgs( + threshold=0.5, + model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + device=-1, + ) + + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "Rule_TC609_0207_DataTypeConsistency requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_match_score( + cls, content, declared_type, model_name, device + ): + classifier = cls._get_classifier(model_name, device) + result = classifier( + content, + candidate_labels=[declared_type], + hypothesis_template="这段文本属于{}类型。", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != declared_type: + raise RuntimeError("Zero-shot classifier returned an invalid result") + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + declared_type = getattr(input_data, "type", None) + content = getattr(input_data, "content", None) + + if not isinstance(declared_type, str) or not declared_type.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Data type is missing or empty"] + return res + + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Content is missing or empty"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]" + ) + + model_name = cls.dynamic_config.model + device = cls.dynamic_config.device + score = cls._calculate_match_score( + content, declared_type, model_name, device + ) + res.score = score + + if score >= threshold: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Content matches declared type {declared_type} " + f"(score: {score:.4f}, threshold: {threshold:.4f})" + ] + else: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Content does not match declared type {declared_type} " + f"(score: {score:.4f}, threshold: {threshold:.4f})" + ] + return res + + +@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao"]) +class Rule_TC609_0303_DataTimeRange(BaseRule): + """Check whether creation/update time fields are within configured ranges.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "TIMELINESS", + "metric_name": "Rule_TC609_0303_DataTimeRange", + "description": ( + "Checks whether created and updated timestamps are within configured " + "time ranges" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0303", + "coverage": "covered", + } + + _required_fields = [RequiredField.DT] + dynamic_config = EvaluatorRuleArgs( + dt_start=None, + dt_end=None, + ) + + @classmethod + def _parse_datetime(cls, value, field_name): + if isinstance(value, datetime): + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc).replace(tzinfo=None) + + if isinstance(value, str): + text = value.strip() + if not text: + raise ValueError(f"{field_name} is empty") + + iso_text = text.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(iso_text) + if parsed.tzinfo is None: + return parsed + return parsed.astimezone(timezone.utc).replace(tzinfo=None) + except ValueError: + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) from None + + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) + + @classmethod + def _validate_time_range( + cls, + dt_value, + start_value, + end_value, + ): + parsed_dt = cls._parse_datetime(dt_value, "time_value") + parsed_dt_start = ( + cls._parse_datetime(start_value, "start_time") + if start_value is not None + else None + ) + parsed_dt_end = ( + cls._parse_datetime(end_value, "end_time") + if end_value is not None + else None + ) + + if ( + parsed_dt_start is not None + and parsed_dt_end is not None + and parsed_dt_start > parsed_dt_end + ): + raise ValueError("time range is invalid: start is later than end") + + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + if parsed_dt_end is not None and parsed_dt > parsed_dt_end: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + return True, parsed_dt, parsed_dt_start, parsed_dt_end + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + + dt_start = getattr(cls.dynamic_config, "dt_start", None) + dt_end = getattr(cls.dynamic_config, "dt_end", None) + + if dt_start is None and dt_end is None: + raise ValueError( + "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config" + ) + + dt_value = getattr(input_data, "dt", None) + if dt_value is None: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["dt is missing"] + return res + + try: + is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range( + dt_value=dt_value, + start_value=dt_start, + end_value=dt_end, + ) + except ValueError as exc: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [f"dt: {exc}"] + return res + + if not is_valid: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is earlier than " + f"allowed start {parsed_dt_start.isoformat(sep=' ')}" + ] + else: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is later than " + f"allowed end {parsed_dt_end.isoformat(sep=' ')}" + ] + return res + + res.label = [QualityLabel.QUALITY_GOOD] + return res + + +@Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao"]) +class Rule_TC609_02080101_TextPerplexity(BaseRule): + """Check whether text perplexity exceeds the configured threshold.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "FLUENCY", + "metric_name": "Rule_TC609_02080101_TextPerplexity", + "description": ( + "Calculates text perplexity with a causal language model and " + "flags text whose PPL exceeds the configured threshold" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "02080101", + "coverage": "covered", + } + + _required_fields = [RequiredField.CONTENT] + dynamic_config = EvaluatorRuleArgs( + threshold=100.0, + model="uer/gpt2-chinese-cluecorpussmall", + stride=512, + ) + + _model_name = None + _tokenizer = None + _model = None + + @classmethod + def _check_dependencies(cls): + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "Rule_TC609_02080101_TextPerplexity requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + @classmethod + def _get_model_components(cls, model_name): + if ( + cls._model_name == model_name + and cls._tokenizer is not None + and cls._model is not None + ): + return cls._tokenizer, cls._model + + try: + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError as exc: + raise ImportError( + "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + cls._tokenizer = AutoTokenizer.from_pretrained(model_name) + cls._model = AutoModelForCausalLM.from_pretrained(model_name) + cls._model.eval() + cls._model_name = model_name + return cls._tokenizer, cls._model + + @classmethod + def _calculate_perplexity(cls, content, tokenizer, model, stride): + try: + import torch + except ImportError as exc: + raise ImportError( + "Rule_TC609_02080101_TextPerplexity requires transformers and torch. " + 'Install them with: pip install "dingo-python[hhem]"' + ) from exc + + encodings = tokenizer(content, return_tensors="pt") + input_ids = encodings["input_ids"] + sequence_length = input_ids.size(1) + if sequence_length < 2: + raise ValueError( + "Rule_TC609_02080101_TextPerplexity requires at least two model tokens" + ) + + model_config = getattr(model, "config", None) + max_length = getattr(model_config, "n_positions", None) + if max_length is None: + max_length = getattr(model_config, "max_position_embeddings", 1024) + max_length = int(max_length) + stride = max(1, min(int(stride), max_length)) + + try: + device = next(model.parameters()).device + except StopIteration: + device = torch.device("cpu") + + total_negative_log_likelihood = 0.0 + total_loss_tokens = 0 + previous_end = 0 + + for begin in range(0, sequence_length, stride): + end = min(begin + max_length, sequence_length) + target_length = end - previous_end + window = input_ids[:, begin:end].to(device) + targets = window.clone() + targets[:, :-target_length] = -100 + + with torch.no_grad(): + output = model(window, labels=targets) + + loss_tokens = int((targets[:, 1:] != -100).sum().item()) + if loss_tokens > 0: + total_negative_log_likelihood += output.loss.item() * loss_tokens + total_loss_tokens += loss_tokens + + previous_end = end + if end == sequence_length: + break + + if total_loss_tokens == 0: + raise ValueError( + "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input" + ) + + mean_loss = total_negative_log_likelihood / total_loss_tokens + try: + return math.exp(mean_loss) + except OverflowError: + return float("inf") + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + cls._check_dependencies() + + res = EvalDetail(metric=cls.__name__) + content = input_data.content + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Text perplexity cannot be calculated for empty content"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or threshold <= 0: + raise ValueError( + "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0" + ) + + model_name = getattr( + cls.dynamic_config, + "model", + "uer/gpt2-chinese-cluecorpussmall", + ) + stride = getattr(cls.dynamic_config, "stride", 512) + tokenizer, model = cls._get_model_components(model_name) + perplexity = cls._calculate_perplexity( + content, + tokenizer, + model, + stride, + ) + + if perplexity > threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Text perplexity {perplexity:.4f} exceeds threshold " + f"{threshold:.4f} (model: {model_name})" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Text perplexity: {perplexity:.4f} " + f"(threshold: {threshold:.4f}, model: {model_name})" + ] + return res + + +class _TC609DatasetDocCompletenessBase(BaseRule): + """Shared logic for dataset documentation completeness checks.""" + + _required_fields = [RequiredField.CONTENT] + _default_threshold = 0.8 + _default_semantic_threshold = 0.5 + _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli" + _default_device = -1 + _default_dimension_name = "Dataset documentation" + _default_aspect_keywords = {} + dynamic_config = EvaluatorRuleArgs( + threshold=_default_threshold, + semantic_threshold=_default_semantic_threshold, + model=_default_model, + device=_default_device, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + f"{cls.__name__} requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_aspect_score(cls, content, aspect_text, model_name, device): + classifier = cls._get_classifier(model_name, device) + result = classifier( + content, + candidate_labels=[aspect_text], + hypothesis_template="这段文本包含{}相关说明。", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != aspect_text: + raise RuntimeError("Zero-shot classifier returned an invalid result") + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def _match_aspects( + cls, + content, + normalized_content, + aspect_keywords, + model_name, + device, + semantic_threshold, + ): + matched = {} + missing = [] + for aspect_name, keywords in aspect_keywords.items(): + aspect_text = ( + f"{aspect_name}（关键词示例：{'、'.join(keywords)}）" + if keywords + else aspect_name + ) + score = cls._calculate_aspect_score( + content, aspect_text, model_name, device + ) + evidence_keyword = next( + ( + keyword + for keyword in keywords + if keyword.lower() in normalized_content + ), + None, + ) + if score >= semantic_threshold: + matched[aspect_name] = { + "score": round(score, 4), + "keyword": evidence_keyword, + } + else: + missing.append(aspect_name) + return matched, missing + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + content = getattr(input_data, "content", None) + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Content is missing or empty"] + res.score = 0.0 + return res + + threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold) + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" + ) + + semantic_threshold = getattr( + cls.dynamic_config, + "semantic_threshold", + cls._default_semantic_threshold, + ) + if semantic_threshold is None or not 0 < semantic_threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]" + ) + + aspect_keywords = getattr( + cls.dynamic_config, + "aspect_keywords", + cls._default_aspect_keywords, + ) + if not isinstance(aspect_keywords, dict) or not aspect_keywords: + raise ValueError( + f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" + ) + + model_name = getattr(cls.dynamic_config, "model", cls._default_model) + if not isinstance(model_name, str) or not model_name.strip(): + raise ValueError( + f"{cls.__name__} dynamic_config.model must be a non-empty string" + ) + + device = getattr(cls.dynamic_config, "device", cls._default_device) + + dimension_name = getattr( + cls.dynamic_config, "dimension_name", cls._default_dimension_name + ) + + normalized_content = re.sub(r"\s+", "", content).lower() + matched, missing = cls._match_aspects( + content=content, + normalized_content=normalized_content, + aspect_keywords=aspect_keywords, + model_name=model_name, + device=device, + semantic_threshold=semantic_threshold, + ) + + total = len(aspect_keywords) + matched_count = len(matched) + score = matched_count / total if total else 0.0 + res.score = round(score, 4) + matched_desc = ( + ", ".join( + ( + f"{aspect}(semantic_score={detail['score']:.4f}, " + f"keyword_hit={detail['keyword'] or 'None'})" + ) + for aspect, detail in matched.items() + ) + if matched + else "None" + ) + missing_desc = ", ".join(missing) if missing else "None" + + if score < threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} is below " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} meets " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + return res + + +@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) +class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase): + """0101: Basic information completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness", + "description": ( + "Checks whether dataset documentation covers basic information " + "aspects such as scale, format, structure, access, and support" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0101", + "coverage": "covered", + } + _default_dimension_name = "Basic information" + _default_aspect_keywords = { + "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"], + "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"], + "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"], + "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"], + "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao"]) +class Rule_TC609_0102_DocContentFeatureCompleteness(_TC609DatasetDocCompletenessBase): + """0102: Content feature completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness", + "description": ( + "Checks whether dataset documentation covers content-feature aspects " + "such as modality, distribution, labels, examples, and limitations" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0102", + "coverage": "covered", + } + _default_dimension_name = "Content feature" + _default_aspect_keywords = { + "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"], + "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"], + "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"], + "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"], + "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao"]) +class Rule_TC609_0103_DocConstructionProcessCompleteness(_TC609DatasetDocCompletenessBase): + """0103: Construction-process completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness", + "description": ( + "Checks whether dataset documentation covers construction-process " + "aspects such as data source, collection, processing, annotation, " + "and version control" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0103", + "coverage": "covered", + } + _default_dimension_name = "Construction process" + _default_aspect_keywords = { + "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"], + "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"], + "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"], + "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"], + "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao"]) +class Rule_TC609_0104_DocApplicationCompleteness(_TC609DatasetDocCompletenessBase): + """0104: Application-description completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0104_DocApplicationCompleteness", + "description": ( + "Checks whether dataset documentation covers application aspects " + "such as license, scenarios, evaluation method, benchmark, and cases" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0104", + "coverage": "covered", + } + _default_dimension_name = "Application description" + _default_aspect_keywords = { + "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"], + "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"], + "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"], + "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"], + "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +# Table 2 - Data quality metrics + + +@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao"]) +class Rule_TC609_0201_FormatCompliance(_TC609CompositeBase): + """0201: Format compliance, partially covered by existing format rules.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleNlpDataFormat", + "dingo.model.rule.rule_common.RuleSftDataFormat", + "dingo.model.rule.rule_common.RuleImageDataFormat", + "dingo.model.rule.rule_common.RuleAudioDataFormat", + "dingo.model.rule.rule_common.RuleVedioDataFormat", + ) + composition_mode = "any" + _metric_info = _tc609_metric_info( + "0201", + "Rule_TC609_0201_FormatCompliance", + "Combines existing NLP, SFT, image, audio, and video format rules.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao"]) +class Rule_TC609_0202_SafetyCompliance(_TC609CompositeBase): + """0202: Safety compliance, composed from safety and PII rules.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleUnsafeWords", + "dingo.model.rule.rule_common.RulePIIDetection", + "dingo.model.rule.rule_common.RuleIDCard", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0202", + "Rule_TC609_0202_SafetyCompliance", + "Combines unsafe-word, PII, and identity-card detection.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao"]) +class Rule_TC609_0203_AnnotationCompliance(_TC609CompositeBase): + """0203: Annotation compliance, partially covered by image label rules.""" + + component_rules = ( + "dingo.model.rule.rule_image.RuleImageLabelOverlap", + "dingo.model.rule.rule_image.RuleImageLabelVisualization", + ) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "0203", + "Rule_TC609_0203_AnnotationCompliance", + "Combines image-label overlap and visualization checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao"]) +class Rule_TC609_0204_StructuralCompleteness(_TC609CompositeBase): + """0204: Structural completeness, composed from content checks.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleContentShort", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0204", + "Rule_TC609_0204_StructuralCompleteness", + "Combines null-content and short-content checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao"]) +class Rule_TC609_0205_ContentAuthenticity(_TC609CompositeBase): + """0205: Content authenticity, partially covered by HHEM.""" + + component_rules = ( + "dingo.model.rule.rule_hallucination_hhem.RuleHallucinationHHEM", + ) + _metric_info = _tc609_metric_info( + "0205", + "Rule_TC609_0205_ContentAuthenticity", + "Uses HHEM consistency checking as partial evidence of authenticity.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao"]) +class Rule_TC609_0206_ContentConsistency(_TC609CompositeBase): + """0206: Content consistency, composed from dict and image-text checks.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleDictConsistency", + "dingo.model.rule.rule_image.RuleImageTextSimilarity", + ) + composition_mode = "any" + _metric_info = _tc609_metric_info( + "0206", + "Rule_TC609_0206_ContentConsistency", + "Combines structured-field and image-text consistency checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao"]) +class Rule_TC609_0208_ContentCleanliness(_TC609CompositeBase): + """0208: Content cleanliness, composed from available cleaning rules.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleAbnormalChar", + "dingo.model.rule.rule_common.RuleAbnormalHtml", + "dingo.model.rule.rule_common.RuleDocRepeat", + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleWatermark", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0208", + "Rule_TC609_0208_ContentCleanliness", + "Combines available text cleanliness checks; modality coverage is partial.", + "partial", + ) + + +# Table 3 - Model application metrics + + +@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_placeholder"]) +class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): + """0301: Placeholder for content diversity.""" + + _metric_info = _tc609_metric_info( + "0301", + "Rule_TC609_0301_ContentDiversity", + "Placeholder: target-scenario distribution coverage is not implemented.", + "uncovered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_placeholder"]) +class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): + """0302: Placeholder for scale completeness.""" + + _metric_info = _tc609_metric_info( + "0302", + "Rule_TC609_0302_ScaleCompleteness", + "Placeholder: dataset scale versus model requirements is not implemented.", + "uncovered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao"]) +class Rule_TC609_0304_AnnotationAccuracy(_TC609CompositeBase): + """0304: Annotation accuracy, partially covered by label checks.""" + + component_rules = ( + "dingo.model.rule.rule_image.RuleImageLabelOverlap", + "dingo.model.rule.rule_image.RuleImageLabelVisualization", + ) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "0304", + "Rule_TC609_0304_AnnotationAccuracy", + "Uses image annotation checks as partial evidence of annotation accuracy.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_placeholder"]) +class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase): + """0305: Placeholder for model adaptability.""" + + _metric_info = _tc609_metric_info( + "0305", + "Rule_TC609_0305_ModelAdaptability", + "Placeholder: before/after model performance comparison is not implemented.", + "uncovered", + ) + + +# Appendix A.1 - Text content cleanliness metrics + + +@Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao"]) +class Rule_TC609_02080102_KnowledgeInformationDensity(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleAlphaWords", + "dingo.model.rule.rule_common.RuleStopWord", + "dingo.model.rule.rule_common.RuleUniqueWords", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080102", + "Rule_TC609_02080102_KnowledgeInformationDensity", + "Combines alphabetic-word, stop-word, and unique-word ratio checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao"]) +class Rule_TC609_02080103_RepeatedContent(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleDocRepeat", + "dingo.model.rule.rule_common.RuleDocFormulaRepeat", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080103", + "Rule_TC609_02080103_RepeatedContent", + "Combines document-text and formula repetition checks.", + "covered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao"]) +class Rule_TC609_02080104_TextCompleteness(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleContentShort", + "dingo.model.rule.rule_common.RuleLineEndWithEllipsis", + "dingo.model.rule.rule_common.RuleLineEndWithTerminal", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080104", + "Rule_TC609_02080104_TextCompleteness", + "Combines null, short, ellipsis-ending, and terminal-ending checks.", + "covered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao"]) +class Rule_TC609_02080105_InformationMissing(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleContentShort", + "dingo.model.rule.rule_common.RuleSentenceNumber", + "dingo.model.rule.rule_common.RuleWordNumber", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080105", + "Rule_TC609_02080105_InformationMissing", + "Uses content length and sentence/word counts as partial missing-information checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao"]) +class Rule_TC609_02080106_TextPurity(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleAbnormalChar", + "dingo.model.rule.rule_common.RuleAbnormalHtml", + "dingo.model.rule.rule_common.RuleInvisibleChar", + "dingo.model.rule.rule_common.RuleSpecialCharacter", + "dingo.model.rule.rule_common.RuleWatermark", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080106", + "Rule_TC609_02080106_TextPurity", + "Combines abnormal HTML, character, invisible-content, and watermark checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao"]) +class Rule_TC609_02080107_TextCoherence(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_common.RuleNoPunc", + "dingo.model.rule.rule_common.RuleWordSplit", + "dingo.model.rule.rule_common.RuleWordStuck", + "dingo.model.rule.rule_common.RuleEnterAndSpace", + "dingo.model.rule.rule_common.RuleEnterMore", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080107", + "Rule_TC609_02080107_TextCoherence", + "Combines punctuation, word-boundary, and line-break fluency checks.", + "partial", + ) + + +# Appendix A.2 - Image content cleanliness metrics + + +@Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao"]) +class Rule_TC609_02080201_ImageResolution(_TC609CompositeBase): + component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "02080201", + "Rule_TC609_02080201_ImageResolution", + "Uses image aspect-ratio validation as partial resolution coverage.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao"]) +class Rule_TC609_02080202_ImageDuplication(_TC609CompositeBase): + component_rules = ("dingo.model.rule.rule_image.RuleImageRepeat",) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080202", + "Rule_TC609_02080202_ImageDuplication", + "Uses PHash and CNN duplicate-image detection.", + "covered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao"]) +class Rule_TC609_02080203_ImageSignalNoiseRatio(_TC609CompositeBase): + component_rules = ("dingo.model.rule.rule_image.RuleImageQuality",) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "02080203", + "Rule_TC609_02080203_ImageSignalNoiseRatio", + "Uses NIMA image quality as partial evidence; it is not a true SNR metric.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao"]) +class Rule_TC609_02080204_ImageClarity(_TC609CompositeBase): + component_rules = ( + "dingo.model.rule.rule_image.RuleImageValid", + "dingo.model.rule.rule_image.RuleImageQuality", + ) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "02080204", + "Rule_TC609_02080204_ImageClarity", + "Combines image validity and NIMA quality as partial clarity coverage.", + "partial", + ) + + +# Appendix A.3 - Video content cleanliness placeholders + + +@Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_placeholder"]) +class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080301", "Rule_TC609_02080301_VideoResolution", + "Placeholder: video resolution is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_placeholder"]) +class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080302", "Rule_TC609_02080302_VideoDuplication", + "Placeholder: duplicate-video detection is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_placeholder"]) +class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080303", "Rule_TC609_02080303_VideoFrameRate", + "Placeholder: video FPS validation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_placeholder"]) +class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080304", "Rule_TC609_02080304_VideoDuration", + "Placeholder: video duration validation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_placeholder"]) +class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080305", "Rule_TC609_02080305_VideoClarity", + "Placeholder: video clarity evaluation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_placeholder"]) +class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080306", "Rule_TC609_02080306_VideoDynamicRange", + "Placeholder: video dynamic-range evaluation is not implemented.", "uncovered" + ) + + +# Appendix A.4 - Audio content cleanliness metrics + + +@Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao"]) +class Rule_TC609_02080401_AudioSignalNoiseRatio(_TC609CompositeBase): + # Existing RuleAudioDuration currently contains the SNR implementation. + component_rules = ("dingo.model.rule.rule_audio.RuleAudioDuration",) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080401", + "Rule_TC609_02080401_AudioSignalNoiseRatio", + "Uses the existing Welch power-spectrum SNR implementation.", + "covered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_placeholder"]) +class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080402", "Rule_TC609_02080402_SignalDistortionRatio", + "Placeholder: signal distortion ratio is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_placeholder"]) +class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080403", "Rule_TC609_02080403_AudioSampleRate", + "Placeholder: sample-rate quality validation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_placeholder"]) +class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080404", "Rule_TC609_02080404_AudioBitDepth", + "Placeholder: audio bit-depth validation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_placeholder"]) +class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): + _metric_info = _tc609_metric_info( + "02080405", "Rule_TC609_02080405_AudioBitRate", + "Placeholder: audio bit-rate validation is not implemented.", "uncovered" + ) + + +@Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao"]) +class Rule_TC609_02080406_AudioDuration(_TC609CompositeBase): + # Existing RuleAudioSnrQuality currently contains the duration implementation. + component_rules = ("dingo.model.rule.rule_audio.RuleAudioSnrQuality",) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "02080406", + "Rule_TC609_02080406_AudioDuration", + "Uses the existing WAV duration implementation.", + "covered", + ) From d825e69175c9ba2d70a62dde20e4e5c75d5cec2c Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 14:34:33 +0800 Subject: [PATCH 35/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/test_rule_tc609_quality.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 test/scripts/model/rule/test_rule_tc609_quality.py diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py new file mode 100644 index 00000000..af4997d3 --- /dev/null +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -0,0 +1,88 @@ +import inspect + +import pytest + +from dingo.io import Data +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.model import Model +from dingo.model.rule.guobiao import rule_tc609_quality +from dingo.model.rule.guobiao.rule_tc609_quality import ( + Rule_TC609_0202_SafetyCompliance, + Rule_TC609_0301_ContentDiversity, +) + + +def test_tc609_quality_defines_all_standard_metrics(): + rule_classes = { + name: cls + for name, cls in inspect.getmembers( + rule_tc609_quality, + lambda value: inspect.isclass(value) + and value.__module__ == rule_tc609_quality.__name__, + ) + if name.startswith("Rule_TC609_") + } + + assert len(rule_classes) == 40 + assert all(name in Model.rule_name_map for name in rule_classes) + + expected_primary_codes = { + "0101", "0102", "0103", "0104", + "0201", "0202", "0203", "0204", "0205", "0206", "0207", "0208", + "0301", "0302", "0303", "0304", "0305", + } + actual_codes = { + name.split("_")[2] + for name in rule_classes + if len(name.split("_")[2]) == 4 + } + assert actual_codes == expected_primary_codes + + +def test_composite_rule_maps_component_failure_to_tc609_label(monkeypatch): + class PassingRule: + @classmethod + def eval(cls, input_data): + return EvalDetail( + metric=cls.__name__, + label=[QualityLabel.QUALITY_GOOD], + ) + + class FailingRule: + @classmethod + def eval(cls, input_data): + return EvalDetail( + metric=cls.__name__, + status=True, + label=["QUALITY_BAD_TEST.FailingRule"], + reason=["component failed"], + ) + + component_map = { + Rule_TC609_0202_SafetyCompliance.component_rules[0]: PassingRule, + Rule_TC609_0202_SafetyCompliance.component_rules[1]: FailingRule, + Rule_TC609_0202_SafetyCompliance.component_rules[2]: PassingRule, + } + monkeypatch.setattr( + Rule_TC609_0202_SafetyCompliance, + "_resolve_rule", + classmethod(lambda cls, path: component_map[path]), + ) + + result = Rule_TC609_0202_SafetyCompliance.eval( + Data(data_id="safety", content="test") + ) + + assert result.status is True + assert result.label == [ + "QUALITY_BAD_TC609_0202.Rule_TC609_0202_SafetyCompliance" + ] + assert result.reason == ["FailingRule: component failed"] + + +def test_uncovered_rule_is_explicit_placeholder(): + assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_placeholder"] + with pytest.raises(NotImplementedError, match="placeholder"): + Rule_TC609_0301_ContentDiversity.eval( + Data(data_id="diversity", content="test") + ) From 29ca958405a3449b3b2e3e2895e708f6804fc922 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 14:52:55 +0800 Subject: [PATCH 36/80] =?UTF-8?q?feat:=20=E9=87=8D=E6=9E=84=EF=BC=8C?= =?UTF-8?q?=E6=8A=BD=E5=87=BA=E6=9D=A5=E5=9F=BA=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 293 +---------------- .../rule/guobiao/rule_tc609_quality_base.py | 294 ++++++++++++++++++ test/scripts/model/rule/test_rule_common.py | 4 +- 3 files changed, 302 insertions(+), 289 deletions(-) create mode 100644 dingo/model/rule/guobiao/rule_tc609_quality_base.py diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 8c59fd51..cf085448 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -1,93 +1,20 @@ -import importlib import importlib.util import math -import re from datetime import datetime, timezone from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model +from dingo.model.rule.guobiao.rule_tc609_quality_base import ( + _tc609_metric_info, + _TC609CompositeBase, + _TC609DatasetDocCompletenessBase, + _TC609PlaceholderBase, +) from dingo.model.rule.base import BaseRule -def _tc609_metric_info(code, name, description, coverage): - """Build consistent documentation metadata for TC609 rules.""" - return { - "category": "SAC/TC609 High-quality Dataset Metrics", - "quality_dimension": code, - "metric_name": name, - "description": description, - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": code, - "coverage": coverage, - } - - -class _TC609CompositeBase(BaseRule): - """Base class for a TC609 metric composed from existing Dingo rules.""" - - component_rules = () - composition_mode = "all" - - @classmethod - def _resolve_rule(cls, dotted_path): - module_name, class_name = dotted_path.rsplit(".", 1) - module = importlib.import_module(module_name) - return getattr(module, class_name) - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - reasons = [] - scores = [] - passed_count = 0 - for dotted_path in cls.component_rules: - rule = cls._resolve_rule(dotted_path) - try: - component_res = rule.eval(input_data) - except (ImportError, ModuleNotFoundError): - raise - except Exception as exc: - reasons.append(f"{rule.__name__}: {type(exc).__name__}: {exc}") - continue - if component_res.score is not None: - scores.append(component_res.score) - if component_res.status: - component_reasons = component_res.reason or ["quality check failed"] - reasons.extend( - f"{rule.__name__}: {reason}" for reason in component_reasons - ) - else: - passed_count += 1 - - if scores: - res.score = sum(scores) / len(scores) - if cls.composition_mode == "any": - res.status = passed_count == 0 - else: - res.status = passed_count != len(cls.component_rules) - if res.status: - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = reasons - else: - res.label = [QualityLabel.QUALITY_GOOD] - return res - - -class _TC609PlaceholderBase(BaseRule): - """Base class for registered TC609 metrics that are not implemented yet.""" - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - raise NotImplementedError( - f"{cls.__name__} is a TC609 placeholder and is not implemented yet" - ) - - @Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao"]) class Rule_TC609_0207_DataTypeConsistency(BaseRule): """Check whether content belongs to the type declared in ``input_data.type``. @@ -542,214 +469,6 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -class _TC609DatasetDocCompletenessBase(BaseRule): - """Shared logic for dataset documentation completeness checks.""" - - _required_fields = [RequiredField.CONTENT] - _default_threshold = 0.8 - _default_semantic_threshold = 0.5 - _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli" - _default_device = -1 - _default_dimension_name = "Dataset documentation" - _default_aspect_keywords = {} - dynamic_config = EvaluatorRuleArgs( - threshold=_default_threshold, - semantic_threshold=_default_semantic_threshold, - model=_default_model, - device=_default_device, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - _model_name = None - _model_device = None - _classifier = None - - @classmethod - def _get_classifier(cls, model_name, device): - if ( - cls._model_name == model_name - and cls._model_device == device - and cls._classifier is not None - ): - return cls._classifier - - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - f"{cls.__name__} requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - from transformers import pipeline - - cls._classifier = pipeline( - "zero-shot-classification", - model=model_name, - device=device, - ) - cls._model_name = model_name - cls._model_device = device - return cls._classifier - - @classmethod - def _calculate_aspect_score(cls, content, aspect_text, model_name, device): - classifier = cls._get_classifier(model_name, device) - result = classifier( - content, - candidate_labels=[aspect_text], - hypothesis_template="这段文本包含{}相关说明。", - multi_label=True, - truncation=True, - ) - labels = result.get("labels", []) - scores = result.get("scores", []) - if not labels or not scores or labels[0] != aspect_text: - raise RuntimeError("Zero-shot classifier returned an invalid result") - score = float(scores[0]) - if not math.isfinite(score) or not 0.0 <= score <= 1.0: - raise RuntimeError( - f"Zero-shot classifier returned an invalid score: {score}" - ) - return score - - @classmethod - def _match_aspects( - cls, - content, - normalized_content, - aspect_keywords, - model_name, - device, - semantic_threshold, - ): - matched = {} - missing = [] - for aspect_name, keywords in aspect_keywords.items(): - aspect_text = ( - f"{aspect_name}（关键词示例：{'、'.join(keywords)}）" - if keywords - else aspect_name - ) - score = cls._calculate_aspect_score( - content, aspect_text, model_name, device - ) - evidence_keyword = next( - ( - keyword - for keyword in keywords - if keyword.lower() in normalized_content - ), - None, - ) - if score >= semantic_threshold: - matched[aspect_name] = { - "score": round(score, 4), - "keyword": evidence_keyword, - } - else: - missing.append(aspect_name) - return matched, missing - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - content = getattr(input_data, "content", None) - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Content is missing or empty"] - res.score = 0.0 - return res - - threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold) - if threshold is None or not 0 < threshold <= 1: - raise ValueError( - f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" - ) - - semantic_threshold = getattr( - cls.dynamic_config, - "semantic_threshold", - cls._default_semantic_threshold, - ) - if semantic_threshold is None or not 0 < semantic_threshold <= 1: - raise ValueError( - f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]" - ) - - aspect_keywords = getattr( - cls.dynamic_config, - "aspect_keywords", - cls._default_aspect_keywords, - ) - if not isinstance(aspect_keywords, dict) or not aspect_keywords: - raise ValueError( - f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" - ) - - model_name = getattr(cls.dynamic_config, "model", cls._default_model) - if not isinstance(model_name, str) or not model_name.strip(): - raise ValueError( - f"{cls.__name__} dynamic_config.model must be a non-empty string" - ) - - device = getattr(cls.dynamic_config, "device", cls._default_device) - - dimension_name = getattr( - cls.dynamic_config, "dimension_name", cls._default_dimension_name - ) - - normalized_content = re.sub(r"\s+", "", content).lower() - matched, missing = cls._match_aspects( - content=content, - normalized_content=normalized_content, - aspect_keywords=aspect_keywords, - model_name=model_name, - device=device, - semantic_threshold=semantic_threshold, - ) - - total = len(aspect_keywords) - matched_count = len(matched) - score = matched_count / total if total else 0.0 - res.score = round(score, 4) - matched_desc = ( - ", ".join( - ( - f"{aspect}(semantic_score={detail['score']:.4f}, " - f"keyword_hit={detail['keyword'] or 'None'})" - ) - for aspect, detail in matched.items() - ) - if matched - else "None" - ) - missing_desc = ", ".join(missing) if missing else "None" - - if score < threshold: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"{dimension_name} completeness score {score:.4f} is below " - f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " - f"matched: {matched_desc}; missing: {missing_desc}" - ] - else: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"{dimension_name} completeness score {score:.4f} meets " - f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " - f"matched: {matched_desc}; missing: {missing_desc}" - ] - return res - - @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase): """0101: Basic information completeness in dataset documentation.""" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py new file mode 100644 index 00000000..ee79f839 --- /dev/null +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -0,0 +1,294 @@ +import importlib +import importlib.util +import math +import re + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io.input import Data, RequiredField +from dingo.io.output.eval_detail import EvalDetail, QualityLabel +from dingo.model.rule.base import BaseRule + + +def _tc609_metric_info(code, name, description, coverage): + """Build consistent documentation metadata for TC609 rules.""" + return { + "category": "SAC/TC609 High-quality Dataset Metrics", + "quality_dimension": code, + "metric_name": name, + "description": description, + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": code, + "coverage": coverage, + } + + +class _TC609CompositeBase(BaseRule): + """Base class for a TC609 metric composed from existing Dingo rules.""" + + component_rules = () + composition_mode = "all" + + @classmethod + def _resolve_rule(cls, dotted_path): + module_name, class_name = dotted_path.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, class_name) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + reasons = [] + scores = [] + passed_count = 0 + for dotted_path in cls.component_rules: + rule = cls._resolve_rule(dotted_path) + try: + component_res = rule.eval(input_data) + except (ImportError, ModuleNotFoundError): + raise + except Exception as exc: + reasons.append(f"{rule.__name__}: {type(exc).__name__}: {exc}") + continue + if component_res.score is not None: + scores.append(component_res.score) + if component_res.status: + component_reasons = component_res.reason or ["quality check failed"] + reasons.extend( + f"{rule.__name__}: {reason}" for reason in component_reasons + ) + else: + passed_count += 1 + + if scores: + res.score = sum(scores) / len(scores) + if cls.composition_mode == "any": + res.status = passed_count == 0 + else: + res.status = passed_count != len(cls.component_rules) + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res + + +class _TC609PlaceholderBase(BaseRule): + """Base class for registered TC609 metrics that are not implemented yet.""" + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + raise NotImplementedError( + f"{cls.__name__} is a TC609 placeholder and is not implemented yet" + ) + + +class _TC609DatasetDocCompletenessBase(BaseRule): + """Shared logic for dataset documentation completeness checks.""" + + _required_fields = [RequiredField.CONTENT] + _default_threshold = 0.8 + _default_semantic_threshold = 0.5 + _default_model = "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli" + _default_device = -1 + _default_dimension_name = "Dataset documentation" + _default_aspect_keywords = {} + dynamic_config = EvaluatorRuleArgs( + threshold=_default_threshold, + semantic_threshold=_default_semantic_threshold, + model=_default_model, + device=_default_device, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + f"{cls.__name__} requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_aspect_score(cls, content, aspect_text, model_name, device): + classifier = cls._get_classifier(model_name, device) + result = classifier( + content, + candidate_labels=[aspect_text], + hypothesis_template="这段文本包含{}相关说明。", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != aspect_text: + raise RuntimeError("Zero-shot classifier returned an invalid result") + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def _match_aspects( + cls, + content, + normalized_content, + aspect_keywords, + model_name, + device, + semantic_threshold, + ): + matched = {} + missing = [] + for aspect_name, keywords in aspect_keywords.items(): + aspect_text = ( + f"{aspect_name}（关键词示例：{'、'.join(keywords)}）" + if keywords + else aspect_name + ) + score = cls._calculate_aspect_score( + content, aspect_text, model_name, device + ) + evidence_keyword = next( + ( + keyword + for keyword in keywords + if keyword.lower() in normalized_content + ), + None, + ) + if score >= semantic_threshold: + matched[aspect_name] = { + "score": round(score, 4), + "keyword": evidence_keyword, + } + else: + missing.append(aspect_name) + return matched, missing + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + content = getattr(input_data, "content", None) + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Content is missing or empty"] + res.score = 0.0 + return res + + threshold = getattr(cls.dynamic_config, "threshold", cls._default_threshold) + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.threshold must be in (0, 1]" + ) + + semantic_threshold = getattr( + cls.dynamic_config, + "semantic_threshold", + cls._default_semantic_threshold, + ) + if semantic_threshold is None or not 0 < semantic_threshold <= 1: + raise ValueError( + f"{cls.__name__} dynamic_config.semantic_threshold must be in (0, 1]" + ) + + aspect_keywords = getattr( + cls.dynamic_config, + "aspect_keywords", + cls._default_aspect_keywords, + ) + if not isinstance(aspect_keywords, dict) or not aspect_keywords: + raise ValueError( + f"{cls.__name__} dynamic_config.aspect_keywords must be a non-empty dict" + ) + + model_name = getattr(cls.dynamic_config, "model", cls._default_model) + if not isinstance(model_name, str) or not model_name.strip(): + raise ValueError( + f"{cls.__name__} dynamic_config.model must be a non-empty string" + ) + + device = getattr(cls.dynamic_config, "device", cls._default_device) + + dimension_name = getattr( + cls.dynamic_config, "dimension_name", cls._default_dimension_name + ) + + normalized_content = re.sub(r"\s+", "", content).lower() + matched, missing = cls._match_aspects( + content=content, + normalized_content=normalized_content, + aspect_keywords=aspect_keywords, + model_name=model_name, + device=device, + semantic_threshold=semantic_threshold, + ) + + total = len(aspect_keywords) + matched_count = len(matched) + score = matched_count / total if total else 0.0 + res.score = round(score, 4) + matched_desc = ( + ", ".join( + ( + f"{aspect}(semantic_score={detail['score']:.4f}, " + f"keyword_hit={detail['keyword'] or 'None'})" + ) + for aspect, detail in matched.items() + ) + if matched + else "None" + ) + missing_desc = ", ".join(missing) if missing else "None" + + if score < threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} is below " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"{dimension_name} completeness score {score:.4f} meets " + f"threshold {threshold:.4f} (A/B={matched_count}/{total}); " + f"matched: {matched_desc}; missing: {missing_desc}" + ] + return res diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 3ecad501..71054730 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -5,8 +5,8 @@ from dingo.io.output.eval_detail import QualityLabel from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, - Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity, - _TC609DatasetDocCompletenessBase) + Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity) +from dingo.model.rule.guobiao.rule_tc609_quality_base import _TC609DatasetDocCompletenessBase class TestRuleDocFormulaRepeat: From 030c77a8ee568fa6d04a24309e28b458ad40fa01 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 14:58:33 +0800 Subject: [PATCH 37/80] =?UTF-8?q?feat:=20=E7=B1=BB=E5=90=8D=E6=8E=92?= =?UTF-8?q?=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 1014 ++++++++--------- 1 file changed, 498 insertions(+), 516 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index cf085448..acec0c03 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -15,6 +15,247 @@ from dingo.model.rule.base import BaseRule +@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) +class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase): + """0101: Basic information completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness", + "description": ( + "Checks whether dataset documentation covers basic information " + "aspects such as scale, format, structure, access, and support" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0101", + "coverage": "covered", + } + _default_dimension_name = "Basic information" + _default_aspect_keywords = { + "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"], + "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"], + "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"], + "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"], + "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao"]) +class Rule_TC609_0102_DocContentFeatureCompleteness(_TC609DatasetDocCompletenessBase): + """0102: Content feature completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness", + "description": ( + "Checks whether dataset documentation covers content-feature aspects " + "such as modality, distribution, labels, examples, and limitations" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0102", + "coverage": "covered", + } + _default_dimension_name = "Content feature" + _default_aspect_keywords = { + "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"], + "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"], + "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"], + "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"], + "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao"]) +class Rule_TC609_0103_DocConstructionProcessCompleteness(_TC609DatasetDocCompletenessBase): + """0103: Construction-process completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness", + "description": ( + "Checks whether dataset documentation covers construction-process " + "aspects such as data source, collection, processing, annotation, " + "and version control" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0103", + "coverage": "covered", + } + _default_dimension_name = "Construction process" + _default_aspect_keywords = { + "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"], + "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"], + "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"], + "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"], + "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao"]) +class Rule_TC609_0104_DocApplicationCompleteness(_TC609DatasetDocCompletenessBase): + """0104: Application-description completeness in dataset documentation.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "COMPLETENESS", + "metric_name": "Rule_TC609_0104_DocApplicationCompleteness", + "description": ( + "Checks whether dataset documentation covers application aspects " + "such as license, scenarios, evaluation method, benchmark, and cases" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0104", + "coverage": "covered", + } + _default_dimension_name = "Application description" + _default_aspect_keywords = { + "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"], + "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"], + "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"], + "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"], + "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"], + } + dynamic_config = EvaluatorRuleArgs( + threshold=0.8, + dimension_name=_default_dimension_name, + aspect_keywords=_default_aspect_keywords, + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao"]) +class Rule_TC609_0201_FormatCompliance(_TC609CompositeBase): + """0201: Format compliance, partially covered by existing format rules.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleNlpDataFormat", + "dingo.model.rule.rule_common.RuleSftDataFormat", + "dingo.model.rule.rule_common.RuleImageDataFormat", + "dingo.model.rule.rule_common.RuleAudioDataFormat", + "dingo.model.rule.rule_common.RuleVedioDataFormat", + ) + composition_mode = "any" + _metric_info = _tc609_metric_info( + "0201", + "Rule_TC609_0201_FormatCompliance", + "Combines existing NLP, SFT, image, audio, and video format rules.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao"]) +class Rule_TC609_0202_SafetyCompliance(_TC609CompositeBase): + """0202: Safety compliance, composed from safety and PII rules.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleUnsafeWords", + "dingo.model.rule.rule_common.RulePIIDetection", + "dingo.model.rule.rule_common.RuleIDCard", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0202", + "Rule_TC609_0202_SafetyCompliance", + "Combines unsafe-word, PII, and identity-card detection.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao"]) +class Rule_TC609_0203_AnnotationCompliance(_TC609CompositeBase): + """0203: Annotation compliance, partially covered by image label rules.""" + + component_rules = ( + "dingo.model.rule.rule_image.RuleImageLabelOverlap", + "dingo.model.rule.rule_image.RuleImageLabelVisualization", + ) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "0203", + "Rule_TC609_0203_AnnotationCompliance", + "Combines image-label overlap and visualization checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao"]) +class Rule_TC609_0204_StructuralCompleteness(_TC609CompositeBase): + """0204: Structural completeness, composed from content checks.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleContentShort", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0204", + "Rule_TC609_0204_StructuralCompleteness", + "Combines null-content and short-content checks.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao"]) +class Rule_TC609_0205_ContentAuthenticity(_TC609CompositeBase): + """0205: Content authenticity, partially covered by HHEM.""" + + component_rules = ( + "dingo.model.rule.rule_hallucination_hhem.RuleHallucinationHHEM", + ) + _metric_info = _tc609_metric_info( + "0205", + "Rule_TC609_0205_ContentAuthenticity", + "Uses HHEM consistency checking as partial evidence of authenticity.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao"]) +class Rule_TC609_0206_ContentConsistency(_TC609CompositeBase): + """0206: Content consistency, composed from dict and image-text checks.""" + + component_rules = ( + "dingo.model.rule.rule_common.RuleDictConsistency", + "dingo.model.rule.rule_image.RuleImageTextSimilarity", + ) + composition_mode = "any" + _metric_info = _tc609_metric_info( + "0206", + "Rule_TC609_0206_ContentConsistency", + "Combines structured-field and image-text consistency checks.", + "partial", + ) + + @Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao"]) class Rule_TC609_0207_DataTypeConsistency(BaseRule): """Check whether content belongs to the type declared in ``input_data.type``. @@ -153,139 +394,24 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao"]) -class Rule_TC609_0303_DataTimeRange(BaseRule): - """Check whether creation/update time fields are within configured ranges.""" +@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao"]) +class Rule_TC609_0208_ContentCleanliness(_TC609CompositeBase): + """0208: Content cleanliness, composed from available cleaning rules.""" - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "TIMELINESS", - "metric_name": "Rule_TC609_0303_DataTimeRange", - "description": ( - "Checks whether created and updated timestamps are within configured " - "time ranges" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0303", - "coverage": "covered", - } - - _required_fields = [RequiredField.DT] - dynamic_config = EvaluatorRuleArgs( - dt_start=None, - dt_end=None, + component_rules = ( + "dingo.model.rule.rule_common.RuleAbnormalChar", + "dingo.model.rule.rule_common.RuleAbnormalHtml", + "dingo.model.rule.rule_common.RuleDocRepeat", + "dingo.model.rule.rule_common.RuleContentNull", + "dingo.model.rule.rule_common.RuleWatermark", + ) + _required_fields = [RequiredField.CONTENT] + _metric_info = _tc609_metric_info( + "0208", + "Rule_TC609_0208_ContentCleanliness", + "Combines available text cleanliness checks; modality coverage is partial.", + "partial", ) - - @classmethod - def _parse_datetime(cls, value, field_name): - if isinstance(value, datetime): - if value.tzinfo is None: - return value - return value.astimezone(timezone.utc).replace(tzinfo=None) - - if isinstance(value, str): - text = value.strip() - if not text: - raise ValueError(f"{field_name} is empty") - - iso_text = text.replace("Z", "+00:00") - try: - parsed = datetime.fromisoformat(iso_text) - if parsed.tzinfo is None: - return parsed - return parsed.astimezone(timezone.utc).replace(tzinfo=None) - except ValueError: - raise ValueError( - f"{field_name} has unsupported datetime format: {value!r}" - ) from None - - raise ValueError( - f"{field_name} has unsupported datetime format: {value!r}" - ) - - @classmethod - def _validate_time_range( - cls, - dt_value, - start_value, - end_value, - ): - parsed_dt = cls._parse_datetime(dt_value, "time_value") - parsed_dt_start = ( - cls._parse_datetime(start_value, "start_time") - if start_value is not None - else None - ) - parsed_dt_end = ( - cls._parse_datetime(end_value, "end_time") - if end_value is not None - else None - ) - - if ( - parsed_dt_start is not None - and parsed_dt_end is not None - and parsed_dt_start > parsed_dt_end - ): - raise ValueError("time range is invalid: start is later than end") - - if parsed_dt_start is not None and parsed_dt < parsed_dt_start: - return False, parsed_dt, parsed_dt_start, parsed_dt_end - if parsed_dt_end is not None and parsed_dt > parsed_dt_end: - return False, parsed_dt, parsed_dt_start, parsed_dt_end - return True, parsed_dt, parsed_dt_start, parsed_dt_end - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - res = EvalDetail(metric=cls.__name__) - - dt_start = getattr(cls.dynamic_config, "dt_start", None) - dt_end = getattr(cls.dynamic_config, "dt_end", None) - - if dt_start is None and dt_end is None: - raise ValueError( - "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config" - ) - - dt_value = getattr(input_data, "dt", None) - if dt_value is None: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["dt is missing"] - return res - - try: - is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range( - dt_value=dt_value, - start_value=dt_start, - end_value=dt_end, - ) - except ValueError as exc: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [f"dt: {exc}"] - return res - - if not is_valid: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - if parsed_dt_start is not None and parsed_dt < parsed_dt_start: - res.reason = [ - f"dt {parsed_dt.isoformat(sep=' ')} is earlier than " - f"allowed start {parsed_dt_start.isoformat(sep=' ')}" - ] - else: - res.reason = [ - f"dt {parsed_dt.isoformat(sep=' ')} is later than " - f"allowed end {parsed_dt_end.isoformat(sep=' ')}" - ] - return res - - res.label = [QualityLabel.QUALITY_GOOD] - return res @Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao"]) @@ -410,386 +536,63 @@ def _calculate_perplexity(cls, content, tokenizer, model, stride): if end == sequence_length: break - if total_loss_tokens == 0: - raise ValueError( - "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input" - ) - - mean_loss = total_negative_log_likelihood / total_loss_tokens - try: - return math.exp(mean_loss) - except OverflowError: - return float("inf") - - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - cls._check_dependencies() - - res = EvalDetail(metric=cls.__name__) - content = input_data.content - if not isinstance(content, str) or not content.strip(): - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Text perplexity cannot be calculated for empty content"] - return res - - threshold = cls.dynamic_config.threshold - if threshold is None or threshold <= 0: - raise ValueError( - "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0" - ) - - model_name = getattr( - cls.dynamic_config, - "model", - "uer/gpt2-chinese-cluecorpussmall", - ) - stride = getattr(cls.dynamic_config, "stride", 512) - tokenizer, model = cls._get_model_components(model_name) - perplexity = cls._calculate_perplexity( - content, - tokenizer, - model, - stride, - ) - - if perplexity > threshold: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"Text perplexity {perplexity:.4f} exceeds threshold " - f"{threshold:.4f} (model: {model_name})" - ] - else: - res.label = [QualityLabel.QUALITY_GOOD] - res.reason = [ - f"Text perplexity: {perplexity:.4f} " - f"(threshold: {threshold:.4f}, model: {model_name})" - ] - return res - - -@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) -class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase): - """0101: Basic information completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0101_DocBasicInfoCompleteness", - "description": ( - "Checks whether dataset documentation covers basic information " - "aspects such as scale, format, structure, access, and support" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0101", - "coverage": "covered", - } - _default_dimension_name = "Basic information" - _default_aspect_keywords = { - "dataset_scale": ["数据集规模", "样本数量", "样本规模", "数据量", "存储体积", "数据体量"], - "format_specification": ["格式规范", "数据格式", "文件格式", "编码格式", "字段格式"], - "file_structure": ["文件结构", "目录结构", "文件组织", "数据组织结构"], - "access_channel": ["访问渠道", "获取方式", "下载方式", "访问方式", "获取渠道"], - "technical_support": ["技术支持", "支持方式", "联系方式", "问题反馈", "维护方式"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao"]) -class Rule_TC609_0102_DocContentFeatureCompleteness(_TC609DatasetDocCompletenessBase): - """0102: Content feature completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0102_DocContentFeatureCompleteness", - "description": ( - "Checks whether dataset documentation covers content-feature aspects " - "such as modality, distribution, labels, examples, and limitations" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0102", - "coverage": "covered", - } - _default_dimension_name = "Content feature" - _default_aspect_keywords = { - "modality_type": ["模态类型", "数据模态", "文本图像", "多模态", "音频视频"], - "data_distribution": ["数据分布", "分布情况", "分布特征", "类别分布", "统计分布"], - "label_statistics": ["标签类别统计", "标签统计", "类别统计", "标签分布"], - "sample_examples": ["样本示例", "样例", "示例数据", "样本展示", "案例样本"], - "limitations": ["局限性说明", "局限性", "限制说明", "不足", "已知问题"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao"]) -class Rule_TC609_0103_DocConstructionProcessCompleteness(_TC609DatasetDocCompletenessBase): - """0103: Construction-process completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0103_DocConstructionProcessCompleteness", - "description": ( - "Checks whether dataset documentation covers construction-process " - "aspects such as data source, collection, processing, annotation, " - "and version control" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0103", - "coverage": "covered", - } - _default_dimension_name = "Construction process" - _default_aspect_keywords = { - "data_source": ["数据来源", "来源说明", "数据源", "来源渠道"], - "collection_method": ["采集方法", "采集方式", "收集方法", "获取流程"], - "processing_pipeline": ["加工处理流程", "处理流程", "清洗流程", "预处理流程"], - "annotation_specification": ["标注规范", "标注标准", "标注规则", "标注说明"], - "version_control": ["版本控制", "版本记录", "变更记录", "版本管理"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao"]) -class Rule_TC609_0104_DocApplicationCompleteness(_TC609DatasetDocCompletenessBase): - """0104: Application-description completeness in dataset documentation.""" - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "COMPLETENESS", - "metric_name": "Rule_TC609_0104_DocApplicationCompleteness", - "description": ( - "Checks whether dataset documentation covers application aspects " - "such as license, scenarios, evaluation method, benchmark, and cases" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0104", - "coverage": "covered", - } - _default_dimension_name = "Application description" - _default_aspect_keywords = { - "license": ["使用许可", "许可协议", "授权协议", "license", "开源协议"], - "target_scenarios": ["目标应用场景", "应用场景", "使用场景", "场景说明"], - "evaluation_method": ["评估方法", "评价方法", "评测方法", "评估方案"], - "benchmark_results": ["基准测试结果", "基准结果", "benchmark", "基线结果"], - "typical_cases": ["典型应用案例", "应用案例", "典型案例", "落地案例"], - } - dynamic_config = EvaluatorRuleArgs( - threshold=0.8, - dimension_name=_default_dimension_name, - aspect_keywords=_default_aspect_keywords, - ) - - -# Table 2 - Data quality metrics - - -@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao"]) -class Rule_TC609_0201_FormatCompliance(_TC609CompositeBase): - """0201: Format compliance, partially covered by existing format rules.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleNlpDataFormat", - "dingo.model.rule.rule_common.RuleSftDataFormat", - "dingo.model.rule.rule_common.RuleImageDataFormat", - "dingo.model.rule.rule_common.RuleAudioDataFormat", - "dingo.model.rule.rule_common.RuleVedioDataFormat", - ) - composition_mode = "any" - _metric_info = _tc609_metric_info( - "0201", - "Rule_TC609_0201_FormatCompliance", - "Combines existing NLP, SFT, image, audio, and video format rules.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao"]) -class Rule_TC609_0202_SafetyCompliance(_TC609CompositeBase): - """0202: Safety compliance, composed from safety and PII rules.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleUnsafeWords", - "dingo.model.rule.rule_common.RulePIIDetection", - "dingo.model.rule.rule_common.RuleIDCard", - ) - _required_fields = [RequiredField.CONTENT] - _metric_info = _tc609_metric_info( - "0202", - "Rule_TC609_0202_SafetyCompliance", - "Combines unsafe-word, PII, and identity-card detection.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao"]) -class Rule_TC609_0203_AnnotationCompliance(_TC609CompositeBase): - """0203: Annotation compliance, partially covered by image label rules.""" - - component_rules = ( - "dingo.model.rule.rule_image.RuleImageLabelOverlap", - "dingo.model.rule.rule_image.RuleImageLabelVisualization", - ) - _required_fields = [RequiredField.IMAGE] - _metric_info = _tc609_metric_info( - "0203", - "Rule_TC609_0203_AnnotationCompliance", - "Combines image-label overlap and visualization checks.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao"]) -class Rule_TC609_0204_StructuralCompleteness(_TC609CompositeBase): - """0204: Structural completeness, composed from content checks.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleContentNull", - "dingo.model.rule.rule_common.RuleContentShort", - ) - _required_fields = [RequiredField.CONTENT] - _metric_info = _tc609_metric_info( - "0204", - "Rule_TC609_0204_StructuralCompleteness", - "Combines null-content and short-content checks.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao"]) -class Rule_TC609_0205_ContentAuthenticity(_TC609CompositeBase): - """0205: Content authenticity, partially covered by HHEM.""" - - component_rules = ( - "dingo.model.rule.rule_hallucination_hhem.RuleHallucinationHHEM", - ) - _metric_info = _tc609_metric_info( - "0205", - "Rule_TC609_0205_ContentAuthenticity", - "Uses HHEM consistency checking as partial evidence of authenticity.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao"]) -class Rule_TC609_0206_ContentConsistency(_TC609CompositeBase): - """0206: Content consistency, composed from dict and image-text checks.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleDictConsistency", - "dingo.model.rule.rule_image.RuleImageTextSimilarity", - ) - composition_mode = "any" - _metric_info = _tc609_metric_info( - "0206", - "Rule_TC609_0206_ContentConsistency", - "Combines structured-field and image-text consistency checks.", - "partial", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao"]) -class Rule_TC609_0208_ContentCleanliness(_TC609CompositeBase): - """0208: Content cleanliness, composed from available cleaning rules.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleAbnormalChar", - "dingo.model.rule.rule_common.RuleAbnormalHtml", - "dingo.model.rule.rule_common.RuleDocRepeat", - "dingo.model.rule.rule_common.RuleContentNull", - "dingo.model.rule.rule_common.RuleWatermark", - ) - _required_fields = [RequiredField.CONTENT] - _metric_info = _tc609_metric_info( - "0208", - "Rule_TC609_0208_ContentCleanliness", - "Combines available text cleanliness checks; modality coverage is partial.", - "partial", - ) - - -# Table 3 - Model application metrics - - -@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_placeholder"]) -class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): - """0301: Placeholder for content diversity.""" - - _metric_info = _tc609_metric_info( - "0301", - "Rule_TC609_0301_ContentDiversity", - "Placeholder: target-scenario distribution coverage is not implemented.", - "uncovered", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_placeholder"]) -class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): - """0302: Placeholder for scale completeness.""" - - _metric_info = _tc609_metric_info( - "0302", - "Rule_TC609_0302_ScaleCompleteness", - "Placeholder: dataset scale versus model requirements is not implemented.", - "uncovered", - ) - - -@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao"]) -class Rule_TC609_0304_AnnotationAccuracy(_TC609CompositeBase): - """0304: Annotation accuracy, partially covered by label checks.""" - - component_rules = ( - "dingo.model.rule.rule_image.RuleImageLabelOverlap", - "dingo.model.rule.rule_image.RuleImageLabelVisualization", - ) - _required_fields = [RequiredField.IMAGE] - _metric_info = _tc609_metric_info( - "0304", - "Rule_TC609_0304_AnnotationAccuracy", - "Uses image annotation checks as partial evidence of annotation accuracy.", - "partial", - ) - + if total_loss_tokens == 0: + raise ValueError( + "Rule_TC609_02080101_TextPerplexity could not calculate loss for the input" + ) -@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_placeholder"]) -class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase): - """0305: Placeholder for model adaptability.""" + mean_loss = total_negative_log_likelihood / total_loss_tokens + try: + return math.exp(mean_loss) + except OverflowError: + return float("inf") - _metric_info = _tc609_metric_info( - "0305", - "Rule_TC609_0305_ModelAdaptability", - "Placeholder: before/after model performance comparison is not implemented.", - "uncovered", - ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + cls._check_dependencies() + + res = EvalDetail(metric=cls.__name__) + content = input_data.content + if not isinstance(content, str) or not content.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["Text perplexity cannot be calculated for empty content"] + return res + + threshold = cls.dynamic_config.threshold + if threshold is None or threshold <= 0: + raise ValueError( + "Rule_TC609_02080101_TextPerplexity dynamic_config.threshold must be greater than 0" + ) + model_name = getattr( + cls.dynamic_config, + "model", + "uer/gpt2-chinese-cluecorpussmall", + ) + stride = getattr(cls.dynamic_config, "stride", 512) + tokenizer, model = cls._get_model_components(model_name) + perplexity = cls._calculate_perplexity( + content, + tokenizer, + model, + stride, + ) -# Appendix A.1 - Text content cleanliness metrics + if perplexity > threshold: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"Text perplexity {perplexity:.4f} exceeds threshold " + f"{threshold:.4f} (model: {model_name})" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Text perplexity: {perplexity:.4f} " + f"(threshold: {threshold:.4f}, model: {model_name})" + ] + return res @Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao"]) @@ -893,9 +696,6 @@ class Rule_TC609_02080107_TextCoherence(_TC609CompositeBase): ) -# Appendix A.2 - Image content cleanliness metrics - - @Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao"]) class Rule_TC609_02080201_ImageResolution(_TC609CompositeBase): component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",) @@ -947,9 +747,6 @@ class Rule_TC609_02080204_ImageClarity(_TC609CompositeBase): ) -# Appendix A.3 - Video content cleanliness placeholders - - @Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_placeholder"]) class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( @@ -998,9 +795,6 @@ class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): ) -# Appendix A.4 - Audio content cleanliness metrics - - @Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao"]) class Rule_TC609_02080401_AudioSignalNoiseRatio(_TC609CompositeBase): # Existing RuleAudioDuration currently contains the SNR implementation. @@ -1057,3 +851,191 @@ class Rule_TC609_02080406_AudioDuration(_TC609CompositeBase): "Uses the existing WAV duration implementation.", "covered", ) + + +@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_placeholder"]) +class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): + """0301: Placeholder for content diversity.""" + + _metric_info = _tc609_metric_info( + "0301", + "Rule_TC609_0301_ContentDiversity", + "Placeholder: target-scenario distribution coverage is not implemented.", + "uncovered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_placeholder"]) +class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): + """0302: Placeholder for scale completeness.""" + + _metric_info = _tc609_metric_info( + "0302", + "Rule_TC609_0302_ScaleCompleteness", + "Placeholder: dataset scale versus model requirements is not implemented.", + "uncovered", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao"]) +class Rule_TC609_0303_DataTimeRange(BaseRule): + """Check whether creation/update time fields are within configured ranges.""" + + _metric_info = { + "category": "National Standard Data Quality Metrics", + "quality_dimension": "TIMELINESS", + "metric_name": "Rule_TC609_0303_DataTimeRange", + "description": ( + "Checks whether created and updated timestamps are within configured " + "time ranges" + ), + "paper_title": "High-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "standard_code": "0303", + "coverage": "covered", + } + + _required_fields = [RequiredField.DT] + dynamic_config = EvaluatorRuleArgs( + dt_start=None, + dt_end=None, + ) + + @classmethod + def _parse_datetime(cls, value, field_name): + if isinstance(value, datetime): + if value.tzinfo is None: + return value + return value.astimezone(timezone.utc).replace(tzinfo=None) + + if isinstance(value, str): + text = value.strip() + if not text: + raise ValueError(f"{field_name} is empty") + + iso_text = text.replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(iso_text) + if parsed.tzinfo is None: + return parsed + return parsed.astimezone(timezone.utc).replace(tzinfo=None) + except ValueError: + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) from None + + raise ValueError( + f"{field_name} has unsupported datetime format: {value!r}" + ) + + @classmethod + def _validate_time_range( + cls, + dt_value, + start_value, + end_value, + ): + parsed_dt = cls._parse_datetime(dt_value, "time_value") + parsed_dt_start = ( + cls._parse_datetime(start_value, "start_time") + if start_value is not None + else None + ) + parsed_dt_end = ( + cls._parse_datetime(end_value, "end_time") + if end_value is not None + else None + ) + + if ( + parsed_dt_start is not None + and parsed_dt_end is not None + and parsed_dt_start > parsed_dt_end + ): + raise ValueError("time range is invalid: start is later than end") + + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + if parsed_dt_end is not None and parsed_dt > parsed_dt_end: + return False, parsed_dt, parsed_dt_start, parsed_dt_end + return True, parsed_dt, parsed_dt_start, parsed_dt_end + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + + dt_start = getattr(cls.dynamic_config, "dt_start", None) + dt_end = getattr(cls.dynamic_config, "dt_end", None) + + if dt_start is None and dt_end is None: + raise ValueError( + "Rule_TC609_0303_DataTimeRange requires at least one configured range boundary in dynamic_config" + ) + + dt_value = getattr(input_data, "dt", None) + if dt_value is None: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["dt is missing"] + return res + + try: + is_valid, parsed_dt, parsed_dt_start, parsed_dt_end = cls._validate_time_range( + dt_value=dt_value, + start_value=dt_start, + end_value=dt_end, + ) + except ValueError as exc: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [f"dt: {exc}"] + return res + + if not is_valid: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + if parsed_dt_start is not None and parsed_dt < parsed_dt_start: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is earlier than " + f"allowed start {parsed_dt_start.isoformat(sep=' ')}" + ] + else: + res.reason = [ + f"dt {parsed_dt.isoformat(sep=' ')} is later than " + f"allowed end {parsed_dt_end.isoformat(sep=' ')}" + ] + return res + + res.label = [QualityLabel.QUALITY_GOOD] + return res + + +@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao"]) +class Rule_TC609_0304_AnnotationAccuracy(_TC609CompositeBase): + """0304: Annotation accuracy, partially covered by label checks.""" + + component_rules = ( + "dingo.model.rule.rule_image.RuleImageLabelOverlap", + "dingo.model.rule.rule_image.RuleImageLabelVisualization", + ) + _required_fields = [RequiredField.IMAGE] + _metric_info = _tc609_metric_info( + "0304", + "Rule_TC609_0304_AnnotationAccuracy", + "Uses image annotation checks as partial evidence of annotation accuracy.", + "partial", + ) + + +@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_placeholder"]) +class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase): + """0305: Placeholder for model adaptability.""" + + _metric_info = _tc609_metric_info( + "0305", + "Rule_TC609_0305_ModelAdaptability", + "Placeholder: before/after model performance comparison is not implemented.", + "uncovered", + ) From b50b6f66bbd6eac7431de480f12f2ea5fe071a92 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 15:17:21 +0800 Subject: [PATCH 38/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=91=BD=E5=90=8DRule?= =?UTF-8?q?=5FTC609=5F01=5FDocCompleteness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/guobiao/rule_tc609_quality.py | 12 +++++++----- dingo/model/rule/guobiao/rule_tc609_quality_base.py | 2 +- test/scripts/model/rule/test_rule_common.py | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index acec0c03..3f09a983 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -9,14 +9,14 @@ from dingo.model.rule.guobiao.rule_tc609_quality_base import ( _tc609_metric_info, _TC609CompositeBase, - _TC609DatasetDocCompletenessBase, + Rule_TC609_01_DocCompleteness, _TC609PlaceholderBase, ) from dingo.model.rule.base import BaseRule @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) -class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase): +class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness): """0101: Basic information completeness in dataset documentation.""" _metric_info = { @@ -50,7 +50,7 @@ class Rule_TC609_0101_DocBasicInfoCompleteness(_TC609DatasetDocCompletenessBase) @Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao"]) -class Rule_TC609_0102_DocContentFeatureCompleteness(_TC609DatasetDocCompletenessBase): +class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompleteness): """0102: Content feature completeness in dataset documentation.""" _metric_info = { @@ -84,7 +84,9 @@ class Rule_TC609_0102_DocContentFeatureCompleteness(_TC609DatasetDocCompleteness @Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao"]) -class Rule_TC609_0103_DocConstructionProcessCompleteness(_TC609DatasetDocCompletenessBase): +class Rule_TC609_0103_DocConstructionProcessCompleteness( + Rule_TC609_01_DocCompleteness +): """0103: Construction-process completeness in dataset documentation.""" _metric_info = { @@ -119,7 +121,7 @@ class Rule_TC609_0103_DocConstructionProcessCompleteness(_TC609DatasetDocComplet @Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao"]) -class Rule_TC609_0104_DocApplicationCompleteness(_TC609DatasetDocCompletenessBase): +class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): """0104: Application-description completeness in dataset documentation.""" _metric_info = { diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index ee79f839..4354b628 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -86,7 +86,7 @@ def eval(cls, input_data: Data) -> EvalDetail: ) -class _TC609DatasetDocCompletenessBase(BaseRule): +class Rule_TC609_01_DocCompleteness(BaseRule): """Shared logic for dataset documentation completeness checks.""" _required_fields = [RequiredField.CONTENT] diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 71054730..ed47cba8 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -6,7 +6,7 @@ from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity) -from dingo.model.rule.guobiao.rule_tc609_quality_base import _TC609DatasetDocCompletenessBase +from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness class TestRuleDocFormulaRepeat: @@ -463,7 +463,7 @@ def mock_match( return matched, missing monkeypatch.setattr( - _TC609DatasetDocCompletenessBase, + Rule_TC609_01_DocCompleteness, "_match_aspects", classmethod(mock_match), ) From dbc4e99243919d9d258aa53f7dd6df9a932acaab Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 15:32:31 +0800 Subject: [PATCH 39/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=91=BD=E5=90=8DRule?= =?UTF-8?q?=5FTC609=5FComposite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 42 +++++++++---------- .../rule/guobiao/rule_tc609_quality_base.py | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 3f09a983..23b5ddcd 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -8,7 +8,7 @@ from dingo.model.model import Model from dingo.model.rule.guobiao.rule_tc609_quality_base import ( _tc609_metric_info, - _TC609CompositeBase, + Rule_TC609_Composite, Rule_TC609_01_DocCompleteness, _TC609PlaceholderBase, ) @@ -155,7 +155,7 @@ class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): @Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao"]) -class Rule_TC609_0201_FormatCompliance(_TC609CompositeBase): +class Rule_TC609_0201_FormatCompliance(Rule_TC609_Composite): """0201: Format compliance, partially covered by existing format rules.""" component_rules = ( @@ -175,7 +175,7 @@ class Rule_TC609_0201_FormatCompliance(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao"]) -class Rule_TC609_0202_SafetyCompliance(_TC609CompositeBase): +class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): """0202: Safety compliance, composed from safety and PII rules.""" component_rules = ( @@ -193,7 +193,7 @@ class Rule_TC609_0202_SafetyCompliance(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao"]) -class Rule_TC609_0203_AnnotationCompliance(_TC609CompositeBase): +class Rule_TC609_0203_AnnotationCompliance(Rule_TC609_Composite): """0203: Annotation compliance, partially covered by image label rules.""" component_rules = ( @@ -210,7 +210,7 @@ class Rule_TC609_0203_AnnotationCompliance(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao"]) -class Rule_TC609_0204_StructuralCompleteness(_TC609CompositeBase): +class Rule_TC609_0204_StructuralCompleteness(Rule_TC609_Composite): """0204: Structural completeness, composed from content checks.""" component_rules = ( @@ -227,7 +227,7 @@ class Rule_TC609_0204_StructuralCompleteness(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao"]) -class Rule_TC609_0205_ContentAuthenticity(_TC609CompositeBase): +class Rule_TC609_0205_ContentAuthenticity(Rule_TC609_Composite): """0205: Content authenticity, partially covered by HHEM.""" component_rules = ( @@ -242,7 +242,7 @@ class Rule_TC609_0205_ContentAuthenticity(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao"]) -class Rule_TC609_0206_ContentConsistency(_TC609CompositeBase): +class Rule_TC609_0206_ContentConsistency(Rule_TC609_Composite): """0206: Content consistency, composed from dict and image-text checks.""" component_rules = ( @@ -397,7 +397,7 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao"]) -class Rule_TC609_0208_ContentCleanliness(_TC609CompositeBase): +class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): """0208: Content cleanliness, composed from available cleaning rules.""" component_rules = ( @@ -598,7 +598,7 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao"]) -class Rule_TC609_02080102_KnowledgeInformationDensity(_TC609CompositeBase): +class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAlphaWords", "dingo.model.rule.rule_common.RuleStopWord", @@ -614,7 +614,7 @@ class Rule_TC609_02080102_KnowledgeInformationDensity(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao"]) -class Rule_TC609_02080103_RepeatedContent(_TC609CompositeBase): +class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleDocRepeat", "dingo.model.rule.rule_common.RuleDocFormulaRepeat", @@ -629,7 +629,7 @@ class Rule_TC609_02080103_RepeatedContent(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao"]) -class Rule_TC609_02080104_TextCompleteness(_TC609CompositeBase): +class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", "dingo.model.rule.rule_common.RuleContentShort", @@ -646,7 +646,7 @@ class Rule_TC609_02080104_TextCompleteness(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao"]) -class Rule_TC609_02080105_InformationMissing(_TC609CompositeBase): +class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", "dingo.model.rule.rule_common.RuleContentShort", @@ -663,7 +663,7 @@ class Rule_TC609_02080105_InformationMissing(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao"]) -class Rule_TC609_02080106_TextPurity(_TC609CompositeBase): +class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAbnormalChar", "dingo.model.rule.rule_common.RuleAbnormalHtml", @@ -681,7 +681,7 @@ class Rule_TC609_02080106_TextPurity(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao"]) -class Rule_TC609_02080107_TextCoherence(_TC609CompositeBase): +class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleNoPunc", "dingo.model.rule.rule_common.RuleWordSplit", @@ -699,7 +699,7 @@ class Rule_TC609_02080107_TextCoherence(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao"]) -class Rule_TC609_02080201_ImageResolution(_TC609CompositeBase): +class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",) _required_fields = [RequiredField.IMAGE] _metric_info = _tc609_metric_info( @@ -711,7 +711,7 @@ class Rule_TC609_02080201_ImageResolution(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao"]) -class Rule_TC609_02080202_ImageDuplication(_TC609CompositeBase): +class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageRepeat",) _required_fields = [RequiredField.CONTENT] _metric_info = _tc609_metric_info( @@ -723,7 +723,7 @@ class Rule_TC609_02080202_ImageDuplication(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao"]) -class Rule_TC609_02080203_ImageSignalNoiseRatio(_TC609CompositeBase): +class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageQuality",) _required_fields = [RequiredField.IMAGE] _metric_info = _tc609_metric_info( @@ -735,7 +735,7 @@ class Rule_TC609_02080203_ImageSignalNoiseRatio(_TC609CompositeBase): @Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao"]) -class Rule_TC609_02080204_ImageClarity(_TC609CompositeBase): +class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_image.RuleImageValid", "dingo.model.rule.rule_image.RuleImageQuality", @@ -798,7 +798,7 @@ class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): @Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao"]) -class Rule_TC609_02080401_AudioSignalNoiseRatio(_TC609CompositeBase): +class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite): # Existing RuleAudioDuration currently contains the SNR implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioDuration",) _required_fields = [RequiredField.CONTENT] @@ -843,7 +843,7 @@ class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): @Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao"]) -class Rule_TC609_02080406_AudioDuration(_TC609CompositeBase): +class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite): # Existing RuleAudioSnrQuality currently contains the duration implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioSnrQuality",) _required_fields = [RequiredField.CONTENT] @@ -1015,7 +1015,7 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao"]) -class Rule_TC609_0304_AnnotationAccuracy(_TC609CompositeBase): +class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite): """0304: Annotation accuracy, partially covered by label checks.""" component_rules = ( diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index 4354b628..306ad4c9 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -25,7 +25,7 @@ def _tc609_metric_info(code, name, description, coverage): } -class _TC609CompositeBase(BaseRule): +class Rule_TC609_Composite(BaseRule): """Base class for a TC609 metric composed from existing Dingo rules.""" component_rules = () From e0988301eba3591101ec447ec3f24cbf268b494c Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 16:53:06 +0800 Subject: [PATCH 40/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87group=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 80 +++++++++---------- docs/metrics.md | 80 +++++++++---------- .../model/rule/test_rule_tc609_quality.py | 27 ++++++- 3 files changed, 106 insertions(+), 81 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 23b5ddcd..07a73444 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -15,7 +15,7 @@ from dingo.model.rule.base import BaseRule -@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness): """0101: Basic information completeness in dataset documentation.""" @@ -49,7 +49,7 @@ class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness): ) -@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao_doc"]) class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompleteness): """0102: Content feature completeness in dataset documentation.""" @@ -83,7 +83,7 @@ class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompletenes ) -@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao_doc"]) class Rule_TC609_0103_DocConstructionProcessCompleteness( Rule_TC609_01_DocCompleteness ): @@ -120,7 +120,7 @@ class Rule_TC609_0103_DocConstructionProcessCompleteness( ) -@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao_doc"]) class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): """0104: Application-description completeness in dataset documentation.""" @@ -154,7 +154,7 @@ class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): ) -@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao_data"]) class Rule_TC609_0201_FormatCompliance(Rule_TC609_Composite): """0201: Format compliance, partially covered by existing format rules.""" @@ -174,7 +174,7 @@ class Rule_TC609_0201_FormatCompliance(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao_data"]) class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): """0202: Safety compliance, composed from safety and PII rules.""" @@ -192,7 +192,7 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"]) class Rule_TC609_0203_AnnotationCompliance(Rule_TC609_Composite): """0203: Annotation compliance, partially covered by image label rules.""" @@ -209,7 +209,7 @@ class Rule_TC609_0203_AnnotationCompliance(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"]) class Rule_TC609_0204_StructuralCompleteness(Rule_TC609_Composite): """0204: Structural completeness, composed from content checks.""" @@ -226,7 +226,7 @@ class Rule_TC609_0204_StructuralCompleteness(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao_data"]) class Rule_TC609_0205_ContentAuthenticity(Rule_TC609_Composite): """0205: Content authenticity, partially covered by HHEM.""" @@ -241,7 +241,7 @@ class Rule_TC609_0205_ContentAuthenticity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"]) class Rule_TC609_0206_ContentConsistency(Rule_TC609_Composite): """0206: Content consistency, composed from dict and image-text checks.""" @@ -258,7 +258,7 @@ class Rule_TC609_0206_ContentConsistency(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao_data"]) class Rule_TC609_0207_DataTypeConsistency(BaseRule): """Check whether content belongs to the type declared in ``input_data.type``. @@ -396,7 +396,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0208", ["guobiao_data"]) class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): """0208: Content cleanliness, composed from available cleaning rules.""" @@ -416,7 +416,7 @@ class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"]) class Rule_TC609_02080101_TextPerplexity(BaseRule): """Check whether text perplexity exceeds the configured threshold.""" @@ -597,7 +597,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao_text"]) class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAlphaWords", @@ -613,7 +613,7 @@ class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao_text"]) class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleDocRepeat", @@ -628,7 +628,7 @@ class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao_text"]) class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", @@ -645,7 +645,7 @@ class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao_text"]) class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", @@ -662,7 +662,7 @@ class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao_text"]) class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAbnormalChar", @@ -680,7 +680,7 @@ class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao_text"]) class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleNoPunc", @@ -698,7 +698,7 @@ class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao_image"]) class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",) _required_fields = [RequiredField.IMAGE] @@ -710,7 +710,7 @@ class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao_image"]) class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageRepeat",) _required_fields = [RequiredField.CONTENT] @@ -722,7 +722,7 @@ class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao_image"]) class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageQuality",) _required_fields = [RequiredField.IMAGE] @@ -734,7 +734,7 @@ class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao_image"]) class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_image.RuleImageValid", @@ -749,7 +749,7 @@ class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_video"]) class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080301", "Rule_TC609_02080301_VideoResolution", @@ -757,7 +757,7 @@ class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_video"]) class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080302", "Rule_TC609_02080302_VideoDuplication", @@ -765,7 +765,7 @@ class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_video"]) class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080303", "Rule_TC609_02080303_VideoFrameRate", @@ -773,7 +773,7 @@ class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_video"]) class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080304", "Rule_TC609_02080304_VideoDuration", @@ -781,7 +781,7 @@ class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_video"]) class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080305", "Rule_TC609_02080305_VideoClarity", @@ -789,7 +789,7 @@ class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_video"]) class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080306", "Rule_TC609_02080306_VideoDynamicRange", @@ -797,7 +797,7 @@ class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao_audio"]) class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite): # Existing RuleAudioDuration currently contains the SNR implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioDuration",) @@ -810,7 +810,7 @@ class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_audio"]) class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080402", "Rule_TC609_02080402_SignalDistortionRatio", @@ -818,7 +818,7 @@ class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_audio"]) class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080403", "Rule_TC609_02080403_AudioSampleRate", @@ -826,7 +826,7 @@ class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_audio"]) class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080404", "Rule_TC609_02080404_AudioBitDepth", @@ -834,7 +834,7 @@ class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_audio"]) class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080405", "Rule_TC609_02080405_AudioBitRate", @@ -842,7 +842,7 @@ class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao_audio"]) class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite): # Existing RuleAudioSnrQuality currently contains the duration implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioSnrQuality",) @@ -855,7 +855,7 @@ class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_model"]) class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): """0301: Placeholder for content diversity.""" @@ -867,7 +867,7 @@ class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_model"]) class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): """0302: Placeholder for scale completeness.""" @@ -879,7 +879,7 @@ class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao_model"]) class Rule_TC609_0303_DataTimeRange(BaseRule): """Check whether creation/update time fields are within configured ranges.""" @@ -1014,7 +1014,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao"]) +@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao_model"]) class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite): """0304: Annotation accuracy, partially covered by label checks.""" @@ -1031,7 +1031,7 @@ class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_placeholder"]) +@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_model"]) class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase): """0305: Placeholder for model adaptability.""" diff --git a/docs/metrics.md b/docs/metrics.md index 745262d9..da990a9c 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -60,46 +60,46 @@ This document provides comprehensive information about all quality metrics used | Type | Rule | Coverage | Group | Description | |---|---|---|---|---| -| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | covered | `guobiao` | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | -| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | covered | `guobiao` | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | -| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | covered | `guobiao` | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | -| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | covered | `guobiao` | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | partial | `guobiao` | Combines existing NLP, SFT, image, audio, and video format rules. | -| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | partial | `guobiao` | Combines unsafe-word, PII, and identity-card detection. | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | partial | `guobiao` | Combines image-label overlap and visualization checks. | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | partial | `guobiao` | Combines null-content and short-content checks. | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | partial | `guobiao` | Uses HHEM consistency checking as partial evidence of authenticity. | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | partial | `guobiao` | Combines structured-field and image-text consistency checks. | -| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | partial | `guobiao` | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | -| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | covered | `pretrain,guobiao` | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | -| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | partial | `guobiao` | Combines alphabetic-word, stop-word, and unique-word ratio checks. | -| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | covered | `guobiao` | Combines document-text and formula repetition checks. | -| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | covered | `guobiao` | Combines null, short, ellipsis-ending, and terminal-ending checks. | -| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | partial | `guobiao` | Uses content length and sentence/word counts as partial missing-information checks. | -| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | partial | `guobiao` | Combines abnormal HTML, character, invisible-content, and watermark checks. | -| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | partial | `guobiao` | Combines punctuation, word-boundary, and line-break fluency checks. | -| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | partial | `guobiao` | Uses image aspect-ratio validation as partial resolution coverage. | -| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | covered | `guobiao` | Uses PHash and CNN duplicate-image detection. | -| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | partial | `guobiao` | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | -| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | partial | `guobiao` | Combines image validity and NIMA quality as partial clarity coverage. | -| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | uncovered | `guobiao_placeholder` | Placeholder: video resolution is not implemented. | -| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | uncovered | `guobiao_placeholder` | Placeholder: duplicate-video detection is not implemented. | -| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | uncovered | `guobiao_placeholder` | Placeholder: video FPS validation is not implemented. | -| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | uncovered | `guobiao_placeholder` | Placeholder: video duration validation is not implemented. | -| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | uncovered | `guobiao_placeholder` | Placeholder: video clarity evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | uncovered | `guobiao_placeholder` | Placeholder: video dynamic-range evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | covered | `guobiao` | Uses the existing Welch power-spectrum SNR implementation. | -| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | uncovered | `guobiao_placeholder` | Placeholder: signal distortion ratio is not implemented. | -| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | uncovered | `guobiao_placeholder` | Placeholder: sample-rate quality validation is not implemented. | -| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | uncovered | `guobiao_placeholder` | Placeholder: audio bit-depth validation is not implemented. | -| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | uncovered | `guobiao_placeholder` | Placeholder: audio bit-rate validation is not implemented. | -| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | covered | `guobiao` | Uses the existing WAV duration implementation. | -| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | partial | `guobiao` | Combines available text cleanliness checks; modality coverage is partial. | -| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | uncovered | `guobiao_placeholder` | Placeholder: target-scenario distribution coverage is not implemented. | -| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | uncovered | `guobiao_placeholder` | Placeholder: dataset scale versus model requirements is not implemented. | -| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | covered | `guobiao` | Checks whether created and updated timestamps are within configured time ranges | -| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | partial | `guobiao` | Uses image annotation checks as partial evidence of annotation accuracy. | -| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | uncovered | `guobiao_placeholder` | Placeholder: before/after model performance comparison is not implemented. | +| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | +| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | +| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | +| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | partial | `guobiao_data` | Combines existing NLP, SFT, image, audio, and video format rules. | +| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | partial | `guobiao_data` | Combines unsafe-word, PII, and identity-card detection. | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | partial | `guobiao_data` | Combines image-label overlap and visualization checks. | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | partial | `guobiao_data` | Combines null-content and short-content checks. | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | partial | `guobiao_data` | Uses HHEM consistency checking as partial evidence of authenticity. | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | partial | `guobiao_data` | Combines structured-field and image-text consistency checks. | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | partial | `guobiao_data` | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | +| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | covered | `pretrain,guobiao_text` | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | +| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | partial | `guobiao_text` | Combines alphabetic-word, stop-word, and unique-word ratio checks. | +| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | covered | `guobiao_text` | Combines document-text and formula repetition checks. | +| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | covered | `guobiao_text` | Combines null, short, ellipsis-ending, and terminal-ending checks. | +| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | partial | `guobiao_text` | Uses content length and sentence/word counts as partial missing-information checks. | +| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | partial | `guobiao_text` | Combines abnormal HTML, character, invisible-content, and watermark checks. | +| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | partial | `guobiao_text` | Combines punctuation, word-boundary, and line-break fluency checks. | +| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | partial | `guobiao_image` | Uses image aspect-ratio validation as partial resolution coverage. | +| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | covered | `guobiao_image` | Uses PHash and CNN duplicate-image detection. | +| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | partial | `guobiao_image` | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | +| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | partial | `guobiao_image` | Combines image validity and NIMA quality as partial clarity coverage. | +| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | uncovered | `guobiao_video` | Placeholder: video resolution is not implemented. | +| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | uncovered | `guobiao_video` | Placeholder: duplicate-video detection is not implemented. | +| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | uncovered | `guobiao_video` | Placeholder: video FPS validation is not implemented. | +| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | uncovered | `guobiao_video` | Placeholder: video duration validation is not implemented. | +| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | uncovered | `guobiao_video` | Placeholder: video clarity evaluation is not implemented. | +| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | uncovered | `guobiao_video` | Placeholder: video dynamic-range evaluation is not implemented. | +| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | covered | `guobiao_audio` | Uses the existing Welch power-spectrum SNR implementation. | +| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | uncovered | `guobiao_audio` | Placeholder: signal distortion ratio is not implemented. | +| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | uncovered | `guobiao_audio` | Placeholder: sample-rate quality validation is not implemented. | +| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | uncovered | `guobiao_audio` | Placeholder: audio bit-depth validation is not implemented. | +| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | uncovered | `guobiao_audio` | Placeholder: audio bit-rate validation is not implemented. | +| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | covered | `guobiao_audio` | Uses the existing WAV duration implementation. | +| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | partial | `guobiao_data` | Combines available text cleanliness checks; modality coverage is partial. | +| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | uncovered | `guobiao_model` | Placeholder: target-scenario distribution coverage is not implemented. | +| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | uncovered | `guobiao_model` | Placeholder: dataset scale versus model requirements is not implemented. | +| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | covered | `guobiao_model` | Checks whether created and updated timestamps are within configured time ranges | +| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | partial | `guobiao_model` | Uses image annotation checks as partial evidence of annotation accuracy. | +| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | uncovered | `guobiao_model` | Placeholder: before/after model performance comparison is not implemented. | ### Rule-Based TEXT Quality Metrics diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index af4997d3..795079ef 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -39,6 +39,31 @@ def test_tc609_quality_defines_all_standard_metrics(): assert actual_codes == expected_primary_codes +def test_tc609_rules_are_grouped_by_evaluation_object(): + expected_group_sizes = { + "guobiao_doc": 4, + "guobiao_data": 8, + "guobiao_text": 7, + "guobiao_image": 4, + "guobiao_video": 6, + "guobiao_audio": 6, + "guobiao_model": 5, + } + + for group_name, expected_size in expected_group_sizes.items(): + tc609_rules = [ + rule + for rule in Model.rule_groups[group_name] + if rule.__name__.startswith("Rule_TC609_") + ] + assert len(tc609_rules) == expected_size + + assert not any( + rule.__name__.startswith("Rule_TC609_") + for rule in Model.rule_groups.get("guobiao", []) + ) + + def test_composite_rule_maps_component_failure_to_tc609_label(monkeypatch): class PassingRule: @classmethod @@ -81,7 +106,7 @@ def eval(cls, input_data): def test_uncovered_rule_is_explicit_placeholder(): - assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_placeholder"] + assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): Rule_TC609_0301_ContentDiversity.eval( Data(data_id="diversity", content="test") From f8065d01ea36d87c2fff75feecaffff1c3b57c85 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 18:35:39 +0800 Subject: [PATCH 41/80] feat: rename --- .../guobiao/{doc_completeness.py => rule_doc_completeness.py} | 0 examples/guobiao/{text_perplexity.py => rule_text_perplexity.py} | 0 examples/guobiao/{time_range.py => rule_time_range.py} | 0 .../guobiao/{type_consistency.py => rule_type_consistency.py} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename examples/guobiao/{doc_completeness.py => rule_doc_completeness.py} (100%) rename examples/guobiao/{text_perplexity.py => rule_text_perplexity.py} (100%) rename examples/guobiao/{time_range.py => rule_time_range.py} (100%) rename examples/guobiao/{type_consistency.py => rule_type_consistency.py} (100%) diff --git a/examples/guobiao/doc_completeness.py b/examples/guobiao/rule_doc_completeness.py similarity index 100% rename from examples/guobiao/doc_completeness.py rename to examples/guobiao/rule_doc_completeness.py diff --git a/examples/guobiao/text_perplexity.py b/examples/guobiao/rule_text_perplexity.py similarity index 100% rename from examples/guobiao/text_perplexity.py rename to examples/guobiao/rule_text_perplexity.py diff --git a/examples/guobiao/time_range.py b/examples/guobiao/rule_time_range.py similarity index 100% rename from examples/guobiao/time_range.py rename to examples/guobiao/rule_time_range.py diff --git a/examples/guobiao/type_consistency.py b/examples/guobiao/rule_type_consistency.py similarity index 100% rename from examples/guobiao/type_consistency.py rename to examples/guobiao/rule_type_consistency.py From 2efdc8763640ec229653aa5cf2042afe758fc722 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 18:36:40 +0800 Subject: [PATCH 42/80] feat: lint --- dingo/model/rule/guobiao/rule_tc609_quality.py | 7 +------ test/scripts/model/rule/test_rule_common.py | 5 +++-- test/scripts/model/rule/test_rule_tc609_quality.py | 5 +---- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 07a73444..00eb66f1 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -6,13 +6,8 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model -from dingo.model.rule.guobiao.rule_tc609_quality_base import ( - _tc609_metric_info, - Rule_TC609_Composite, - Rule_TC609_01_DocCompleteness, - _TC609PlaceholderBase, -) from dingo.model.rule.base import BaseRule +from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness, Rule_TC609_Composite, _tc609_metric_info, _TC609PlaceholderBase @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index ed47cba8..eba98916 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -3,10 +3,11 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data from dingo.io.output.eval_detail import QualityLabel -from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0101_DocBasicInfoCompleteness, Rule_TC609_0102_DocContentFeatureCompleteness, Rule_TC609_0103_DocConstructionProcessCompleteness, - Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity) + Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, + Rule_TC609_02080101_TextPerplexity) from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness +from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords class TestRuleDocFormulaRepeat: diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 795079ef..927150b3 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -6,10 +6,7 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.guobiao import rule_tc609_quality -from dingo.model.rule.guobiao.rule_tc609_quality import ( - Rule_TC609_0202_SafetyCompliance, - Rule_TC609_0301_ContentDiversity, -) +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0202_SafetyCompliance, Rule_TC609_0301_ContentDiversity def test_tc609_quality_defines_all_standard_metrics(): From 50bfb55991b6dc51bcd096e8ac9aaabcc4145d81 Mon Sep 17 00:00:00 2001 From: shijin Date: Tue, 28 Jul 2026 18:53:38 +0800 Subject: [PATCH 43/80] =?UTF-8?q?feat:=20=E7=9B=AE=E5=BD=95=E9=81=8D?= =?UTF-8?q?=E5=8E=86=EF=BC=8C=E6=96=87=E4=BB=B6=E7=B1=BB=E5=9E=8B=E9=99=90?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/data/datasource/local.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dingo/data/datasource/local.py b/dingo/data/datasource/local.py index 035daf3a..a8176b7a 100644 --- a/dingo/data/datasource/local.py +++ b/dingo/data/datasource/local.py @@ -51,7 +51,17 @@ def _find_all_files(self, path: str, file_list: List[str]): for _f in os.listdir(path): f = os.path.join(path, _f) if os.path.isfile(f): - file_list.append(f) + if getattr(self.input_args.dataset, "format", None) == "jsonl": + if f.lower().endswith(".jsonl"): + file_list.append(f) + elif getattr(self.input_args.dataset, "format", None) == "json": + if f.lower().endswith(".json"): + file_list.append(f) + elif getattr(self.input_args.dataset, "format", None) == "md": + if f.lower().endswith(".md"): + file_list.append(f) + else: + file_list.append(f) if os.path.isdir(f): self._find_all_files(f, file_list) From 3231266e49770b7d1cfac694cab34eda31cb6151 Mon Sep 17 00:00:00 2001 From: chupei Date: Wed, 29 Jul 2026 11:14:09 +0800 Subject: [PATCH 44/80] feat: add token usage --- dingo/exec/local.py | 16 ++ dingo/io/output/eval_detail.py | 19 ++- dingo/io/output/result_info.py | 17 +- dingo/io/output/summary_model.py | 59 +++++++ dingo/model/llm/agent/agent_hallucination.py | 3 +- dingo/model/llm/agent/base_agent.py | 3 +- .../model/llm/agent/tools/claims_extractor.py | 25 ++- dingo/model/llm/base.py | 16 +- dingo/model/llm/base_litellm.py | 10 +- dingo/model/llm/base_openai.py | 129 ++++++++++++++- dingo/model/llm/llm_custom_metric.py | 20 ++- dingo/model/llm/llm_factcheck_public.py | 39 ++++- .../llm/llm_search_result_effectiveness.py | 18 +- .../model/llm/llm_search_result_relevance.py | 16 +- .../model/llm/rag/llm_rag_answer_relevancy.py | 20 ++- .../llm/rag/llm_rag_context_precision.py | 11 +- dingo/model/llm/vlm_layout_quality.py | 10 +- docs/config.md | 49 ++++++ test/scripts/exec/test_local.py | 83 +++++++++- test/scripts/io/test_summary_model.py | 44 +++++ test/scripts/model/llm/test_litellm.py | 24 ++- .../model/llm/test_llm_custom_metric.py | 38 +++++ test/scripts/model/llm/test_token_usage.py | 156 ++++++++++++++++++ 23 files changed, 784 insertions(+), 41 deletions(-) create mode 100644 test/scripts/model/llm/test_token_usage.py diff --git a/dingo/exec/local.py b/dingo/exec/local.py index 18f40328..7321015b 100644 --- a/dingo/exec/local.py +++ b/dingo/exec/local.py @@ -134,6 +134,13 @@ def execute(self) -> SummaryModel: self.summary.type_count[field_key].setdefault(label, 0) self.summary.type_count[field_key][label] += 1 + for field_key, eval_detail_list in result_info.token_usage_details.items(): + for eval_detail in eval_detail_list: + if eval_detail.usage is not None and eval_detail.metric: + self.summary.add_token_usage( + field_key, eval_detail.metric, eval_detail.usage + ) + if result_info.eval_status: self.summary.num_bad += 1 else: @@ -210,6 +217,9 @@ def evaluate_single_data(self, dingo_id: str, eval_fields: dict, eval_type: str, # Set result_info fields join_fields = ','.join(eval_fields.values()) if eval_fields else 'default' + usage_detail_list = [mr for mr in eval_detail_list if mr.usage is not None] + if usage_detail_list: + result_info.token_usage_details = {join_fields: usage_detail_list} # 根据配置决定保存哪些结果 if self.input_args.executor.result_save.all_labels or self.input_args.executor.result_save.merge: @@ -242,6 +252,12 @@ def merge_result_info(self, existing_list: List[ResultInfo], new_item: ResultInf # 第一层是字段名，如果不存在，则直接赋值 else: existing_item.eval_details[key] = value + + for key, value in new_item.token_usage_details.items(): + if key in existing_item.token_usage_details: + existing_item.token_usage_details[key].extend(value) + else: + existing_item.token_usage_details[key] = value else: existing_list.append(new_item) diff --git a/dingo/io/output/eval_detail.py b/dingo/io/output/eval_detail.py index f2073dca..03592d35 100644 --- a/dingo/io/output/eval_detail.py +++ b/dingo/io/output/eval_detail.py @@ -1,6 +1,6 @@ -from typing import Any, Dict, List, Optional +from typing import Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel class QualityLabel: @@ -9,6 +9,20 @@ class QualityLabel: QUALITY_BAD_PREFIX = "QUALITY_BAD_" # Indicates not pass the quality check +class TokenUsage(BaseModel): + """Token usage returned by an LLM provider for one evaluator call.""" + + prompt_tokens: Optional[int] = None + completion_tokens: Optional[int] = None + total_tokens: Optional[int] = None + reasoning_tokens: Optional[int] = None + cached_tokens: Optional[int] = None + model: Optional[str] = None + provider: Optional[str] = None + calls: int = 1 + source: str = "provider" + + class EvalDetail(BaseModel): metric: str status: bool = False @@ -16,3 +30,4 @@ class EvalDetail(BaseModel): score: Optional[float] = None label: Optional[list[str]] = None reason: Optional[list] = None + usage: Optional[TokenUsage] = None diff --git a/dingo/io/output/result_info.py b/dingo/io/output/result_info.py index 6a4a9549..6adc4abf 100644 --- a/dingo/io/output/result_info.py +++ b/dingo/io/output/result_info.py @@ -5,7 +5,7 @@ from decimal import Decimal from typing import Any, Dict, List, Optional -from pydantic import BaseModel +from pydantic import BaseModel, Field from dingo.io.output.eval_detail import EvalDetail @@ -15,6 +15,17 @@ class ResultInfo(BaseModel): raw_data: Dict = {} eval_status: bool = False eval_details: Dict[str, List[EvalDetail]] = {} + token_usage_details: Dict[str, List[EvalDetail]] = Field( + default_factory=dict, + exclude=True, + ) + + @staticmethod + def _eval_detail_to_dict(model_res: EvalDetail) -> Dict[str, Any]: + detail = model_res.model_dump() + if detail.get('usage') is None: + detail.pop('usage', None) + return detail @staticmethod def _apply_field_filter(output_data: Dict[str, Any], field_list: Optional[List[str]]) -> Dict[str, Any]: @@ -82,7 +93,7 @@ def to_dict(self, field_list: Optional[List[str]] = None): 'raw_data': self._normalize_value(self.raw_data), 'eval_status': self.eval_status, 'eval_details': { - k: [model_res.model_dump() for model_res in v] + k: [self._eval_detail_to_dict(model_res) for model_res in v] for k, v in self.eval_details.items() }, } @@ -112,7 +123,7 @@ def move_conflict_field(field_name: str): dingo_result = { 'eval_status': self.eval_status, 'eval_details': { - k: [model_res.model_dump() for model_res in v] + k: [self._eval_detail_to_dict(model_res) for model_res in v] for k, v in self.eval_details.items() }, } diff --git a/dingo/io/output/summary_model.py b/dingo/io/output/summary_model.py index f2de1df2..cb2a2fe9 100644 --- a/dingo/io/output/summary_model.py +++ b/dingo/io/output/summary_model.py @@ -3,6 +3,8 @@ from pydantic import BaseModel, Field +from dingo.io.output.eval_detail import TokenUsage + class SummaryModel(BaseModel): task_id: str = '' @@ -22,6 +24,7 @@ class SummaryModel(BaseModel): # 新增：指标分数统计（用于RAG等评估场景） # 结构：{field_key: {metric_name: {scores, score_average, ...}}} metrics_score_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict) + token_usage_stats: Dict[str, Dict[str, Dict[str, Any]]] = Field(default_factory=dict) def add_metric_score(self, field_key: str, metric_name: str, score: float): """ @@ -46,6 +49,59 @@ def add_metric_score(self, field_key: str, metric_name: str, score: float): metric_stats['scores'].append(score) metric_stats['score_count'] += 1 + def add_token_usage(self, field_key: str, metric_name: str, usage: TokenUsage): + """ + 添加 LLM token 使用量到统计中 + + Args: + field_key: 字段名（如 'user_input,response'） + metric_name: 指标名称（如 LLMTextQualityV5） + usage: 单次 LLM 调用的 token 使用量 + """ + usage_stats = self.token_usage_stats.setdefault(field_key, {}).setdefault( + metric_name, + { + 'prompt_tokens': 0, + 'completion_tokens': 0, + 'total_tokens': 0, + 'reasoning_tokens': 0, + 'cached_tokens': 0, + 'calls': 0, + 'records': 0, + 'models': {}, + 'providers': {}, + 'sources': {}, + }, + ) + + for token_field in [ + 'prompt_tokens', + 'completion_tokens', + 'total_tokens', + 'reasoning_tokens', + 'cached_tokens', + ]: + value = getattr(usage, token_field, None) + if value is not None: + usage_stats[token_field] += int(value) + + calls = int(usage.calls or 1) + usage_stats['calls'] += calls + usage_stats['records'] += 1 + + if usage.model: + usage_stats['models'][usage.model] = ( + usage_stats['models'].get(usage.model, 0) + calls + ) + if usage.provider: + usage_stats['providers'][usage.provider] = ( + usage_stats['providers'].get(usage.provider, 0) + calls + ) + if usage.source: + usage_stats['sources'][usage.source] = ( + usage_stats['sources'].get(usage.source, 0) + calls + ) + def calculate_metrics_score_averages(self): """ 计算所有字段和指标分数的平均值、最小值、最大值、标准差 @@ -133,4 +189,7 @@ def to_dict(self): for field_key, metrics in self.metrics_score_stats.items() } + if self.token_usage_stats: + result['token_usage'] = self.token_usage_stats + return result diff --git a/dingo/model/llm/agent/agent_hallucination.py b/dingo/model/llm/agent/agent_hallucination.py index fc22ba56..889cd07d 100644 --- a/dingo/model/llm/agent/agent_hallucination.py +++ b/dingo/model/llm/agent/agent_hallucination.py @@ -20,6 +20,7 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model import Model from dingo.model.llm.agent.base_agent import BaseAgent +from dingo.model.llm.base import llm_response_content from dingo.utils import log @@ -339,7 +340,7 @@ def _extract_claims(cls, input_data: Data) -> List[str]: # Call LLM messages = [{"role": "user", "content": prompt}] - response = cls.send_messages(messages) + response = llm_response_content(cls.send_messages(messages)) # Parse JSON response # Handle markdown code blocks diff --git a/dingo/model/llm/agent/base_agent.py b/dingo/model/llm/agent/base_agent.py index d3db23d2..8308ccbd 100644 --- a/dingo/model/llm/agent/base_agent.py +++ b/dingo/model/llm/agent/base_agent.py @@ -16,6 +16,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.llm.agent.tools import ToolRegistry +from dingo.model.llm.base import llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log @@ -251,7 +252,7 @@ def eval(cls, input_data: Data) -> EvalDetail: prompt = step.get('prompt', '') # Use parent's send_messages method messages = [{"role": "user", "content": prompt}] - response = cls.send_messages(messages) + response = llm_response_content(cls.send_messages(messages)) results.append(response) else: diff --git a/dingo/model/llm/agent/tools/claims_extractor.py b/dingo/model/llm/agent/tools/claims_extractor.py index f3204b96..d8dc786c 100644 --- a/dingo/model/llm/agent/tools/claims_extractor.py +++ b/dingo/model/llm/agent/tools/claims_extractor.py @@ -23,8 +23,10 @@ from pydantic import Field +from dingo.io.output.eval_detail import TokenUsage from dingo.model.llm.agent.tools.base_tool import BaseTool, ToolConfig from dingo.model.llm.agent.tools.tool_registry import tool_register +from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log @@ -351,10 +353,11 @@ def execute( # Extract claims from each chunk all_claims = [] + token_usage: TokenUsage | None = None for i, chunk_data in enumerate(chunks): - log.debug(f"Processing chunk {i+1}/{len(chunks)}") + log.debug(f"Processing chunk {i + 1}/{len(chunks)}") - chunk_claims = cls._extract_claims_from_chunk( + chunk_claims, chunk_usage = cls._extract_claims_from_chunk( client, chunk_data['text'], chunk_data['start_pos'], @@ -362,6 +365,7 @@ def execute( include_context ) all_claims.extend(chunk_claims) + token_usage = BaseOpenAI._merge_token_usage(token_usage, chunk_usage) # Deduplicate and merge similar claims unique_claims = cls._deduplicate_claims(all_claims) @@ -377,6 +381,8 @@ def execute( # Build metadata metadata = cls._build_metadata(unique_claims) + if token_usage is not None: + metadata['token_usage'] = token_usage.model_dump() result = { 'success': True, @@ -460,7 +466,7 @@ def _extract_claims_from_chunk( start_pos: int, claim_types: List[str], include_context: bool - ) -> List[Dict]: + ) -> tuple[List[Dict], TokenUsage | None]: """ Extract claims from a single text chunk using LLM. @@ -472,7 +478,7 @@ def _extract_claims_from_chunk( include_context: Whether to include context Returns: - List of extracted claims + Tuple of extracted claims and token usage """ # Build user prompt user_prompt = f"""Extract verifiable claims from the following text. @@ -496,6 +502,11 @@ def _extract_claims_from_chunk( temperature=cls.config.temperature, response_format={"type": "json_object"} # Force JSON output ) + token_usage = BaseOpenAI._extract_token_usage( + response, + model_name=cls.config.model, + provider="openai", + ) output_text = response.choices[0].message.content @@ -520,14 +531,14 @@ def _extract_claims_from_chunk( filtered_claims.append(claim) - return filtered_claims + return filtered_claims, token_usage except json.JSONDecodeError as e: log.warning(f"Failed to parse LLM output as JSON: {e}") - return [] + return [], token_usage if 'token_usage' in locals() else None except Exception as e: log.error(f"LLM call failed: {e}") - return [] + return [], None @classmethod def _deduplicate_claims(cls, claims: List[Dict]) -> List[Dict]: diff --git a/dingo/model/llm/base.py b/dingo/model/llm/base.py index 440193e2..e0fb53cd 100644 --- a/dingo/model/llm/base.py +++ b/dingo/model/llm/base.py @@ -1,8 +1,20 @@ -from typing import List +from typing import List, Optional from dingo.config.input_args import EvaluatorLLMArgs from dingo.io import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage + + +class LLMCallResult: + def __init__(self, content: str, usage: Optional[TokenUsage] = None): + self.content = content + self.usage = usage + + +def llm_response_content(response) -> str: + if isinstance(response, LLMCallResult): + return response.content + return str(response) class BaseLLM: diff --git a/dingo/model/llm/base_litellm.py b/dingo/model/llm/base_litellm.py index bcf92528..b3ffbe93 100644 --- a/dingo/model/llm/base_litellm.py +++ b/dingo/model/llm/base_litellm.py @@ -1,6 +1,7 @@ from typing import List from dingo.config.input_args import EvaluatorLLMArgs +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils.exception import ExceedMaxTokens @@ -106,4 +107,11 @@ def send_messages(cls, messages: List) -> str: ) content = choice.message.content # type: ignore[union-attr] - return str(content) if content is not None else "" + return LLMCallResult( + content=str(content) if content is not None else "", + usage=cls._extract_token_usage( + response, + model_name=model_name, + provider="litellm", + ), + ) diff --git a/dingo/model/llm/base_openai.py b/dingo/model/llm/base_openai.py index c3911699..a644fa3a 100644 --- a/dingo/model/llm/base_openai.py +++ b/dingo/model/llm/base_openai.py @@ -6,8 +6,8 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data, RequiredField -from dingo.io.output.eval_detail import EvalDetail, QualityLabel -from dingo.model.llm.base import BaseLLM +from dingo.io.output.eval_detail import EvalDetail, QualityLabel, TokenUsage +from dingo.model.llm.base import BaseLLM, LLMCallResult from dingo.model.response.response_class import ResponseScoreReason from dingo.utils import log from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens @@ -96,7 +96,121 @@ def send_messages(cls, messages: List): f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}" ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=cls._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) + + @staticmethod + def _usage_value(data, key: str): + if data is None: + return None + if isinstance(data, dict): + return data.get(key) + return getattr(data, key, None) + + @staticmethod + def _coerce_optional_int(value): + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + @classmethod + def _extract_token_usage( + cls, + completion, + model_name: str, + provider: str = "openai", + ) -> TokenUsage | None: + raw_usage = getattr(completion, "usage", None) + if raw_usage is None: + return None + + if hasattr(raw_usage, "model_dump"): + usage_data = raw_usage.model_dump() + elif isinstance(raw_usage, dict): + usage_data = raw_usage + else: + usage_data = raw_usage + + completion_details = cls._usage_value( + usage_data, "completion_tokens_details" + ) + prompt_details = cls._usage_value(usage_data, "prompt_tokens_details") + + return TokenUsage( + prompt_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "prompt_tokens") + ), + completion_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "completion_tokens") + ), + total_tokens=cls._coerce_optional_int( + cls._usage_value(usage_data, "total_tokens") + ), + reasoning_tokens=cls._coerce_optional_int( + cls._usage_value(completion_details, "reasoning_tokens") + ), + cached_tokens=cls._coerce_optional_int( + cls._usage_value(prompt_details, "cached_tokens") + ), + model=model_name, + provider=provider, + source="provider", + ) + + @staticmethod + def _copy_token_usage(usage: TokenUsage) -> TokenUsage: + if hasattr(usage, "model_copy"): + return usage.model_copy(deep=True) + return usage.copy(deep=True) + + @classmethod + def _merge_token_usage( + cls, + current: TokenUsage | None, + new_usage: TokenUsage | None, + ) -> TokenUsage | None: + if new_usage is None: + return current + if current is None: + return cls._copy_token_usage(new_usage) + + def _sum_optional(left, right): + if left is None and right is None: + return None + return int(left or 0) + int(right or 0) + + current.prompt_tokens = _sum_optional( + current.prompt_tokens, new_usage.prompt_tokens + ) + current.completion_tokens = _sum_optional( + current.completion_tokens, new_usage.completion_tokens + ) + current.total_tokens = _sum_optional( + current.total_tokens, new_usage.total_tokens + ) + current.reasoning_tokens = _sum_optional( + current.reasoning_tokens, new_usage.reasoning_tokens + ) + current.cached_tokens = _sum_optional( + current.cached_tokens, new_usage.cached_tokens + ) + current.calls += int(new_usage.calls or 1) + if current.model != new_usage.model: + current.model = current.model or new_usage.model + if current.provider != new_usage.provider: + current.provider = current.provider or new_usage.provider + if current.source != new_usage.source: + current.source = current.source or new_usage.source + return current @classmethod def validate_numeric_range(cls, value, min_val, max_val, param_name): @@ -191,10 +305,16 @@ def eval(cls, input_data: Data) -> EvalDetail: attempts = 0 except_msg = "" except_name = Exception.__class__.__name__ + usage: TokenUsage | None = None while attempts < 3: try: response = cls.send_messages(messages) - res: EvalDetail = cls.process_response(response) + if isinstance(response, LLMCallResult): + usage = cls._merge_token_usage(usage, response.usage) + res: EvalDetail = cls.process_response(response.content) + res.usage = usage + else: + res: EvalDetail = cls.process_response(response) return res except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: except_msg = str(e) @@ -216,4 +336,5 @@ def eval(cls, input_data: Data) -> EvalDetail: res.status = True res.label = [f"QUALITY_BAD.{except_name}"] res.reason = [except_msg] + res.usage = usage return res diff --git a/dingo/model/llm/llm_custom_metric.py b/dingo/model/llm/llm_custom_metric.py index 53d2968e..8624932b 100644 --- a/dingo/model/llm/llm_custom_metric.py +++ b/dingo/model/llm/llm_custom_metric.py @@ -7,6 +7,7 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data from dingo.io.output.eval_detail import EvalDetail +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.model.model import Model from dingo.utils.exception import ConvertJsonError, ExceedMaxTokens @@ -112,7 +113,14 @@ def send_messages(self, messages: List): f"Exceed max tokens: {extra_params.get('max_tokens', 4000)}" ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=self._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) def _eval_detail_from_response(self, response_json: dict) -> EvalDetail: custom_metric = self._get_custom_metric() @@ -189,9 +197,15 @@ def eval(self, input_data: Data) -> EvalDetail: attempts = 0 except_msg = "" except_name = Exception.__name__ + usage = None while attempts < 3: try: response = self.send_messages(messages) + if isinstance(response, LLMCallResult): + usage = self._merge_token_usage(usage, response.usage) + result = self.process_response(response.content) + result.usage = usage + return result return self.process_response(response) except (ValidationError, ExceedMaxTokens, ConvertJsonError) as e: except_msg = str(e) @@ -203,9 +217,11 @@ def eval(self, input_data: Data) -> EvalDetail: except_msg = str(e) except_name = e.__class__.__name__ - return EvalDetail( + result = EvalDetail( metric=self._get_custom_metric().metric, status=True, label=[f"QUALITY_BAD.{except_name}"], reason=[except_msg], ) + result.usage = usage + return result diff --git a/dingo/model/llm/llm_factcheck_public.py b/dingo/model/llm/llm_factcheck_public.py index 966c44a1..18e0aee8 100644 --- a/dingo/model/llm/llm_factcheck_public.py +++ b/dingo/model/llm/llm_factcheck_public.py @@ -4,6 +4,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI @@ -199,17 +200,25 @@ def eval(cls, input_data: Data) -> EvalDetail: cls.create_client() # 1. 提取声明 - claims = cls._extract_claims(input_data.prompt, input_data.content) + usage = None + claims, claim_usage = cls._extract_claims_with_usage( + input_data.prompt, input_data.content + ) + usage = cls._merge_token_usage(usage, claim_usage) if not claims: result = EvalDetail(metric=cls.__name__) result.reason = ["No factual claims found"] + result.usage = usage return result # 2. 分批验证 all_results = [] for i in range(0, len(claims), cls.batch_size): batch = claims[i:i + cls.batch_size] - results = cls._verify_claims(input_data.prompt, input_data.content, batch) + results, batch_usage = cls._verify_claims_with_usage( + input_data.prompt, input_data.content, batch + ) + usage = cls._merge_token_usage(usage, batch_usage) all_results.extend(results) # 3. 计算指标 @@ -218,6 +227,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 4. 设置评估结果 result = EvalDetail(metric=cls.__name__) result.reason = [cls._format_reason(metrics)] + result.usage = usage # 5. 根据分数设置状态 if metrics["factual_ratio"] < cls.threshold: @@ -237,6 +247,11 @@ def eval(cls, input_data: Data) -> EvalDetail: @classmethod def _extract_claims(cls, prompt: str, response: str) -> List[str]: + claims, _ = cls._extract_claims_with_usage(prompt, response) + return claims + + @classmethod + def _extract_claims_with_usage(cls, prompt: str, response: str): """提取事实性声明""" messages = [ {"role": "user", "content": (cls.prompt["CLAIM_LISTING"] + @@ -245,10 +260,12 @@ def _extract_claims(cls, prompt: str, response: str) -> List[str]: response=response )} ] - result = cls.send_messages(messages) + response_result = cls.send_messages(messages) + result = llm_response_content(response_result) try: claims = cls._parse_json_list(result) - return [c for c in claims if c.strip()] # 过滤空声明 + usage = response_result.usage if isinstance(response_result, LLMCallResult) else None + return [c for c in claims if c.strip()], usage # 过滤空声明 except Exception as e: raise ValueError(f"Failed to parse claims: {str(e)}") @@ -257,6 +274,14 @@ def _verify_claims(cls, prompt: str, response: str, claims: List[str]) -> List[FactCheckResult]: + results, _ = cls._verify_claims_with_usage(prompt, response, claims) + return results + + @classmethod + def _verify_claims_with_usage(cls, + prompt: str, + response: str, + claims: List[str]): """验证一批声明""" messages = [ {"role": "user", "content": (cls.prompt["FACT_CHECKING"] + @@ -266,9 +291,11 @@ def _verify_claims(cls, claims=claims )} ] - result = cls.send_messages(messages) + response_result = cls.send_messages(messages) + result = llm_response_content(response_result) try: - return cls._parse_check_results(result) + usage = response_result.usage if isinstance(response_result, LLMCallResult) else None + return cls._parse_check_results(result), usage except Exception as e: raise ValueError(f"Failed to parse check results: {str(e)}") diff --git a/dingo/model/llm/llm_search_result_effectiveness.py b/dingo/model/llm/llm_search_result_effectiveness.py index 86e76ae2..fcf25f2c 100644 --- a/dingo/model/llm/llm_search_result_effectiveness.py +++ b/dingo/model/llm/llm_search_result_effectiveness.py @@ -20,8 +20,9 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI logger = logging.getLogger(__name__) @@ -367,6 +368,7 @@ class LLMFieldQuality: issues: list[str] | None = None reason: str = "" error: str = "" + usage: TokenUsage | None = None def field_score(self, field: str) -> float: return { @@ -430,6 +432,7 @@ class EffectivenessGrade: issues: list[str] | None = None llm_quality_reason: str = "" llm_quality_error: str = "" + usage: TokenUsage | None = None def to_dict(self) -> dict[str, Any]: return { @@ -555,6 +558,7 @@ def _judge_llm_field_quality( return LLMFieldQuality() client = self._get_client() last_result = LLMFieldQuality(error="LLM field quality judgment failed") + usage: TokenUsage | None = None for attempt in range(3): try: completion = client.chat.completions.create( @@ -577,14 +581,24 @@ def _judge_llm_field_quality( max_tokens=self.max_tokens, timeout=self.timeout, ) + usage = BaseOpenAI._merge_token_usage( + usage, + BaseOpenAI._extract_token_usage( + completion, + model_name=self.model, + provider="openai", + ), + ) response_text = completion.choices[0].message.content or "" last_result = _parse_llm_field_quality_response(response_text) + last_result.usage = usage if not last_result.error: return last_result error: Exception | str = last_result.error except Exception as exc: error = exc last_result = LLMFieldQuality(error=str(exc)) + last_result.usage = usage logger.warning( "LLM field quality attempt %s/3 failed for title=%r: %s", @@ -722,6 +736,7 @@ def apply_confirmed_field_issue(field: str, score: float) -> float: issues=issues, llm_quality_reason=llm_quality.reason, llm_quality_error=llm_quality.error, + usage=llm_quality.usage, ) @classmethod @@ -766,6 +781,7 @@ def eval(cls, input_data: Data) -> EvalDetail: score=round(grade.score, 5), label=labels, reason=[grade.to_dict()], + usage=grade.usage, ) diff --git a/dingo/model/llm/llm_search_result_relevance.py b/dingo/model/llm/llm_search_result_relevance.py index febea86f..a15c2d0a 100644 --- a/dingo/model/llm/llm_search_result_relevance.py +++ b/dingo/model/llm/llm_search_result_relevance.py @@ -25,8 +25,9 @@ from dingo.config.input_args import EvaluatorLLMArgs from dingo.io.input import Data -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI logger = logging.getLogger(__name__) @@ -127,6 +128,7 @@ class RelevanceGrade: confidence: float = 0.0 reasoning: str = "" error: str = "" + usage: TokenUsage | None = None def to_dict(self) -> dict[str, Any]: d: dict[str, Any] = { @@ -475,6 +477,7 @@ def grade( client = self._get_client() last_grade = RelevanceGrade(error="LLM grading failed") + usage: TokenUsage | None = None for attempt in range(3): try: completion = client.chat.completions.create( @@ -487,14 +490,24 @@ def grade( max_tokens=self.max_tokens, timeout=self.timeout, ) + usage = BaseOpenAI._merge_token_usage( + usage, + BaseOpenAI._extract_token_usage( + completion, + model_name=self.model, + provider="openai", + ), + ) response_text = completion.choices[0].message.content or "" last_grade = _parse_grade_response(response_text) + last_grade.usage = usage if not last_grade.error: return last_grade error: Exception | str = last_grade.error except Exception as exc: error = exc last_grade = RelevanceGrade(error=str(exc)) + last_grade.usage = usage logger.warning( "LLM grading attempt %s/3 failed for query=%r title=%r: %s", @@ -577,6 +590,7 @@ def eval(cls, input_data: Data) -> EvalDetail: score=round(grade.score, 5), label=labels, reason=[reason], + usage=grade.usage, ) diff --git a/dingo/model/llm/rag/llm_rag_answer_relevancy.py b/dingo/model/llm/rag/llm_rag_answer_relevancy.py index 86febfa7..898d31b2 100644 --- a/dingo/model/llm/rag/llm_rag_answer_relevancy.py +++ b/dingo/model/llm/rag/llm_rag_answer_relevancy.py @@ -13,6 +13,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.exception import ConvertJsonError @@ -106,7 +107,14 @@ def build_messages(cls, input_data: Data) -> List: @classmethod def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[str, Any]]: """生成多个相关问题""" + questions, _ = cls._generate_multiple_questions_with_usage(input_data, n) + return questions + + @classmethod + def _generate_multiple_questions_with_usage(cls, input_data: Data, n: int = 3): + """生成多个相关问题，同时返回 LLM token 使用量""" questions = [] + usage = None # 确保客户端已经创建 if not hasattr(cls, 'client') or cls.client is None: @@ -117,13 +125,16 @@ def generate_multiple_questions(cls, input_data: Data, n: int = 3) -> List[Dict[ messages = cls.build_messages(input_data) # 调用LLM生成问题 - response = cls.send_messages(messages) + response_result = cls.send_messages(messages) + response = llm_response_content(response_result) + if isinstance(response_result, LLMCallResult): + usage = cls._merge_token_usage(usage, response_result.usage) # 处理响应 processed_response = cls.process_question_response(response) questions.append(processed_response) - return questions + return questions, usage @classmethod def process_question_response(cls, response: str) -> Dict[str, Any]: @@ -246,7 +257,9 @@ def eval(cls, input_data: Data) -> EvalDetail: cls.dynamic_config.temperature = 0.7 # 生成多个相关问题 - generated_questions = cls.generate_multiple_questions(input_data, cls.strictness) + generated_questions, usage = cls._generate_multiple_questions_with_usage( + input_data, cls.strictness + ) # 计算相关性分数和详细信息 score, details = cls.calculate_score(generated_questions, original_question) @@ -254,6 +267,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 构建结果 result = EvalDetail(metric=cls.__name__) result.score = score + result.usage = usage # 根据分数判断是否通过，默认阈值为5 threshold = 5 diff --git a/dingo/model/llm/rag/llm_rag_context_precision.py b/dingo/model/llm/rag/llm_rag_context_precision.py index a5927ec3..c1be34ed 100644 --- a/dingo/model/llm/rag/llm_rag_context_precision.py +++ b/dingo/model/llm/rag/llm_rag_context_precision.py @@ -10,6 +10,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult, llm_response_content from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.exception import ConvertJsonError @@ -293,6 +294,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # 获取所有上下文的消息 messages_list = cls.build_messages(input_data) responses = [] + usage = None # 为每个上下文发送单独的请求 for item in messages_list: @@ -302,7 +304,10 @@ def eval(cls, input_data: Data) -> EvalDetail: while attempts < 3: try: - response = cls.send_messages(messages) + response_result = cls.send_messages(messages) + response = llm_response_content(response_result) + if isinstance(response_result, LLMCallResult): + usage = cls._merge_token_usage(usage, response_result.usage) break except Exception as e: attempts += 1 @@ -326,4 +331,6 @@ def eval(cls, input_data: Data) -> EvalDetail: responses.append(response) # 处理所有响应 - return cls.process_response(responses) + result = cls.process_response(responses) + result.usage = usage + return result diff --git a/dingo/model/llm/vlm_layout_quality.py b/dingo/model/llm/vlm_layout_quality.py index 14c03fc2..812ad145 100644 --- a/dingo/model/llm/vlm_layout_quality.py +++ b/dingo/model/llm/vlm_layout_quality.py @@ -4,6 +4,7 @@ from dingo.io.input import Data, RequiredField from dingo.io.output.eval_detail import EvalDetail from dingo.model import Model +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.base_openai import BaseOpenAI from dingo.utils import log from dingo.utils.image_loader import ImageLoader @@ -159,7 +160,14 @@ def send_messages(cls, messages: List): temperature=0.1 ) - return str(completions.choices[0].message.content) + return LLMCallResult( + content=str(completions.choices[0].message.content), + usage=cls._extract_token_usage( + completions, + model_name=model_name, + provider="openai", + ), + ) @classmethod def process_response(cls, response: str) -> EvalDetail: diff --git a/docs/config.md b/docs/config.md index ec595fcf..ba7ba129 100644 --- a/docs/config.md +++ b/docs/config.md @@ -100,6 +100,55 @@ HuggingFace 特定配置： | all_labels | bool | false | No | 是否保存所有标签 | | raw | bool | false | No | 是否保存原始数据 | +### LLM Token 使用量输出 + +当 LLM 服务返回 token usage 时，Dingo 会在对应的 `EvalDetail` 中写入 `usage` 字段，并在 `summary.json` 中按字段组合和 evaluator 汇总到 `token_usage`。该统计来自模型服务商返回的 `usage`，不会本地估算；如果兼容 API 不返回 usage，则对应字段为空。 + +单条结果示例： + +```json +{ + "metric": "LLMTextQualityV5", + "status": false, + "label": ["QUALITY_GOOD"], + "reason": ["pass"], + "usage": { + "prompt_tokens": 812, + "completion_tokens": 96, + "total_tokens": 908, + "reasoning_tokens": null, + "cached_tokens": null, + "model": "gpt-4o-mini", + "provider": "openai", + "calls": 1, + "source": "provider" + } +} +``` + +汇总结果示例： + +```json +{ + "token_usage": { + "content": { + "LLMTextQualityV5": { + "prompt_tokens": 81200, + "completion_tokens": 9600, + "total_tokens": 90800, + "reasoning_tokens": 0, + "cached_tokens": 12000, + "calls": 100, + "records": 100, + "models": {"gpt-4o-mini": 100}, + "providers": {"openai": 100}, + "sources": {"provider": 100} + } + } + } +} +``` + ### Evaluator 配置 (evaluator) 评估器相关配置： diff --git a/test/scripts/exec/test_local.py b/test/scripts/exec/test_local.py index 44ca5014..f96f3525 100644 --- a/test/scripts/exec/test_local.py +++ b/test/scripts/exec/test_local.py @@ -3,7 +3,8 @@ from dingo.config import InputArgs from dingo.exec import Executor, LocalExecutor from dingo.io import ResultInfo -from dingo.io.output.eval_detail import EvalDetail +from dingo.io.output.eval_detail import EvalDetail, TokenUsage +from dingo.model import Model class TestLocal: @@ -192,6 +193,86 @@ def test_merge_result_info(self): assert "�I am 8 years old. ^I love apple because:" in all_reasons assert "文本中包含不可见字符或乱码（如�和^），可能影响阅读理解。" in all_reasons + def test_merge_result_info_preserves_token_usage_details(self): + localexecutor = LocalExecutor({}) + item1 = ResultInfo( + dingo_id="1", + token_usage_details={ + "content": [ + EvalDetail( + metric="LLMMetricA", + usage=TokenUsage(total_tokens=3), + ) + ] + }, + ) + item2 = ResultInfo( + dingo_id="1", + token_usage_details={ + "content": [ + EvalDetail( + metric="LLMMetricB", + usage=TokenUsage(total_tokens=5), + ) + ] + }, + ) + + merged = localexecutor.merge_result_info([], item1) + merged = localexecutor.merge_result_info(merged, item2) + + assert len(merged[0].token_usage_details["content"]) == 2 + + def test_token_usage_summary_can_include_unsaved_good_eval_details(self): + class TokenUsageGoodLLM: + def eval(self, input_data): + return EvalDetail( + metric="TokenUsageGoodLLM", + status=False, + label=["QUALITY_GOOD"], + usage=TokenUsage( + prompt_tokens=10, + completion_tokens=5, + total_tokens=15, + ), + ) + + old_model = Model.llm_name_map.get("TokenUsageGoodLLM") + Model.llm_name_map["TokenUsageGoodLLM"] = TokenUsageGoodLLM + try: + input_args = InputArgs( + executor={ + "result_save": { + "bad": True, + "good": False, + "all_labels": False, + } + }, + evaluator=[ + { + "fields": {"content": "content"}, + "evals": [{"name": "TokenUsageGoodLLM"}], + } + ], + ) + executor = LocalExecutor(input_args) + + result = executor.evaluate_single_data( + dingo_id="1", + eval_fields={"content": "content"}, + eval_type="llm", + map_data={"content": "ok"}, + eval_list=input_args.evaluator[0].evals, + ) + + assert result.eval_details == {} + assert result.token_usage_details["content"][0].usage.total_tokens == 15 + finally: + if old_model is None: + Model.llm_name_map.pop("TokenUsageGoodLLM", None) + else: + Model.llm_name_map["TokenUsageGoodLLM"] = old_model + def test_all_labels_config(self): input_data = { "input_path": "test/data/test_local_jsonl.jsonl", diff --git a/test/scripts/io/test_summary_model.py b/test/scripts/io/test_summary_model.py index c9420ffc..9e7fac5f 100644 --- a/test/scripts/io/test_summary_model.py +++ b/test/scripts/io/test_summary_model.py @@ -11,6 +11,7 @@ import pytest +from dingo.io.output.eval_detail import TokenUsage from dingo.io.output.summary_model import SummaryModel @@ -221,6 +222,49 @@ def test_to_dict_without_scores(self): # 验证没有分数统计字段 assert "metrics_score" not in result + def test_add_token_usage_and_to_dict(self): + """测试 LLM token 使用量统计输出""" + summary = SummaryModel(task_name="test_task", task_id="test_token_001") + + summary.add_token_usage( + "content", + "LLMTextQualityV5", + TokenUsage( + prompt_tokens=10, + completion_tokens=4, + total_tokens=14, + reasoning_tokens=1, + cached_tokens=3, + model="gpt-test", + provider="openai", + ), + ) + summary.add_token_usage( + "content", + "LLMTextQualityV5", + TokenUsage( + prompt_tokens=8, + completion_tokens=5, + total_tokens=13, + model="gpt-test", + provider="openai", + ), + ) + + result = summary.to_dict() + + stats = result["token_usage"]["content"]["LLMTextQualityV5"] + assert stats["prompt_tokens"] == 18 + assert stats["completion_tokens"] == 9 + assert stats["total_tokens"] == 27 + assert stats["reasoning_tokens"] == 1 + assert stats["cached_tokens"] == 3 + assert stats["calls"] == 2 + assert stats["records"] == 2 + assert stats["models"] == {"gpt-test": 2} + assert stats["providers"] == {"openai": 2} + assert stats["sources"] == {"provider": 2} + def test_multiple_metrics_different_score_counts(self): """测试不同指标有不同数量的分数""" summary = SummaryModel( diff --git a/test/scripts/model/llm/test_litellm.py b/test/scripts/model/llm/test_litellm.py index 43d1ed90..124d5ee4 100644 --- a/test/scripts/model/llm/test_litellm.py +++ b/test/scripts/model/llm/test_litellm.py @@ -26,12 +26,12 @@ class _Provider(BaseLiteLLM): return _Provider -def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop"): +def _stub_response(content='{"score": 1, "reason": "ok"}', finish_reason="stop", usage=None): choice = SimpleNamespace( finish_reason=finish_reason, message=SimpleNamespace(content=content), ) - return SimpleNamespace(choices=[choice]) + return SimpleNamespace(choices=[choice], usage=usage) # --------------------------------------------------------------------------- @@ -127,7 +127,25 @@ def test_none_content_returns_empty_string(self): none_resp = _stub_response(content=None) with mock.patch("litellm.completion", return_value=none_resp): result = P.send_messages([{"role": "user", "content": "hi"}]) - assert result == "" + assert result.content == "" + + def test_returns_token_usage_when_provider_supplies_usage(self): + P = _make_provider(model="gpt-4o") + provider_resp = _stub_response( + usage={ + "prompt_tokens": 6, + "completion_tokens": 3, + "total_tokens": 9, + } + ) + with mock.patch("litellm.completion", return_value=provider_resp): + result = P.send_messages([{"role": "user", "content": "hi"}]) + + assert result.content == '{"score": 1, "reason": "ok"}' + assert result.usage.prompt_tokens == 6 + assert result.usage.completion_tokens == 3 + assert result.usage.total_tokens == 9 + assert result.usage.provider == "litellm" # --------------------------------------------------------------------------- diff --git a/test/scripts/model/llm/test_llm_custom_metric.py b/test/scripts/model/llm/test_llm_custom_metric.py index 5eeefdf7..5665bf3d 100644 --- a/test/scripts/model/llm/test_llm_custom_metric.py +++ b/test/scripts/model/llm/test_llm_custom_metric.py @@ -3,6 +3,8 @@ from dingo.config.input_args import EvaluatorLLMArgs, InputArgs from dingo.io.input import Data +from dingo.io.output.eval_detail import TokenUsage +from dingo.model.llm.base import LLMCallResult from dingo.model.llm.llm_custom_metric import LLMCustomMetric from dingo.model.model import Model @@ -197,6 +199,42 @@ def test_eval_detail_response_uses_llm_returned_fields(): assert result.reason == ["The content contains AI-style phrasing."] +def test_eval_detail_response_attaches_token_usage(): + llm = LLMCustomMetric() + Model.set_config_llm( + llm, EvaluatorLLMArgs(custom_metric=_custom_metric(metric="SourceLabel")) + ) + llm.create_client = Mock() + llm.send_messages = Mock( + return_value=LLMCallResult( + content=json.dumps( + { + "status": False, + "label": ["SOURCE.AI_GENERATED"], + "score": 0.82, + "reason": ["The content contains AI-style phrasing."], + } + ), + usage=TokenUsage( + prompt_tokens=12, + completion_tokens=5, + total_tokens=17, + model="gpt-test", + provider="openai", + ), + ) + ) + + result = llm.eval( + Data(prompt="Classify source", content="As an AI language model...") + ) + + assert result.usage is not None + assert result.usage.prompt_tokens == 12 + assert result.usage.completion_tokens == 5 + assert result.usage.total_tokens == 17 + + def test_eval_detail_response_rejects_missing_fields(): llm = LLMCustomMetric() Model.set_config_llm( diff --git a/test/scripts/model/llm/test_token_usage.py b/test/scripts/model/llm/test_token_usage.py new file mode 100644 index 00000000..aea2779e --- /dev/null +++ b/test/scripts/model/llm/test_token_usage.py @@ -0,0 +1,156 @@ +from types import SimpleNamespace + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io import ResultInfo +from dingo.io.input import Data +from dingo.io.output.eval_detail import EvalDetail +from dingo.model.llm.base import LLMCallResult +from dingo.model.llm.base_openai import BaseOpenAI + + +def _completion( + content='{"score": 1, "reason": "ok"}', + usage=None, + finish_reason="stop", +): + return SimpleNamespace( + choices=[ + SimpleNamespace( + finish_reason=finish_reason, + message=SimpleNamespace(content=content), + ) + ], + usage=usage, + ) + + +def test_extract_token_usage_from_openai_response_object(): + usage = SimpleNamespace( + prompt_tokens=11, + completion_tokens=7, + total_tokens=18, + prompt_tokens_details=SimpleNamespace(cached_tokens=3), + completion_tokens_details=SimpleNamespace(reasoning_tokens=2), + ) + + result = BaseOpenAI._extract_token_usage( + _completion(usage=usage), + model_name="gpt-test", + provider="openai", + ) + + assert result.prompt_tokens == 11 + assert result.completion_tokens == 7 + assert result.total_tokens == 18 + assert result.cached_tokens == 3 + assert result.reasoning_tokens == 2 + assert result.model == "gpt-test" + assert result.provider == "openai" + assert result.calls == 1 + + +def test_base_openai_eval_attaches_token_usage(): + class UsageLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return LLMCallResult( + content='{"score": 1, "reason": "ok"}', + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 5, + "completion_tokens": 2, + "total_tokens": 7, + } + ), + model_name="gpt-test", + ), + ) + + result = UsageLLM.eval(Data(content="sample")) + + assert result.status is False + assert result.usage is not None + assert result.usage.prompt_tokens == 5 + assert result.usage.completion_tokens == 2 + assert result.usage.total_tokens == 7 + + +def test_base_openai_error_result_keeps_token_usage(): + class ParseErrorLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return LLMCallResult( + content="not json", + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 3, + "completion_tokens": 1, + "total_tokens": 4, + } + ), + model_name="gpt-test", + ), + ) + + result = ParseErrorLLM.eval(Data(content="sample")) + + assert result.status is True + assert result.label == ["QUALITY_BAD.ConvertJsonError"] + assert result.usage is not None + assert result.usage.total_tokens == 4 + + +def test_base_openai_eval_still_accepts_legacy_string_send_messages(): + class LegacyLLM(BaseOpenAI): + prompt = "" + dynamic_config = EvaluatorLLMArgs() + client = True + + @classmethod + def send_messages(cls, messages): + return '{"score": 1, "reason": "ok"}' + + result = LegacyLLM.eval(Data(content="sample")) + + assert result.status is False + assert result.usage is None + + +def test_result_info_only_serializes_usage_when_present(): + with_usage = ResultInfo( + dingo_id="1", + eval_details={ + "content": [ + EvalDetail( + metric="LLMMetric", + usage=BaseOpenAI._extract_token_usage( + _completion( + usage={ + "prompt_tokens": 1, + "completion_tokens": 2, + "total_tokens": 3, + } + ), + model_name="gpt-test", + ), + ) + ] + }, + ).to_dict() + without_usage = ResultInfo( + dingo_id="2", + eval_details={"content": [EvalDetail(metric="RuleMetric")]}, + ).to_dict() + + assert with_usage["eval_details"]["content"][0]["usage"]["total_tokens"] == 3 + assert "usage" not in without_usage["eval_details"]["content"][0] From abf2222621ceae2923ce2db3bc3389348be31344 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 29 Jul 2026 03:15:42 +0000 Subject: [PATCH 45/80] =?UTF-8?q?=F0=9F=93=9A=20Auto-update=20metrics=20do?= =?UTF-8?q?cumentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 156 ++++++++++++++++++++++++++---------------------- 1 file changed, 84 insertions(+), 72 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index da990a9c..f339d264 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -8,11 +8,11 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | -| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGAnswerRelevancy` | LLMRAGAnswerRelevancy | 评估答案是否直接回答问题，检测无关和冗余信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextPrecision` | LLMRAGContextPrecision | 评估检索上下文的精确度，包括相关性和排序质量 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRecall` | LLMRAGContextRecall | 评估检索上下文的完整性，判断上下文是否能支持答案中的所有陈述 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGContextRelevancy` | LLMRAGContextRelevancy | 评估检索上下文与问题的相关性，检测噪声信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | +| `LLMRAGFaithfulness` | LLMRAGFaithfulness | 评估生成答案是否忠实于给定上下文，检测幻觉和编造信息 | [RAGAS: Automated Evaluation of Retrieval Augmented Generation](https://arxiv.org/abs/2309.15217) | N/A | [📝 View Example](../examples/rag/dataset_rag_eval_baseline.py) | ### Pretrain Text Quality Assessment Metrics @@ -25,10 +25,10 @@ This document provides comprehensive information about all quality metrics used | `LLMMathCompare` | LLMMathCompare | Compares the effectiveness of two tools in extracting mathematical formulas from HTML to Markdown format by evaluatin... | Internal Implementation | N/A | N/A | | `LLMSecurityPolitics` | LLMSecurityPolitics | Evaluates whether the text contains politics-related content | Internal Implementation | N/A | N/A | | `LLMTableCompare` | LLMTableCompare | Compares the effectiveness of two tools in extracting tables from HTML to Markdown format by evaluating recognition r... | Internal Implementation | N/A | N/A | -| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | -| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | -| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextEquation` | LLMTextEquation | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextQualityV4` | LLMTextQualityV4 | Enhanced text quality evaluation covering completeness (formulas, tables, code), effectiveness (garbled text, spacing... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | N/A | +| `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +| `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | ### SFT Data Assessment Metrics @@ -36,82 +36,37 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `LLMFactCheckPublic` | LLMFactCheckPublic | Two-stage factuality evaluation pipeline from GPT-5 | [GPT-5 System Card](https://cdn.openai.com/pdf/8124a3ce-ab78-4f06-96eb-49ea29ffb52f/gpt5-system-card-aug7.pdf) (OpenAI) | N/A | N/A | | `LLMHallucination` | LLMHallucination | Evaluates whether the response contains factual contradictions or hallucinations against provided context information | [TruthfulQA: Measuring How Models Mimic Human Falsehoods](https://arxiv.org/abs/2109.07958) (Lin et al., 2021) | N/A | N/A | -| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [See Results](Returns clarity score (0-10) and detailed analysis) | [View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [See Results](Returns difficulty level (1-10) with detailed breakdown) | [View Example](../examples/sft/evaluate_instruction_quality.py) | -| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMInstructionClarity` | LLMInstructionClarity | Evaluates instruction clarity across four dimensions: self-descriptiveness, consistency, specificity, and completeness | Internal Implementation | [📊 See Results](Returns clarity score (0-10) and detailed analysis) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMTaskDifficulty` | LLMTaskDifficulty | Evaluates task difficulty across cognitive complexity, step complexity, domain knowledge, and constraint density | Internal Implementation | [📊 See Results](Returns difficulty level (1-10) with detailed breakdown) | [📝 View Example](../examples/sft/evaluate_instruction_quality.py) | +| `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | +| `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A | ### Classification Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | +| `LLMClassifyTopic` | LLMClassifyTopic | Classifies text into categories like language processing, writing, code, mathematics, role-play, or knowledge Q&A. Ba... | [BERTopic](https://maartengr.github.io/BERTopic/index.html#quick-start) & [INSTAG](https://arxiv.org/pdf/2308.07074) (Grootendorst, 2022; Wei et al., 2023) | [📊 See Results](eval/prompt/text_data_classified_by_topic.md) | N/A | ### Multimodality Assessment Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `LLMClassifyQR` | LLMClassifyQR | Identifies images as CAPTCHA, QR code, or normal images | Internal Implementation | N/A | N/A | -| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用 DeepSeek-OCR 作为 Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [See Results](通过对比 VLM 输出与 OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | - -### TC609-5-2025-04 Quality Evaluation Metrics - -| Type | Rule | Coverage | Group | Description | -|---|---|---|---|---| -| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | -| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | -| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | -| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | covered | `guobiao_doc` | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | partial | `guobiao_data` | Combines existing NLP, SFT, image, audio, and video format rules. | -| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | partial | `guobiao_data` | Combines unsafe-word, PII, and identity-card detection. | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | partial | `guobiao_data` | Combines image-label overlap and visualization checks. | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | partial | `guobiao_data` | Combines null-content and short-content checks. | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | partial | `guobiao_data` | Uses HHEM consistency checking as partial evidence of authenticity. | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | partial | `guobiao_data` | Combines structured-field and image-text consistency checks. | -| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | partial | `guobiao_data` | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | -| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | covered | `pretrain,guobiao_text` | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | -| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | partial | `guobiao_text` | Combines alphabetic-word, stop-word, and unique-word ratio checks. | -| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | covered | `guobiao_text` | Combines document-text and formula repetition checks. | -| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | covered | `guobiao_text` | Combines null, short, ellipsis-ending, and terminal-ending checks. | -| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | partial | `guobiao_text` | Uses content length and sentence/word counts as partial missing-information checks. | -| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | partial | `guobiao_text` | Combines abnormal HTML, character, invisible-content, and watermark checks. | -| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | partial | `guobiao_text` | Combines punctuation, word-boundary, and line-break fluency checks. | -| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | partial | `guobiao_image` | Uses image aspect-ratio validation as partial resolution coverage. | -| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | covered | `guobiao_image` | Uses PHash and CNN duplicate-image detection. | -| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | partial | `guobiao_image` | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | -| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | partial | `guobiao_image` | Combines image validity and NIMA quality as partial clarity coverage. | -| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | uncovered | `guobiao_video` | Placeholder: video resolution is not implemented. | -| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | uncovered | `guobiao_video` | Placeholder: duplicate-video detection is not implemented. | -| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | uncovered | `guobiao_video` | Placeholder: video FPS validation is not implemented. | -| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | uncovered | `guobiao_video` | Placeholder: video duration validation is not implemented. | -| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | uncovered | `guobiao_video` | Placeholder: video clarity evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | uncovered | `guobiao_video` | Placeholder: video dynamic-range evaluation is not implemented. | -| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | covered | `guobiao_audio` | Uses the existing Welch power-spectrum SNR implementation. | -| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | uncovered | `guobiao_audio` | Placeholder: signal distortion ratio is not implemented. | -| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | uncovered | `guobiao_audio` | Placeholder: sample-rate quality validation is not implemented. | -| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | uncovered | `guobiao_audio` | Placeholder: audio bit-depth validation is not implemented. | -| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | uncovered | `guobiao_audio` | Placeholder: audio bit-rate validation is not implemented. | -| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | covered | `guobiao_audio` | Uses the existing WAV duration implementation. | -| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | partial | `guobiao_data` | Combines available text cleanliness checks; modality coverage is partial. | -| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | uncovered | `guobiao_model` | Placeholder: target-scenario distribution coverage is not implemented. | -| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | uncovered | `guobiao_model` | Placeholder: dataset scale versus model requirements is not implemented. | -| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | covered | `guobiao_model` | Checks whether created and updated timestamps are within configured time ranges | -| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | partial | `guobiao_model` | Uses image annotation checks as partial evidence of annotation accuracy. | -| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | uncovered | `guobiao_model` | Placeholder: before/after model performance comparison is not implemented. | +| `VLMOCRUnderstanding` | VLMOCRUnderstanding | 评估多模态模型对图片中文字内容的识别和理解能力，使用DeepSeek-OCR作为Ground Truth | [DeepSeek-OCR: Contexts Optical Compression](https://github.com/deepseek-ai/DeepSeek-OCR) | [📊 See Results](通过对比VLM输出与OCR ground truth，识别文字遗漏、错误、幻觉等问题) | N/A | ### Rule-Based TEXT Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks text-ending, sentence-count, and word-count completeness. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; detects abnormal character and word splitting. | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023); High-quality dataset quality evaluation specification (SAC/TC609) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | -| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_COMPLETENESS` | RuleLineEndWithEllipsis, RuleLineEndWithTerminal, RuleSentenceNumber, RuleWordNumber | Checks whether the ratio of lines ending with ellipsis is below threshold; Checks whether the ratio of lines ending w... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleAbnormalChar, RuleAbnormalHtml, RuleAlphaWords, RuleAudioDataFormat, RuleCharNumber, RuleColonEnd, RuleContentNull, RuleContentShort, RuleContentShortMultiLan, RuleEnterAndSpace, RuleEnterMore, RuleEnterRatioMore, RuleHtmlEntity, RuleHtmlTag, RuleInvisibleChar, RuleImageDataFormat, RuleLatexSpecialChar, RuleLineJavascriptCount, RuleLoremIpsum, RuleMeanWordLength, RuleNlpDataFormat, RuleSftDataFormat, RuleSpaceMore, RuleSpecialCharacter, RuleStopWord, RuleSymbolWordRatio, RuleVedioDataFormat, RuleOnlyUrl, RuleDictConsistency, RuleDoi, RuleIsbn | Detects garbled text and anti-crawling characters by combining special character and invisible character detection; D... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_FLUENCY` | RuleAbnormalNumber, RuleCharSplit, RuleNoPunc, RuleWordSplit, RuleWordStuck | Checks PDF content for abnormal book page or index numbers that disrupt text flow; Checks PDF content for abnormal ch... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_RELEVANCE` | RuleHeadWordAr, RuleHeadWordCs, RuleHeadWordHu, RuleHeadWordKo, RuleHeadWordRu, RuleHeadWordSr, RuleHeadWordTh, RuleHeadWordVi, RulePatternSearch, RuleWatermark | Checks whether Arabic content contains irrelevant tail source information; Checks whether Czech content contains irre... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SECURITY` | RuleIDCard, RuleUnsafeWords, RulePIIDetection | Checks whether content contains ID card information; Checks whether content contains unsafe words; Detects Personal I... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_SIMILARITY` | RuleDocRepeat, RuleDocFormulaRepeat | Evaluates text for consecutive repeated content and multiple occurrences of special characters; Evaluates text for co... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | +| `QUALITY_BAD_UNDERSTANDABILITY` | RuleCapitalWords, RuleCurlyBracket, RuleLineStartWithBulletpoint, RuleUniqueWords | Checks whether the ratio of capital words is above threshold, indicating poor readability; Checks whether the ratio o... | [RedPajama: an Open Dataset for Training Large Language Models](https://github.com/togethercomputer/RedPajama-Data) (Together Computer, 2023) | [📊 See Results](eval/rule/slimpajama_data_evaluated_by_rule.md) | N/A | ### Rule-Based IMG Quality Metrics @@ -131,6 +86,12 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_EFFECTIVENESS` | RuleAudioDuration | Check whether the audio duration meets the standard | Internal Implementation | N/A | N/A | | `QUALITY_BAD_EFFECTIVENESS` | RuleAudioSnrQuality | Check whether the audio signal-to-noise ratio meets the standard | Internal Implementation | N/A | N/A | +### Document Quality Assessment Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `LLMAISmell` | LLMAISmell | Detects AI-generated writing patterns in requirement documents across 5 dimensions: hollow truisms, repetition, rainb... | Internal Implementation | N/A | [📝 View Example](../examples/llm_and_rule/llm_local.py) | + ### Job Hunting Strategy Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | @@ -146,18 +107,30 @@ This document provides comprehensive information about all quality metrics used | `LLMMetaRaterReadability` | LLMMetaRaterReadability | Evaluates the clarity and coherence of text using appropriate vocabulary and sentence structures on a 5-point scale | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | | `LLMMetaRaterReasoning` | LLMMetaRaterReasoning | Evaluates the reasoning complexity and logical depth of text content, from simple logical judgments to complex multid... | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | +### National Standard Data Quality Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and s... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | + ### OCR Eval Metric | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | -| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [See Results](error_category and error_label) | N/A | +| `LLMMinerURecognizeQuality` | LLMMinerURecognizeQuality | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | +| `VLMDocumentParsingOCRTrain` | VLMDocumentParsingOCRTrain | Evaluate the quality of mineru recognize | Internal Implementation | [📊 See Results](error_category and error_label) | N/A | ### RAG Retrieved Evidence Chunk Quality Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [View Example](../examples/rag/sdk_chunk_eval.py) | +| `LLMChunkQuality` | LLMChunkQuality | Assesses retrieved citation chunks referenced by LLM answers, detecting start-boundary truncation and duplicated lead... | Internal Implementation | N/A | [📝 View Example](../examples/rag/sdk_chunk_eval.py) | ### Resume Quality Assessment Metrics @@ -171,7 +144,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleQuanliangFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为 0.6; Validate Quanliang metadata fields and report invalid fields | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics @@ -185,6 +158,44 @@ This document provides comprehensive information about all quality metrics used | `RESUME_QUALITY_BAD_PROFESSIONALISM` | RuleResumeEmoji, RuleResumeInformal | Detects emoji usage in resume which reduces professionalism; Detects informal or colloquial expressions in resume | Internal Implementation | N/A | N/A | | `RESUME_QUALITY_BAD_STRUCTURE` | RuleResumeNameMissing, RuleResumeSectionMissing | Checks if resume contains a name in the first 200 characters; Checks if resume contains required sections like educat... | Internal Implementation | N/A | N/A | +### SAC/TC609 High-quality Dataset Metrics + +| Type | Metric | Description | Paper Source | Evaluation Results | Examples | +|------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Combines existing NLP, SFT, image, audio, and video format rules. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Combines image-label overlap and visualization checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Uses HHEM consistency checking as partial evidence of authenticity. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | Uses content length and sentence/word counts as partial missing-information checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | Combines abnormal HTML, character, invisible-content, and watermark checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | Combines punctuation, word-boundary, and line-break fluency checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | Uses image aspect-ratio validation as partial resolution coverage. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | Uses PHash and CNN duplicate-image detection. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | Combines image validity and NIMA quality as partial clarity coverage. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | Placeholder: video resolution is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | Placeholder: duplicate-video detection is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | Placeholder: video FPS validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | Placeholder: video duration validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | Placeholder: video clarity evaluation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | Placeholder: video dynamic-range evaluation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | Uses the existing Welch power-spectrum SNR implementation. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | Placeholder: signal distortion ratio is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | Placeholder: sample-rate quality validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | Placeholder: audio bit-depth validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | Placeholder: audio bit-rate validation is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | Uses the existing WAV duration implementation. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | Combines available text cleanliness checks; modality coverage is partial. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | Placeholder: target-scenario distribution coverage is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | Placeholder: dataset scale versus model requirements is not implemented. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | Uses image annotation checks as partial evidence of annotation accuracy. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | Placeholder: before/after model performance comparison is not implemented. | Internal Implementation | N/A | N/A | + ### SFT Data Assessment Metrics - Agent-Enhanced | Type | Metric | Description | Paper Source | Evaluation Results | Examples | @@ -204,3 +215,4 @@ This document provides comprehensive information about all quality metrics used | `AgentFactCheck` | AgentFactCheck | Agent-based hallucination detection with autonomous web search | Internal Implementation | N/A | N/A | | `ArticleFactChecker` | ArticleFactChecker | Article-level fact checking with autonomous claims extraction and verification | Internal Implementation | N/A | N/A | | `LLMCustomMetric` | LLMCustomMetric | Unified metric for user customization | Internal Implementation | N/A | N/A | + From 9782209baafbef4a014e80d5e5b8a2bd0c4c56bd Mon Sep 17 00:00:00 2001 From: shijin Date: Wed, 29 Jul 2026 16:09:12 +0800 Subject: [PATCH 46/80] feat: Rule_TC609_0201_FormatCompliance --- .../model/rule/guobiao/rule_tc609_quality.py | 95 ++++++++++-- docs/metrics.md | 1 + .../model/rule/test_rule_tc609_quality.py | 145 +++++++++++++++++- 3 files changed, 227 insertions(+), 14 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 00eb66f1..df39e93e 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -150,24 +150,93 @@ class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): @Model.rule_register("QUALITY_BAD_TC609_0201", ["guobiao_data"]) -class Rule_TC609_0201_FormatCompliance(Rule_TC609_Composite): - """0201: Format compliance, partially covered by existing format rules.""" - - component_rules = ( - "dingo.model.rule.rule_common.RuleNlpDataFormat", - "dingo.model.rule.rule_common.RuleSftDataFormat", - "dingo.model.rule.rule_common.RuleImageDataFormat", - "dingo.model.rule.rule_common.RuleAudioDataFormat", - "dingo.model.rule.rule_common.RuleVedioDataFormat", - ) - composition_mode = "any" +class Rule_TC609_0201_FormatCompliance(BaseRule): + """Check whether a data record matches a user-provided field schema.""" + + _supported_types = { + "str": {"expected_type": str, "allow_none": False}, + "int": {"expected_type": int, "allow_none": False}, + "float": {"expected_type": float, "allow_none": False}, + "bool": {"expected_type": bool, "allow_none": False}, + "list": {"expected_type": list, "allow_none": False}, + "dict": {"expected_type": dict, "allow_none": False}, + "Optional[str]": {"expected_type": str, "allow_none": True}, + "Optional[int]": {"expected_type": int, "allow_none": True}, + "Optional[float]": {"expected_type": float, "allow_none": True}, + "Optional[bool]": {"expected_type": bool, "allow_none": True}, + "Optional[list]": {"expected_type": list, "allow_none": True}, + "Optional[dict]": {"expected_type": dict, "allow_none": True}, + } + dynamic_config = EvaluatorRuleArgs(field_schema=None) _metric_info = _tc609_metric_info( "0201", "Rule_TC609_0201_FormatCompliance", - "Combines existing NLP, SFT, image, audio, and video format rules.", - "partial", + "Checks required fields and their types against a user-provided schema.", + "covered", ) + @classmethod + def _validate_schema(cls, schema): + if not isinstance(schema, dict) or not schema: + raise ValueError( + "Rule_TC609_0201_FormatCompliance requires a non-empty " + "dynamic_config.field_schema" + ) + + for field_name, type_name in schema.items(): + if not isinstance(field_name, str) or not field_name: + raise ValueError( + "Rule_TC609_0201_FormatCompliance schema field names " + "must be non-empty strings" + ) + if ( + not isinstance(type_name, str) + or type_name not in cls._supported_types + ): + supported = sorted(cls._supported_types.keys()) + raise ValueError( + f"Unsupported schema type for field {field_name!r}: " + f"{type_name!r}. Supported types: {', '.join(supported)}" + ) + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + schema = getattr(cls.dynamic_config, "field_schema", None) + cls._validate_schema(schema) + + record = input_data.model_dump() + res = EvalDetail(metric=cls.__name__) + reasons = [] + for field_name, type_name in schema.items(): + expected_type = cls._supported_types[type_name]["expected_type"] + allow_none = cls._supported_types[type_name]["allow_none"] + if field_name not in record: + reasons.append(f"{field_name}: required field is missing") + res.status = True + continue + + value = record[field_name] + if allow_none and value is None: + continue + + if type(value) is not expected_type: + reasons.append( + f"{field_name}: expected {type_name}, " + f"got {type(value).__name__}" + ) + res.status = True + + for field_name in sorted(record.keys() - schema.keys()): + reasons.append(f"{field_name}: unexpected field") + res.status = True + + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res + @Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao_data"]) class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): diff --git a/docs/metrics.md b/docs/metrics.md index f339d264..0e200690 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -115,6 +115,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Requires record fields to exactly match the configured `field_schema` and checks their types. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and their nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 927150b3..20f73401 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -2,11 +2,16 @@ import pytest +from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.guobiao import rule_tc609_quality -from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0202_SafetyCompliance, Rule_TC609_0301_ContentDiversity +from dingo.model.rule.guobiao.rule_tc609_quality import ( + Rule_TC609_0201_FormatCompliance, + Rule_TC609_0202_SafetyCompliance, + Rule_TC609_0301_ContentDiversity, +) def test_tc609_quality_defines_all_standard_metrics(): @@ -61,6 +66,144 @@ def test_tc609_rules_are_grouped_by_evaluation_object(): ) +def test_format_compliance_accepts_matching_record(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs( + field_schema={ + "data_id": "str", + "content": "str", + "type": "str", + "dt": "str", + } + ), + ) + + result = Rule_TC609_0201_FormatCompliance.eval( + Data( + data_id="demo-001", + content="example", + type="medical", + dt="2026-07-20 09:00:00", + ) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_format_compliance_reports_missing_and_wrong_type(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs( + field_schema={ + "data_id": "str", + "content": "str", + "dt": "str", + } + ), + ) + + result = Rule_TC609_0201_FormatCompliance.eval( + Data(data_id=1, content="example") + ) + + assert result.status is True + assert result.label == [ + "QUALITY_BAD_TC609_0201.Rule_TC609_0201_FormatCompliance" + ] + assert result.reason == [ + "data_id: expected str, got int", + "dt: required field is missing", + ] + + +def test_format_compliance_reports_unexpected_fields(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs(field_schema={"content": "str"}), + ) + + result = Rule_TC609_0201_FormatCompliance.eval( + Data(content="example", source="demo") + ) + + assert result.status is True + assert result.reason == ["source: unexpected field"] + + +@pytest.mark.parametrize( + "value, type_name", + [ + (None, "Optional[str]"), + ("text", "Optional[str]"), + (None, "Optional[int]"), + (1, "Optional[int]"), + (None, "Optional[float]"), + (1.5, "Optional[float]"), + (None, "Optional[bool]"), + (True, "Optional[bool]"), + (None, "Optional[list]"), + ([], "Optional[list]"), + (None, "Optional[dict]"), + ({}, "Optional[dict]"), + ], +) +def test_format_compliance_accepts_optional_types( + monkeypatch, value, type_name +): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs(field_schema={"value": type_name}), + ) + + result = Rule_TC609_0201_FormatCompliance.eval(Data(value=value)) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_format_compliance_still_requires_optional_field(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs(field_schema={"value": "Optional[str]"}), + ) + + result = Rule_TC609_0201_FormatCompliance.eval(Data()) + + assert result.status is True + assert result.reason == ["value: required field is missing"] + +@pytest.mark.parametrize( + "schema, error", + [ + (None, "requires a non-empty dynamic_config.field_schema"), + ({}, "requires a non-empty dynamic_config.field_schema"), + ({"content": "string"}, "Unsupported schema type"), + ({"content": {"type": "str"}}, "Unsupported schema type"), + ({"content": "optional_string"}, "Unsupported schema type"), + ({"content": "optional_str"}, "Unsupported schema type"), + ({"content": "optional[str]"}, "Unsupported schema type"), + ({"content": "typing.Optional[str]"}, "Unsupported schema type"), + ({"content": "optianal[str]"}, "Unsupported schema type"), + ], +) +def test_format_compliance_rejects_invalid_schema(monkeypatch, schema, error): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs(field_schema=schema), + ) + + with pytest.raises(ValueError, match=error): + Rule_TC609_0201_FormatCompliance.eval(Data(content="example")) + + def test_composite_rule_maps_component_failure_to_tc609_label(monkeypatch): class PassingRule: @classmethod From b0da972ea278f043648763f16ac3996d4a26f8e9 Mon Sep 17 00:00:00 2001 From: shijin Date: Wed, 29 Jul 2026 16:25:43 +0800 Subject: [PATCH 47/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0201=5FFormatCompl?= =?UTF-8?q?iance=E6=94=AF=E6=8C=81allow=5Fextra?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 12 +++++++---- docs/metrics.md | 3 +-- .../model/rule/test_rule_tc609_quality.py | 20 ++++++++++++++++++- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index df39e93e..c3866dd4 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -167,7 +167,10 @@ class Rule_TC609_0201_FormatCompliance(BaseRule): "Optional[list]": {"expected_type": list, "allow_none": True}, "Optional[dict]": {"expected_type": dict, "allow_none": True}, } - dynamic_config = EvaluatorRuleArgs(field_schema=None) + dynamic_config = EvaluatorRuleArgs( + field_schema=None, + allow_extra=True, + ) _metric_info = _tc609_metric_info( "0201", "Rule_TC609_0201_FormatCompliance", @@ -226,9 +229,10 @@ def eval(cls, input_data: Data) -> EvalDetail: ) res.status = True - for field_name in sorted(record.keys() - schema.keys()): - reasons.append(f"{field_name}: unexpected field") - res.status = True + if not getattr(cls.dynamic_config, "allow_extra", True): + for field_name in sorted(record.keys() - schema.keys()): + reasons.append(f"{field_name}: unexpected field") + res.status = True if res.status: res.label = [f"{cls.metric_type}.{cls.__name__}"] diff --git a/docs/metrics.md b/docs/metrics.md index 0e200690..f623c69c 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -115,7 +115,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Requires record fields to exactly match the configured `field_schema` and checks their types. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and their nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | @@ -163,7 +163,6 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Combines existing NLP, SFT, image, audio, and video format rules. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Combines image-label overlap and visualization checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 20f73401..0fdbdd2b 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -124,7 +124,10 @@ def test_format_compliance_reports_unexpected_fields(monkeypatch): monkeypatch.setattr( Rule_TC609_0201_FormatCompliance, "dynamic_config", - EvaluatorRuleArgs(field_schema={"content": "str"}), + EvaluatorRuleArgs( + field_schema={"content": "str"}, + allow_extra=False, + ), ) result = Rule_TC609_0201_FormatCompliance.eval( @@ -135,6 +138,21 @@ def test_format_compliance_reports_unexpected_fields(monkeypatch): assert result.reason == ["source: unexpected field"] +def test_format_compliance_allows_unexpected_fields_by_default(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0201_FormatCompliance, + "dynamic_config", + EvaluatorRuleArgs(field_schema={"content": "str"}), + ) + + result = Rule_TC609_0201_FormatCompliance.eval( + Data(content="example", source="demo") + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + @pytest.mark.parametrize( "value, type_name", [ From 9f4e782e3e0bda8247269ebc7294d754f47cc61c Mon Sep 17 00:00:00 2001 From: shijin Date: Wed, 29 Jul 2026 19:14:09 +0800 Subject: [PATCH 48/80] =?UTF-8?q?feat:=20RuleUnsafeWords=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 13 ++++ dingo/model/rule/rule_common.py | 40 +++++++--- test/scripts/model/rule/test_rule_common.py | 74 ++++++++++++++++++- .../model/rule/test_rule_tc609_quality.py | 53 +++++++++++++ 4 files changed, 166 insertions(+), 14 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index c3866dd4..9ae0262d 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -246,6 +246,10 @@ def eval(cls, input_data: Data) -> EvalDetail: class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): """0202: Safety compliance, composed from safety and PII rules.""" + dynamic_config = EvaluatorRuleArgs( + key_list=[], + refer_path=[], + ) component_rules = ( "dingo.model.rule.rule_common.RuleUnsafeWords", "dingo.model.rule.rule_common.RulePIIDetection", @@ -259,6 +263,15 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): "partial", ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + rule_unsafe_words = cls._resolve_rule(cls.component_rules[0]) + rule_unsafe_words.dynamic_config = EvaluatorRuleArgs( + key_list=cls.dynamic_config.key_list or [], + refer_path=cls.dynamic_config.refer_path or [], + ) + return super().eval(input_data) + @Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"]) class Rule_TC609_0203_AnnotationCompliance(Rule_TC609_Composite): diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 4ab70635..00229dbb 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -2108,7 +2108,12 @@ class RuleUnsafeWords(BaseRule): "paper_authors": "Together Computer, 2023", "evaluation_results": "docs/eval/rule/slimpajama_data_evaluated_by_rule.md" } - dynamic_config = EvaluatorRuleArgs(refer_path=[]) + dynamic_config = EvaluatorRuleArgs( + key_list=[], + refer_path=[], + ) + _unsafe_words_list = None + _unsafe_words_automaton = None _required_fields = [RequiredField.CONTENT] @@ -2121,17 +2126,32 @@ def eval(cls, input_data: Data) -> EvalDetail: res = EvalDetail(metric=cls.__name__) content = input_data.content - key_list = cls.dynamic_config.key_list - if key_list is None: - key_list = get_unsafe_words(cls.dynamic_config.refer_path) - - A = ahocorasick.Automaton() - for index, key in enumerate(key_list): - A.add_word(key, (index, key)) - A.make_automaton() + if cls._unsafe_words_list is None: + unsafe_words_list = [] + unsafe_words_list.extend(cls.dynamic_config.key_list or []) + unsafe_words_list.extend(get_unsafe_words(cls.dynamic_config.refer_path or [])) + unsafe_words_list = list(dict.fromkeys( + word for word in unsafe_words_list + if isinstance(word, str) and word + )) + if not unsafe_words_list: + raise ValueError( + "RuleUnsafeWords requires unsafe words from dynamic_config." + "key_list or files configured in dynamic_config.refer_path" + ) + cls._unsafe_words_list = unsafe_words_list + + if cls._unsafe_words_automaton is None: + automaton = ahocorasick.Automaton() + for index, key in enumerate(cls._unsafe_words_list): + automaton.add_word(key, (index, key)) + automaton.make_automaton() + cls._unsafe_words_automaton = automaton matches = [] - for end_index, (index, keyword) in A.iter(content): + for end_index, (index, keyword) in cls._unsafe_words_automaton.iter( + content + ): start_index = end_index - len(keyword) + 1 # 检查单词边界 diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index eba98916..7b02509e 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -20,16 +20,82 @@ def test_rule_doc_formula_repeat(self): assert res.metric == "RuleDocFormulaRepeat" assert res.reason == ["Formula has too many consecutive repeated characters, total repeat length: 130, found 1 repeat patterns"] - def test_rule_unsafe_words(self): + def test_rule_unsafe_words(self, monkeypatch): data = Data(data_id="", prompt="", content="java is good\n \n \n \n hello \n \n but python is better") - r = RuleUnsafeWords - r.dynamic_config.key_list = ['av', 'b', 'java'] - tmp = r.eval(data) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) + monkeypatch.setattr( + RuleUnsafeWords, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["av", "b", "java"], + refer_path=[], + ), + ) + tmp = RuleUnsafeWords.eval(data) assert tmp.status is True assert 'av' not in tmp.reason assert 'b' not in tmp.reason assert 'java' in tmp.reason + def test_rule_unsafe_words_requires_configured_words(self, monkeypatch): + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) + monkeypatch.setattr( + RuleUnsafeWords, + "dynamic_config", + EvaluatorRuleArgs(key_list=[], refer_path=[]), + ) + + with pytest.raises( + ValueError, + match="key_list.*refer_path", + ): + RuleUnsafeWords.eval(Data(content="safe text")) + + def test_rule_unsafe_words_combines_key_list_and_refer_path( + self, monkeypatch, tmp_path + ): + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) + unsafe_words_file = tmp_path / "unsafe_words.jsonl" + unsafe_words_file.write_text( + '{"word": "python"}\n', + encoding="utf-8", + ) + monkeypatch.setattr( + RuleUnsafeWords, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["java"], + refer_path=[str(unsafe_words_file)], + ), + ) + + result = RuleUnsafeWords.eval( + Data(content="java and python are programming languages") + ) + + assert result.status is True + assert result.reason == ["java", "python"] + + def test_rule_unsafe_words_reuses_cached_automaton(self, monkeypatch): + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) + monkeypatch.setattr( + RuleUnsafeWords, + "dynamic_config", + EvaluatorRuleArgs(key_list=["java"], refer_path=[]), + ) + + RuleUnsafeWords.eval(Data(content="java")) + cached_words = RuleUnsafeWords._unsafe_words_list + cached_automaton = RuleUnsafeWords._unsafe_words_automaton + RuleUnsafeWords.eval(Data(content="java")) + + assert RuleUnsafeWords._unsafe_words_list is cached_words + assert RuleUnsafeWords._unsafe_words_automaton is cached_automaton + class TestRule_TC609_02080101_TextPerplexity: @staticmethod diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 0fdbdd2b..d8512a92 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -263,6 +263,59 @@ def eval(cls, input_data): assert result.reason == ["FailingRule: component failed"] +def test_safety_compliance_passes_words_config_to_unsafe_rule(monkeypatch): + class UnsafeWordsRule: + dynamic_config = EvaluatorRuleArgs() + _unsafe_words_list = None + _unsafe_words_automaton = None + + @classmethod + def eval(cls, input_data): + if "unsafe" in cls.dynamic_config.key_list: + return EvalDetail( + metric=cls.__name__, + status=True, + label=["QUALITY_BAD_SECURITY.UnsafeWordsRule"], + reason=["unsafe"], + ) + return EvalDetail( + metric=cls.__name__, + label=[QualityLabel.QUALITY_GOOD], + ) + + class PassingRule: + @classmethod + def eval(cls, input_data): + return EvalDetail( + metric=cls.__name__, + label=[QualityLabel.QUALITY_GOOD], + ) + + component_map = { + Rule_TC609_0202_SafetyCompliance.component_rules[0]: UnsafeWordsRule, + Rule_TC609_0202_SafetyCompliance.component_rules[1]: PassingRule, + Rule_TC609_0202_SafetyCompliance.component_rules[2]: PassingRule, + } + monkeypatch.setattr( + Rule_TC609_0202_SafetyCompliance, + "_resolve_rule", + classmethod(lambda cls, path: component_map[path]), + ) + monkeypatch.setattr( + Rule_TC609_0202_SafetyCompliance, + "dynamic_config", + EvaluatorRuleArgs(key_list=["unsafe"], refer_path=[]), + ) + + result = Rule_TC609_0202_SafetyCompliance.eval( + Data(content="unsafe") + ) + + assert UnsafeWordsRule.dynamic_config.key_list == ["unsafe"] + assert result.status is True + assert result.reason == ["UnsafeWordsRule: unsafe"] + + def test_uncovered_rule_is_explicit_placeholder(): assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): From d5405b1f50bcbb1ec4438447e80e834025494db1 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 11:07:09 +0800 Subject: [PATCH 49/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99Rule=5FTC609=5F?= =?UTF-8?q?0203=5FAnnotationCompliance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 36 ++++++++++---- docs/metrics.md | 2 +- .../model/rule/test_rule_tc609_quality.py | 47 +++++++++++++++++++ 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 9ae0262d..bed228d0 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -274,21 +274,39 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"]) -class Rule_TC609_0203_AnnotationCompliance(Rule_TC609_Composite): - """0203: Annotation compliance, partially covered by image label rules.""" +class Rule_TC609_0203_AnnotationCompliance(BaseRule): + """Check whether content is one of the configured annotation values.""" - component_rules = ( - "dingo.model.rule.rule_image.RuleImageLabelOverlap", - "dingo.model.rule.rule_image.RuleImageLabelVisualization", - ) - _required_fields = [RequiredField.IMAGE] + dynamic_config = EvaluatorRuleArgs(key_list=[]) + _required_fields = [RequiredField.CONTENT] _metric_info = _tc609_metric_info( "0203", "Rule_TC609_0203_AnnotationCompliance", - "Combines image-label overlap and visualization checks.", - "partial", + "Checks whether content belongs to a user-provided annotation value list.", + "covered", ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + allowed_values = cls.dynamic_config.key_list or [] + if not allowed_values: + raise ValueError( + "Rule_TC609_0203_AnnotationCompliance requires a non-empty " + "dynamic_config.key_list" + ) + + res = EvalDetail(metric=cls.__name__) + content = getattr(input_data, "content", None) + if content not in allowed_values: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"content: value {content!r} is not in dynamic_config.key_list" + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res + @Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"]) class Rule_TC609_0204_StructuralCompleteness(Rule_TC609_Composite): diff --git a/docs/metrics.md b/docs/metrics.md index f623c69c..b73f877a 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -164,7 +164,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Combines image-label overlap and visualization checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether `content` belongs to the annotation values configured in `key_list`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Uses HHEM consistency checking as partial evidence of authenticity. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index d8512a92..e0d45ba3 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -10,6 +10,7 @@ from dingo.model.rule.guobiao.rule_tc609_quality import ( Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, + Rule_TC609_0203_AnnotationCompliance, Rule_TC609_0301_ContentDiversity, ) @@ -316,6 +317,52 @@ def eval(cls, input_data): assert result.reason == ["UnsafeWordsRule: unsafe"] +def test_annotation_compliance_accepts_allowed_content(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0203_AnnotationCompliance, + "dynamic_config", + EvaluatorRuleArgs(key_list=["positive", "negative"]), + ) + + result = Rule_TC609_0203_AnnotationCompliance.eval( + Data(content="positive") + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_annotation_compliance_rejects_unknown_content(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0203_AnnotationCompliance, + "dynamic_config", + EvaluatorRuleArgs(key_list=["positive", "negative"]), + ) + + result = Rule_TC609_0203_AnnotationCompliance.eval( + Data(content="neutral") + ) + + assert result.status is True + assert result.label == [ + "QUALITY_BAD_TC609_0203.Rule_TC609_0203_AnnotationCompliance" + ] + assert result.reason == [ + "content: value 'neutral' is not in dynamic_config.key_list" + ] + + +def test_annotation_compliance_requires_allowed_values(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0203_AnnotationCompliance, + "dynamic_config", + EvaluatorRuleArgs(key_list=[]), + ) + + with pytest.raises(ValueError, match="non-empty dynamic_config.key_list"): + Rule_TC609_0203_AnnotationCompliance.eval(Data(content="positive")) + + def test_uncovered_rule_is_explicit_placeholder(): assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): From 7c651bebb786548e256b03874c5ba71459101c84 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 14:00:55 +0800 Subject: [PATCH 50/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99Rule=5FTC609=5F?= =?UTF-8?q?0204=5FStructuralCompleteness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 58 ++++++++++++-- docs/metrics.md | 1 + .../model/rule/test_rule_tc609_quality.py | 79 +++++++++++++++++++ 3 files changed, 130 insertions(+), 8 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index bed228d0..9250e760 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -309,21 +309,63 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"]) -class Rule_TC609_0204_StructuralCompleteness(Rule_TC609_Composite): - """0204: Structural completeness, composed from content checks.""" +class Rule_TC609_0204_StructuralCompleteness(BaseRule): + """Check required fields for missing, None, and empty values.""" - component_rules = ( - "dingo.model.rule.rule_common.RuleContentNull", - "dingo.model.rule.rule_common.RuleContentShort", + dynamic_config = EvaluatorRuleArgs( + key_list=[], + allow_none=False, + allow_empty=False, ) - _required_fields = [RequiredField.CONTENT] _metric_info = _tc609_metric_info( "0204", "Rule_TC609_0204_StructuralCompleteness", - "Combines null-content and short-content checks.", - "partial", + "Checks configured fields for missing, None, and empty values.", + "covered", ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + key_list = cls.dynamic_config.key_list or [] + if not key_list: + raise ValueError( + "Rule_TC609_0204_StructuralCompleteness requires a non-empty " + "dynamic_config.key_list" + ) + + record = input_data.model_dump() + allow_none = getattr(cls.dynamic_config, "allow_none", False) + allow_empty = getattr(cls.dynamic_config, "allow_empty", False) + res = EvalDetail(metric=cls.__name__) + reasons = [] + + for field_name in key_list: + if field_name not in record: + reasons.append(f"{field_name}: required field is missing") + res.status = True + continue + + value = record[field_name] + if value is None and not allow_none: + reasons.append(f"{field_name}: None is not allowed") + res.status = True + continue + + if ( + not allow_empty + and isinstance(value, (str, list, dict)) + and len(value) == 0 + ): + reasons.append(f"{field_name}: empty value is not allowed") + res.status = True + + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + else: + res.label = [QualityLabel.QUALITY_GOOD] + return res + @Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao_data"]) class Rule_TC609_0205_ContentAuthenticity(Rule_TC609_Composite): diff --git a/docs/metrics.md b/docs/metrics.md index b73f877a..9fc043c4 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -165,6 +165,7 @@ This document provides comprehensive information about all quality metrics used |------|--------|-------------|--------------|-------------------|----------| | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether `content` belongs to the annotation values configured in `key_list`. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Uses HHEM consistency checking as partial evidence of authenticity. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index e0d45ba3..ce7c2a40 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -11,6 +11,7 @@ Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance, + Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0301_ContentDiversity, ) @@ -363,6 +364,84 @@ def test_annotation_compliance_requires_allowed_values(monkeypatch): Rule_TC609_0203_AnnotationCompliance.eval(Data(content="positive")) +def test_structural_completeness_accepts_present_values(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0204_StructuralCompleteness, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["content", "labels", "metadata"], + allow_none=False, + allow_empty=False, + ), + ) + + result = Rule_TC609_0204_StructuralCompleteness.eval( + Data(content="example", labels=["valid"], metadata={"source": "demo"}) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_structural_completeness_reports_missing_none_and_empty(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0204_StructuralCompleteness, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["missing", "nullable", "text", "items", "metadata"], + allow_none=False, + allow_empty=False, + ), + ) + + result = Rule_TC609_0204_StructuralCompleteness.eval( + Data(nullable=None, text="", items=[], metadata={}) + ) + + assert result.status is True + assert result.reason == [ + "missing: required field is missing", + "nullable: None is not allowed", + "text: empty value is not allowed", + "items: empty value is not allowed", + "metadata: empty value is not allowed", + ] + + +def test_structural_completeness_allows_none_and_empty(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0204_StructuralCompleteness, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["nullable", "text", "items", "metadata"], + allow_none=True, + allow_empty=True, + ), + ) + + result = Rule_TC609_0204_StructuralCompleteness.eval( + Data(nullable=None, text="", items=[], metadata={}) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_structural_completeness_requires_key_list(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0204_StructuralCompleteness, + "dynamic_config", + EvaluatorRuleArgs( + key_list=[], + allow_none=False, + allow_empty=False, + ), + ) + + with pytest.raises(ValueError, match="non-empty dynamic_config.key_list"): + Rule_TC609_0204_StructuralCompleteness.eval(Data(content="example")) + + def test_uncovered_rule_is_explicit_placeholder(): assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): From 3ee72672c9b165218306b3b7a6082d068d34716a Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 15:12:30 +0800 Subject: [PATCH 51/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99Rule=5FTC609=5F?= =?UTF-8?q?0205=5FContentAuthenticity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + .../model/rule/guobiao/rule_tc609_quality.py | 69 ++++++++- docs/metrics.md | 2 +- .../model/rule/test_rule_tc609_quality.py | 136 ++++++++++++++++++ 4 files changed, 200 insertions(+), 8 deletions(-) diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index 869cba47..b94f06ee 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -9,3 +9,4 @@ class RequiredField(Enum): METADATA = "metadata" TYPE = "type" DT = "dt" + SOURCE = "source" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 9250e760..87f0bc95 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -368,19 +368,74 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao_data"]) -class Rule_TC609_0205_ContentAuthenticity(Rule_TC609_Composite): - """0205: Content authenticity, partially covered by HHEM.""" +class Rule_TC609_0205_ContentAuthenticity(BaseRule): + """Check whether a record's HTTP or HTTPS source returns status 200.""" - component_rules = ( - "dingo.model.rule.rule_hallucination_hhem.RuleHallucinationHHEM", - ) + _required_fields = [RequiredField.CONTENT, RequiredField.SOURCE] + dynamic_config = EvaluatorRuleArgs(timeout=10) _metric_info = _tc609_metric_info( "0205", "Rule_TC609_0205_ContentAuthenticity", - "Uses HHEM consistency checking as partial evidence of authenticity.", - "partial", + "Checks whether source is an HTTP or HTTPS URL that returns status 200.", + "covered", ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + source = getattr(input_data, "source", None) + res = EvalDetail(metric=cls.__name__) + if not ( + isinstance(source, str) + and source + and source.lower().startswith(("http://", "https://")) + ): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["source: expected a valid HTTP or HTTPS URL"] + return res + + import requests + + timeout = getattr(cls.dynamic_config, "timeout") + if ( + isinstance(timeout, bool) + or not isinstance(timeout, int) + or timeout <= 0 + ): + raise ValueError( + "Rule_TC609_0205_ContentAuthenticity requires " + "dynamic_config.timeout to be a positive integer" + ) + response = None + try: + response = requests.get( + source, + timeout=timeout, + allow_redirects=True, + stream=True, + ) + if response.status_code != 200: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"source: URL returned HTTP status {response.status_code}, " + "expected 200" + ] + return res + except requests.RequestException as exc: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"source: URL request failed: {type(exc).__name__}: {exc}" + ] + return res + finally: + if response is not None: + response.close() + + res.label = [QualityLabel.QUALITY_GOOD] + return res + @Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"]) class Rule_TC609_0206_ContentConsistency(Rule_TC609_Composite): diff --git a/docs/metrics.md b/docs/metrics.md index 9fc043c4..4a413cba 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -166,8 +166,8 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether `content` belongs to the annotation values configured in `key_list`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `content` and `source`, and checks whether the HTTP or HTTPS `source` returns status 200. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Uses HHEM consistency checking as partial evidence of authenticity. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index ce7c2a40..609d400b 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -4,6 +4,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data +from dingo.io.input import RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.guobiao import rule_tc609_quality @@ -12,6 +13,7 @@ Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance, Rule_TC609_0204_StructuralCompleteness, + Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0301_ContentDiversity, ) @@ -442,6 +444,140 @@ def test_structural_completeness_requires_key_list(monkeypatch): Rule_TC609_0204_StructuralCompleteness.eval(Data(content="example")) +@pytest.mark.parametrize( + "source", + [ + "https://example.com/data/1", + "http://localhost:8080/record?id=1", + ], +) +def test_content_authenticity_accepts_source_returning_200(source, monkeypatch): + class Response: + status_code = 200 + + def close(self): + pass + + monkeypatch.setattr("requests.get", lambda *args, **kwargs: Response()) + result = Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source=source) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_content_authenticity_rejects_source_not_returning_200(monkeypatch): + class Response: + status_code = 404 + + def close(self): + pass + + monkeypatch.setattr("requests.get", lambda *args, **kwargs: Response()) + result = Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source="https://example.com/missing") + ) + + assert result.status is True + assert result.reason == [ + "source: URL returned HTTP status 404, expected 200" + ] + + +def test_content_authenticity_rejects_request_failure(monkeypatch): + import requests + + def raise_timeout(*args, **kwargs): + raise requests.Timeout("timed out") + + monkeypatch.setattr("requests.get", raise_timeout) + result = Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source="https://example.com/slow") + ) + + assert result.status is True + assert result.reason == [ + "source: URL request failed: Timeout: timed out" + ] + + +@pytest.mark.parametrize("timeout", [10.0, "10", 0, -1, True, None]) +def test_content_authenticity_requires_positive_integer_timeout( + timeout, monkeypatch +): + monkeypatch.setattr( + Rule_TC609_0205_ContentAuthenticity, + "dynamic_config", + EvaluatorRuleArgs(timeout=timeout), + ) + + with pytest.raises(ValueError, match="positive integer"): + Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source="https://example.com/data") + ) + + +@pytest.mark.parametrize( + "source", + [ + None, + "", + "example.com/data/1", + "ftp://example.com/data/1", + ], +) +def test_content_authenticity_rejects_source_without_http_prefix(source): + result = Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source=source) + ) + + assert result.status is True + assert result.label == [ + "QUALITY_BAD_TC609_0205.Rule_TC609_0205_ContentAuthenticity" + ] + assert result.reason == [ + "source: expected a valid HTTP or HTTPS URL" + ] + + +@pytest.mark.parametrize( + "source", + [ + "https://", + "https://exa mple.com/data/1", + "https://example.com:invalid/data/1", + ], +) +def test_content_authenticity_handles_prefixed_url_request_failure( + source, monkeypatch +): + import requests + + def raise_invalid_url(*args, **kwargs): + raise requests.exceptions.InvalidURL("failed to parse URL") + + monkeypatch.setattr("requests.get", raise_invalid_url) + result = Rule_TC609_0205_ContentAuthenticity.eval( + Data(content="example", source=source) + ) + + assert result.status is True + assert result.label == [ + "QUALITY_BAD_TC609_0205.Rule_TC609_0205_ContentAuthenticity" + ] + assert result.reason == [ + "source: URL request failed: InvalidURL: failed to parse URL" + ] + + +def test_content_authenticity_declares_required_fields(): + assert Rule_TC609_0205_ContentAuthenticity._required_fields == [ + RequiredField.CONTENT, + RequiredField.SOURCE, + ] + + def test_uncovered_rule_is_explicit_placeholder(): assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): From 1c8e1f0fdae692f56dae0f75cb9a023f493e2d46 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 15:31:17 +0800 Subject: [PATCH 52/80] =?UTF-8?q?feat:=20=E9=87=8D=E5=86=99Rule=5FTC609=5F?= =?UTF-8?q?0206=5FContentConsistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 157 +++++++++++++++++- docs/metrics.md | 2 +- examples/guobiao/rule_content_consistency.py | 36 ++++ .../model/rule/test_rule_tc609_quality.py | 102 ++++++++++++ 4 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 examples/guobiao/rule_content_consistency.py diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 87f0bc95..e5f56fe3 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -438,21 +438,162 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"]) -class Rule_TC609_0206_ContentConsistency(Rule_TC609_Composite): - """0206: Content consistency, composed from dict and image-text checks.""" +class Rule_TC609_0206_ContentConsistency(BaseRule): + """Check semantic consistency among string fields configured in key_list.""" - component_rules = ( - "dingo.model.rule.rule_common.RuleDictConsistency", - "dingo.model.rule.rule_image.RuleImageTextSimilarity", + dynamic_config = EvaluatorRuleArgs( + key_list=[], + threshold=0.5, + model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + device=-1, ) - composition_mode = "any" _metric_info = _tc609_metric_info( "0206", "Rule_TC609_0206_ContentConsistency", - "Combines structured-field and image-text consistency checks.", - "partial", + "Uses a local model to check semantic consistency among configured string fields.", + "covered", ) + _model_name = None + _model_device = None + _classifier = None + + @classmethod + def _get_classifier(cls, model_name, device): + if ( + cls._model_name == model_name + and cls._model_device == device + and cls._classifier is not None + ): + return cls._classifier + + required_packages = ("torch", "transformers") + missing_packages = [ + package + for package in required_packages + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "Rule_TC609_0206_ContentConsistency requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + from transformers import pipeline + + cls._classifier = pipeline( + "zero-shot-classification", + model=model_name, + device=device, + ) + cls._model_name = model_name + cls._model_device = device + return cls._classifier + + @classmethod + def _calculate_consistency_score( + cls, reference, candidate, model_name, device + ): + classifier = cls._get_classifier(model_name, device) + result = classifier( + reference, + candidate_labels=[candidate], + hypothesis_template="这段文本与以下内容语义一致：{}", + multi_label=True, + truncation=True, + ) + labels = result.get("labels", []) + scores = result.get("scores", []) + if not labels or not scores or labels[0] != candidate: + raise RuntimeError("Zero-shot classifier returned an invalid result") + + score = float(scores[0]) + if not math.isfinite(score) or not 0.0 <= score <= 1.0: + raise RuntimeError( + f"Zero-shot classifier returned an invalid score: {score}" + ) + return score + + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + key_list = cls.dynamic_config.key_list + if not isinstance(key_list, list) or len(key_list) < 2: + raise ValueError( + "Rule_TC609_0206_ContentConsistency requires " + "dynamic_config.key_list to contain at least two fields" + ) + if ( + any(not isinstance(key, str) or not key for key in key_list) + or len(set(key_list)) != len(key_list) + ): + raise ValueError( + "Rule_TC609_0206_ContentConsistency requires " + "dynamic_config.key_list to contain unique non-empty strings" + ) + + threshold = cls.dynamic_config.threshold + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not 0 < threshold <= 1 + ): + raise ValueError( + "Rule_TC609_0206_ContentConsistency requires " + "dynamic_config.threshold to be in (0, 1]" + ) + + record = input_data.model_dump() + invalid_fields = [ + field + for field in key_list + if field not in record or not isinstance(record[field], str) + ] + res = EvalDetail(metric=cls.__name__) + if invalid_fields: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"{field}: field must exist and its value must be str" + for field in invalid_fields + ] + return res + + reference_field = key_list[0] + pair_scores = [] + for candidate_field in key_list[1:]: + score = cls._calculate_consistency_score( + record[reference_field], + record[candidate_field], + cls.dynamic_config.model, + cls.dynamic_config.device, + ) + pair_scores.append((candidate_field, score)) + + minimum_score = min(score for _, score in pair_scores) + res.score = minimum_score + inconsistent_pairs = [ + (candidate_field, score) + for candidate_field, score in pair_scores + if score < threshold + ] + if inconsistent_pairs: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + f"{reference_field} and {candidate_field} are inconsistent " + f"(score: {score:.4f}, threshold: {threshold:.4f})" + for candidate_field, score in inconsistent_pairs + ] + else: + res.label = [QualityLabel.QUALITY_GOOD] + res.reason = [ + f"Configured fields are consistent " + f"(minimum score: {minimum_score:.4f}, " + f"threshold: {threshold:.4f})" + ] + return res + @Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao_data"]) class Rule_TC609_0207_DataTypeConsistency(BaseRule): diff --git a/docs/metrics.md b/docs/metrics.md index 4a413cba..022f6246 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -168,7 +168,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `content` and `source`, and checks whether the HTTP or HTTPS `source` returns status 200. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Combines structured-field and image-text consistency checks. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses a local multilingual NLI model to check semantic consistency among string fields configured in `key_list`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | diff --git a/examples/guobiao/rule_content_consistency.py b/examples/guobiao/rule_content_consistency.py new file mode 100644 index 00000000..938f34a6 --- /dev/null +++ b/examples/guobiao/rule_content_consistency.py @@ -0,0 +1,36 @@ +"""Evaluate string fields using the national-standard content-consistency rule. + +Optional dependencies: + conda run -n dingo pip install "dingo-python[hhem]" + +The first run downloads the configured Hugging Face model unless it is already +available locally. +""" + +from dingo.config.input_args import EvaluatorRuleArgs +from dingo.io import Data +from dingo.model.rule.guobiao.rule_tc609_quality import ( + Rule_TC609_0206_ContentConsistency, +) + + +def main(): + data = Data( + data_id="guobiao-content-consistency-example", + title="高血压患者的日常健康管理", + content="高血压患者应遵医嘱规律用药，并定期监测血压。", + summary="高血压患者需要规律服药和监测血压。", + ) + + Rule_TC609_0206_ContentConsistency.dynamic_config = EvaluatorRuleArgs( + key_list=["title", "content", "summary"], + threshold=0.5, + model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + device=-1, + ) + result = Rule_TC609_0206_ContentConsistency.eval(data) + print(result) + + +if __name__ == "__main__": + main() diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 609d400b..11af0b94 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -14,6 +14,7 @@ Rule_TC609_0203_AnnotationCompliance, Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, + Rule_TC609_0206_ContentConsistency, Rule_TC609_0301_ContentDiversity, ) @@ -578,6 +579,107 @@ def test_content_authenticity_declares_required_fields(): ] +def test_content_consistency_accepts_consistent_string_fields(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["title", "content", "summary"], + threshold=0.5, + model="test-model", + device=-1, + ), + ) + scores = iter([0.9, 0.8]) + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "_calculate_consistency_score", + classmethod(lambda cls, *args: next(scores)), + ) + + result = Rule_TC609_0206_ContentConsistency.eval( + Data(title="健康", content="健康知识", summary="健康摘要") + ) + + assert result.status is False + assert result.score == 0.8 + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_content_consistency_rejects_inconsistent_string_fields(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["title", "content"], + threshold=0.5, + model="test-model", + device=-1, + ), + ) + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "_calculate_consistency_score", + classmethod(lambda cls, *args: 0.2), + ) + + result = Rule_TC609_0206_ContentConsistency.eval( + Data(title="健康", content="金融市场") + ) + + assert result.status is True + assert result.score == 0.2 + assert result.reason == [ + "title and content are inconsistent " + "(score: 0.2000, threshold: 0.5000)" + ] + + +@pytest.mark.parametrize( + "data", + [ + Data(title="健康"), + Data(title="健康", content=["健康知识"]), + Data(title="健康", content=None), + ], +) +def test_content_consistency_requires_existing_string_fields(data, monkeypatch): + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "dynamic_config", + EvaluatorRuleArgs( + key_list=["title", "content"], + threshold=0.5, + model="test-model", + device=-1, + ), + ) + + result = Rule_TC609_0206_ContentConsistency.eval(data) + + assert result.status is True + assert result.reason == [ + "content: field must exist and its value must be str" + ] + + +@pytest.mark.parametrize("key_list", [[], ["content"], ["content", "content"]]) +def test_content_consistency_rejects_invalid_key_list(key_list, monkeypatch): + monkeypatch.setattr( + Rule_TC609_0206_ContentConsistency, + "dynamic_config", + EvaluatorRuleArgs( + key_list=key_list, + threshold=0.5, + model="test-model", + device=-1, + ), + ) + + with pytest.raises(ValueError, match="key_list"): + Rule_TC609_0206_ContentConsistency.eval(Data(content="example")) + + def test_uncovered_rule_is_explicit_placeholder(): assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] with pytest.raises(NotImplementedError, match="placeholder"): From 9dfa8b099d638f44cdca24d6ecda4836cc73a429 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 16:12:56 +0800 Subject: [PATCH 53/80] =?UTF-8?q?feat:=20RuleWatermark=E4=BC=98=E5=8C=96?= =?UTF-8?q?=EF=BC=8Ckey-list=E4=B8=BA=E7=A9=BA=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/rule/rule_common.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 00229dbb..2293d60b 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -2294,6 +2294,11 @@ class RuleWatermark(BaseRule): @classmethod def eval(cls, input_data: Data) -> EvalDetail: + if not cls.dynamic_config.key_list: + raise ValueError( + "RuleWatermark requires non-empty dynamic_config.key_list" + ) + res = EvalDetail(metric=cls.__name__) matches = re.findall("|".join(cls.dynamic_config.key_list), input_data.content) if matches: From 69073e0e0786147ed58a26bc8a5445a53c3d91ed Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 16:13:43 +0800 Subject: [PATCH 54/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0208=5FContentClea?= =?UTF-8?q?nliness=20=E4=BC=A0=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 9 ++++ .../rule/guobiao/rule_tc609_quality_base.py | 4 ++ test/scripts/model/rule/test_rule_common.py | 20 ++++++- .../model/rule/test_rule_tc609_quality.py | 52 +++++++++++++++++++ 4 files changed, 84 insertions(+), 1 deletion(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index e5f56fe3..3571dd7c 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -737,6 +737,7 @@ def eval(cls, input_data: Data) -> EvalDetail: class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): """0208: Content cleanliness, composed from available cleaning rules.""" + dynamic_config = EvaluatorRuleArgs(key_list=[]) component_rules = ( "dingo.model.rule.rule_common.RuleAbnormalChar", "dingo.model.rule.rule_common.RuleAbnormalHtml", @@ -752,6 +753,14 @@ class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): "partial", ) + @classmethod + def eval(cls, input_data: Data) -> EvalDetail: + rule_watermark = cls._resolve_rule(cls.component_rules[-1]) + rule_watermark.dynamic_config = EvaluatorRuleArgs( + key_list=cls.dynamic_config.key_list or [], + ) + return super().eval(input_data) + @Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"]) class Rule_TC609_02080101_TextPerplexity(BaseRule): diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index 306ad4c9..bbb9f066 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -49,6 +49,10 @@ def eval(cls, input_data: Data) -> EvalDetail: component_res = rule.eval(input_data) except (ImportError, ModuleNotFoundError): raise + except ValueError: + # Invalid evaluator configuration must stop the composite + # instead of being converted into a data-quality finding. + raise except Exception as exc: reasons.append(f"{rule.__name__}: {type(exc).__name__}: {exc}") continue diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 7b02509e..8c167da4 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -7,7 +7,12 @@ Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity) from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness -from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords +from dingo.model.rule.rule_common import ( + RuleDocFormulaRepeat, + RulePIIDetection, + RuleUnsafeWords, + RuleWatermark, +) class TestRuleDocFormulaRepeat: @@ -20,6 +25,19 @@ def test_rule_doc_formula_repeat(self): assert res.metric == "RuleDocFormulaRepeat" assert res.reason == ["Formula has too many consecutive repeated characters, total repeat length: 130, found 1 repeat patterns"] + def test_requires_configured_watermarks(self, monkeypatch): + monkeypatch.setattr( + RuleWatermark, + "dynamic_config", + EvaluatorRuleArgs(key_list=[]), + ) + + with pytest.raises( + ValueError, + match="RuleWatermark requires non-empty dynamic_config.key_list", + ): + RuleWatermark.eval(Data(content="safe text")) + def test_rule_unsafe_words(self, monkeypatch): data = Data(data_id="", prompt="", content="java is good\n \n \n \n hello \n \n but python is better") monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 11af0b94..1438aaca 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -15,8 +15,10 @@ Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0206_ContentConsistency, + Rule_TC609_0208_ContentCleanliness, Rule_TC609_0301_ContentDiversity, ) +from dingo.model.rule.rule_common import RuleWatermark def test_tc609_quality_defines_all_standard_metrics(): @@ -367,6 +369,56 @@ def test_annotation_compliance_requires_allowed_values(monkeypatch): Rule_TC609_0203_AnnotationCompliance.eval(Data(content="positive")) +def test_content_cleanliness_propagates_empty_watermark_config(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0208_ContentCleanliness, + "dynamic_config", + EvaluatorRuleArgs(key_list=[]), + ) + + with pytest.raises( + ValueError, + match="RuleWatermark requires non-empty dynamic_config.key_list", + ): + Rule_TC609_0208_ContentCleanliness.eval(Data(content="safe text")) + + +def test_content_cleanliness_passes_key_list_to_watermark(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0208_ContentCleanliness, + "dynamic_config", + EvaluatorRuleArgs(key_list=["DINGO-WATERMARK"]), + ) + monkeypatch.setattr( + RuleWatermark, + "dynamic_config", + EvaluatorRuleArgs(key_list=["stale-value"]), + ) + + result = Rule_TC609_0208_ContentCleanliness.eval( + Data(content="text with DINGO-WATERMARK") + ) + + assert RuleWatermark.dynamic_config.key_list == ["DINGO-WATERMARK"] + assert result.status is True + assert result.reason == ["RuleWatermark: DINGO-WATERMARK"] + + +def test_content_cleanliness_returns_good_without_watermark(monkeypatch): + monkeypatch.setattr( + Rule_TC609_0208_ContentCleanliness, + "dynamic_config", + EvaluatorRuleArgs(key_list=["DINGO-WATERMARK"]), + ) + + result = Rule_TC609_0208_ContentCleanliness.eval( + Data(content="ordinary clean text") + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + def test_structural_completeness_accepts_present_values(monkeypatch): monkeypatch.setattr( Rule_TC609_0204_StructuralCompleteness, From 86867851cb5eb12ea16bb766bd3b1235cad33f9f Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 17:35:11 +0800 Subject: [PATCH 55/80] =?UTF-8?q?feat:=20=E6=97=A0=E7=94=A8init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/config/input_args.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/dingo/config/input_args.py b/dingo/config/input_args.py index c0226257..d6cc310d 100644 --- a/dingo/config/input_args.py +++ b/dingo/config/input_args.py @@ -194,6 +194,3 @@ class InputArgs(BaseModel): dataset: DatasetArgs = DatasetArgs() executor: ExecutorArgs = ExecutorArgs() evaluator: List[EvalPipline] = [] - - def __init__(self, **kwargs): - super().__init__(**kwargs) From 51af2dd475358f1265bb0187fa183bf77cc89953 Mon Sep 17 00:00:00 2001 From: shijin Date: Thu, 30 Jul 2026 17:55:44 +0800 Subject: [PATCH 56/80] feat: lint --- examples/guobiao/rule_content_consistency.py | 4 +--- test/scripts/model/rule/test_rule_common.py | 7 +------ test/scripts/model/rule/test_rule_tc609_quality.py | 14 ++++---------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/examples/guobiao/rule_content_consistency.py b/examples/guobiao/rule_content_consistency.py index 938f34a6..6cac94e1 100644 --- a/examples/guobiao/rule_content_consistency.py +++ b/examples/guobiao/rule_content_consistency.py @@ -9,9 +9,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io import Data -from dingo.model.rule.guobiao.rule_tc609_quality import ( - Rule_TC609_0206_ContentConsistency, -) +from dingo.model.rule.guobiao.rule_tc609_quality import Rule_TC609_0206_ContentConsistency def main(): diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 8c167da4..73556546 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -7,12 +7,7 @@ Rule_TC609_0104_DocApplicationCompleteness, Rule_TC609_0207_DataTypeConsistency, Rule_TC609_0303_DataTimeRange, Rule_TC609_02080101_TextPerplexity) from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness -from dingo.model.rule.rule_common import ( - RuleDocFormulaRepeat, - RulePIIDetection, - RuleUnsafeWords, - RuleWatermark, -) +from dingo.model.rule.rule_common import RuleDocFormulaRepeat, RulePIIDetection, RuleUnsafeWords, RuleWatermark class TestRuleDocFormulaRepeat: diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 1438aaca..8ee99f9f 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -8,16 +8,9 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.guobiao import rule_tc609_quality -from dingo.model.rule.guobiao.rule_tc609_quality import ( - Rule_TC609_0201_FormatCompliance, - Rule_TC609_0202_SafetyCompliance, - Rule_TC609_0203_AnnotationCompliance, - Rule_TC609_0204_StructuralCompleteness, - Rule_TC609_0205_ContentAuthenticity, - Rule_TC609_0206_ContentConsistency, - Rule_TC609_0208_ContentCleanliness, - Rule_TC609_0301_ContentDiversity, -) +from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance, + Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0206_ContentConsistency, + Rule_TC609_0208_ContentCleanliness, Rule_TC609_0301_ContentDiversity) from dingo.model.rule.rule_common import RuleWatermark @@ -204,6 +197,7 @@ def test_format_compliance_still_requires_optional_field(monkeypatch): assert result.status is True assert result.reason == ["value: required field is missing"] + @pytest.mark.parametrize( "schema, error", [ From 8cd64e2f15c15d5243ce5e3f7a91e92ea825e341 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 11:28:08 +0800 Subject: [PATCH 57/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0201=5FFormatCompl?= =?UTF-8?q?iance=20=E9=BB=98=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 14 ++++- .../model/rule/test_rule_tc609_quality.py | 55 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 3571dd7c..164ea81c 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -168,7 +168,19 @@ class Rule_TC609_0201_FormatCompliance(BaseRule): "Optional[dict]": {"expected_type": dict, "allow_none": True}, } dynamic_config = EvaluatorRuleArgs( - field_schema=None, + field_schema={ + "id": "str", + "rid": "Optional[list]", + "data_content": "list", + "annotation": "Optional[dict]", + "original_time": "str", + "last_modified_time": "str", + "version": "str", + "license": "str", + "source": "str", + "source_details": "str", + "generated_data_indicator": "int", + }, allow_extra=True, ) _metric_info = _tc609_metric_info( diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 8ee99f9f..4f153ab4 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -93,6 +93,61 @@ def test_format_compliance_accepts_matching_record(monkeypatch): assert result.label == [QualityLabel.QUALITY_GOOD] +def test_format_compliance_default_schema_matches_tc609_metadata(): + assert Rule_TC609_0201_FormatCompliance.dynamic_config.field_schema == { + "id": "str", + "rid": "Optional[list]", + "data_content": "list", + "annotation": "Optional[dict]", + "original_time": "str", + "last_modified_time": "str", + "version": "str", + "license": "str", + "source": "str", + "source_details": "str", + "generated_data_indicator": "int", + } + + result = Rule_TC609_0201_FormatCompliance.eval( + Data( + id="d6c9a4d5e57597df8fe30f09ae44c985", + rid=None, + data_content=[ + { + "media_type": "image", + "content": "../data/images/streetscape.jpg", + } + ], + annotation=None, + original_time="2025-1-1", + last_modified_time="2025-1-1", + version="1.0.0-alpha", + license="其他", + source="互联网", + source_details="https://example.com/image.jpg", + generated_data_indicator=0, + ) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_format_compliance_default_schema_requires_mandatory_fields(): + result = Rule_TC609_0201_FormatCompliance.eval( + Data( + id="dataset-id", + data_content={}, + ) + ) + + assert result.status is True + assert "data_content: expected list, got dict" in result.reason + assert "rid: required field is missing" in result.reason + assert "annotation: required field is missing" in result.reason + assert "original_time: required field is missing" in result.reason + + def test_format_compliance_reports_missing_and_wrong_type(monkeypatch): monkeypatch.setattr( Rule_TC609_0201_FormatCompliance, From 08e2d64a6622682dfa15e888188a95f75f904692 Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 31 Jul 2026 11:35:28 +0800 Subject: [PATCH 58/80] fix: LLMPerspective: UnboundLocalError: cannot access local variable 'discovery' --- dingo/model/llm/llm_perspective.py | 9 ++-- dingo/model/rule/rule_hallucination_hhem.py | 57 +++++++++++++++------ setup.py | 4 +- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/dingo/model/llm/llm_perspective.py b/dingo/model/llm/llm_perspective.py index 77876cca..a4fbb34a 100644 --- a/dingo/model/llm/llm_perspective.py +++ b/dingo/model/llm/llm_perspective.py @@ -20,10 +20,11 @@ class LLMPerspective(BaseLLM): def create_client(cls): try: from googleapiclient import discovery - except ImportError: - log.warning( - "=========== perspective register fail. Please check whether install googleapiclient. ===========" - ) + except ImportError as exc: + raise ImportError( + "google-api-python-client is required for LLMPerspective. " + "Install with: pip install google-api-python-client" + ) from exc if cls.client is None: if not cls.dynamic_config.key: diff --git a/dingo/model/rule/rule_hallucination_hhem.py b/dingo/model/rule/rule_hallucination_hhem.py index bf2a759d..a9fd3ff8 100644 --- a/dingo/model/rule/rule_hallucination_hhem.py +++ b/dingo/model/rule/rule_hallucination_hhem.py @@ -12,6 +12,7 @@ """ import json +from threading import Lock from typing import List from dingo.config.input_args import EvaluatorRuleArgs @@ -48,28 +49,52 @@ class RuleHallucinationHHEM(BaseRule): _required_fields = [RequiredField.CONTENT, RequiredField.CONTEXT] dynamic_config = EvaluatorRuleArgs(threshold=0.5) model = None + _load_lock = Lock() + _model_repo_id = "vectara/hallucination_evaluation_model" @classmethod def load_model(cls): """Load HHEM-2.1-Open model""" if cls.model is None: - try: - from transformers import AutoModelForSequenceClassification - - log.info("Loading HHEM-2.1-Open model...") - cls.model = AutoModelForSequenceClassification.from_pretrained( - 'vectara/hallucination_evaluation_model', - trust_remote_code=True - ) - log.info("✅ HHEM-2.1-Open model loaded successfully") + with cls._load_lock: + if cls.model is not None: + return + try: + from huggingface_hub import snapshot_download + from transformers import AutoModelForSequenceClassification + + log.info("Loading HHEM-2.1-Open model...") + try: + model_path = snapshot_download( + repo_id=cls._model_repo_id, + repo_type="model", + local_files_only=True, + ) + except Exception: + model_path = snapshot_download( + repo_id=cls._model_repo_id, + repo_type="model", + ) - except ImportError: - raise ImportError( - "transformers library is required for HHEM model. " - "Install with: pip install transformers" - ) - except Exception as e: - raise RuntimeError(f"Failed to load HHEM model: {e}") + cls.model = AutoModelForSequenceClassification.from_pretrained( + model_path, + trust_remote_code=True, + local_files_only=True, + ) + log.info("✅ HHEM-2.1-Open model loaded successfully") + + except ImportError: + raise ImportError( + "transformers and huggingface_hub are required for HHEM model. " + "Install with: pip install transformers huggingface_hub" + ) + except Exception as e: + raise RuntimeError( + "Failed to load HHEM model. " + "The first run requires network access to download " + f"'{cls._model_repo_id}', or a populated Hugging Face cache. " + f"Original error: {e}" + ) from e @classmethod def eval(cls, input_data: Data) -> EvalDetail: diff --git a/setup.py b/setup.py index fafbe638..307e9e31 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ def _read_requirements(path): requirements = _read_requirements("./requirements/runtime.txt") +optional_requirements = _read_requirements("./requirements/optional.txt") agent_requirements = _read_requirements("./requirements/agent.txt") hhem_requirements = _read_requirements("./requirements/hhem_integration.txt") litellm_requirements = ["litellm>=1.80.0,<1.87.0"] @@ -18,11 +19,12 @@ def _read_requirements(path): extras_require = { + 'optional': optional_requirements, 'agent': agent_requirements, 'hhem': hhem_requirements, 'litellm': litellm_requirements, 'retrieval': retrieval_requirements, - 'all': hhem_requirements + agent_requirements + litellm_requirements + retrieval_requirements, + 'all': optional_requirements + hhem_requirements + agent_requirements + litellm_requirements + retrieval_requirements, } From 5d2f6bc2c1b00c6edcd73f80b46100c5ae542705 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 12:24:55 +0800 Subject: [PATCH 59/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0202=5FSafetyCompl?= =?UTF-8?q?iance=20=E9=BB=98=E8=AE=A4=E7=9A=84=E5=AE=89=E5=85=A8=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 22 +++++++++++++++++-- dingo/model/rule/rule_common.py | 4 ++++ .../model/rule/test_rule_tc609_quality.py | 15 +++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 164ea81c..cb5381a8 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -259,7 +259,18 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): """0202: Safety compliance, composed from safety and PII rules.""" dynamic_config = EvaluatorRuleArgs( - key_list=[], + key_list=[ + "制作炸弹", + "购买毒品", + "贩卖毒品", + "实施诈骗", + "洗钱教程", + "自杀方法", + "色情交易", + "儿童色情", + "恐怖袭击", + "非法枪支", + ], refer_path=[], ) component_rules = ( @@ -278,10 +289,17 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): @classmethod def eval(cls, input_data: Data) -> EvalDetail: rule_unsafe_words = cls._resolve_rule(cls.component_rules[0]) - rule_unsafe_words.dynamic_config = EvaluatorRuleArgs( + unsafe_words_config = EvaluatorRuleArgs( key_list=cls.dynamic_config.key_list or [], refer_path=cls.dynamic_config.refer_path or [], ) + if ( + getattr(rule_unsafe_words, "dynamic_config", None) + != unsafe_words_config + ): + rule_unsafe_words._unsafe_words_list = None + rule_unsafe_words._unsafe_words_automaton = None + rule_unsafe_words.dynamic_config = unsafe_words_config return super().eval(input_data) diff --git a/dingo/model/rule/rule_common.py b/dingo/model/rule/rule_common.py index 2293d60b..75d3d0db 100644 --- a/dingo/model/rule/rule_common.py +++ b/dingo/model/rule/rule_common.py @@ -2169,6 +2169,10 @@ def eval(cls, input_data: Data) -> EvalDetail: @classmethod def _is_whole_word(cls, text: str, start: int, end: int) -> bool: """检查匹配是否是一个完整的单词""" + keyword = text[start:end + 1] + if not keyword.isascii(): + return True + # 检查左侧边界 if start > 0 and text[start - 1].isalnum(): return False diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 4f153ab4..244477d1 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -372,6 +372,21 @@ def eval(cls, input_data): assert result.reason == ["UnsafeWordsRule: unsafe"] +def test_safety_compliance_has_usable_default_words(monkeypatch): + from dingo.model.rule.rule_common import RuleUnsafeWords + + assert Rule_TC609_0202_SafetyCompliance.dynamic_config.key_list + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_list", None) + monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) + + result = Rule_TC609_0202_SafetyCompliance.eval( + Data(content="该内容提供制作炸弹的具体步骤") + ) + + assert result.status is True + assert "RuleUnsafeWords: 制作炸弹" in result.reason + + def test_annotation_compliance_accepts_allowed_content(monkeypatch): monkeypatch.setattr( Rule_TC609_0203_AnnotationCompliance, From 147d5811366f58a7940d5d871e02bd866415471b Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 31 Jul 2026 13:47:52 +0800 Subject: [PATCH 60/80] fix --- dingo/__init__.py | 7 +++++++ requirements/optional.txt | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dingo/__init__.py b/dingo/__init__.py index e69de29b..0ec17bd2 100644 --- a/dingo/__init__.py +++ b/dingo/__init__.py @@ -0,0 +1,7 @@ +import os + +# 为无法访问 huggingface.co 的环境提供默认镜像。 +# 必须在任何 huggingface_hub / transformers 被导入之前设置， +# 因为 HF_ENDPOINT 在 huggingface_hub 导入时即被固定读取。 +# 使用 setdefault 保证用户可通过外部环境变量覆盖为其他镜像或官方地址。 +os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com") diff --git a/requirements/optional.txt b/requirements/optional.txt index 92aad3e6..bd51970a 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -14,5 +14,3 @@ tiktoken torch>=1.7.1 torchvision tqdm - -git+https://github.com/openai/CLIP.git From 11e46b7e8208fc2f010ba11ba47e5ca4efbd9be9 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 14:25:22 +0800 Subject: [PATCH 61/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0203=5FAnnotationC?= =?UTF-8?q?ompliance=20=E9=87=8D=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + .../model/rule/guobiao/rule_tc609_quality.py | 84 +++++++++++++++---- docs/metrics.md | 2 +- docs/rules.md | 2 +- .../model/rule/test_rule_tc609_quality.py | 73 ++++++++++------ 5 files changed, 121 insertions(+), 41 deletions(-) diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index b94f06ee..e050faca 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -10,3 +10,4 @@ class RequiredField(Enum): TYPE = "type" DT = "dt" SOURCE = "source" + ANNOTATION = "annotation" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index cb5381a8..b5eaa8ef 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -305,39 +305,93 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"]) class Rule_TC609_0203_AnnotationCompliance(BaseRule): - """Check whether content is one of the configured annotation values.""" + """Check annotation metadata against TC609 format requirements.""" - dynamic_config = EvaluatorRuleArgs(key_list=[]) - _required_fields = [RequiredField.CONTENT] + _annotation_methods = { + "人工标注", + "自动标注", + "半自动标注", + "其他", + } + _annotator_types = { + "普通标注员", + "专业标注员", + "行业领域专家", + "其他", + } + _required_fields = [RequiredField.ANNOTATION] _metric_info = _tc609_metric_info( "0203", "Rule_TC609_0203_AnnotationCompliance", - "Checks whether content belongs to a user-provided annotation value list.", + "Checks annotation metadata fields, types, and enumerated values.", "covered", ) @classmethod def eval(cls, input_data: Data) -> EvalDetail: - allowed_values = cls.dynamic_config.key_list or [] - if not allowed_values: - raise ValueError( - "Rule_TC609_0203_AnnotationCompliance requires a non-empty " - "dynamic_config.key_list" - ) - res = EvalDetail(metric=cls.__name__) - content = getattr(input_data, "content", None) - if content not in allowed_values: + reasons = [] + annotation = input_data.annotation + + if annotation is None: + res.label = [QualityLabel.QUALITY_GOOD] + return res + + if not isinstance(annotation, dict): res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] res.reason = [ - f"content: value {content!r} is not in dynamic_config.key_list" + "annotation: expected dict or None, " + f"got {type(annotation).__name__}" ] + return res + + if "label" not in annotation: + res.status = True + reasons.append("annotation.label: required field is missing") + elif not isinstance(annotation["label"], list): + res.status = True + reasons.append( + "annotation.label: expected list, " + f"got {type(annotation['label']).__name__}" + ) + elif not annotation["label"]: + res.status = True + reasons.append("annotation.label: empty value is not allowed") + + for field_name, allowed_values in ( + ("annotation_method", cls._annotation_methods), + ("annotator", cls._annotator_types), + ): + qualified_name = f"annotation.{field_name}" + if field_name not in annotation: + res.status = True + reasons.append(f"{qualified_name}: required field is missing") + continue + + value = annotation[field_name] + if value is None: + continue + if not isinstance(value, str): + res.status = True + reasons.append( + f"{qualified_name}: expected str or None, " + f"got {type(value).__name__}" + ) + elif value not in allowed_values: + res.status = True + reasons.append( + f"{qualified_name}: unsupported value {value!r}; " + f"allowed values: {', '.join(sorted(allowed_values))}" + ) + + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons else: res.label = [QualityLabel.QUALITY_GOOD] return res - @Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"]) class Rule_TC609_0204_StructuralCompleteness(BaseRule): """Check required fields for missing, None, and empty values.""" diff --git a/docs/metrics.md b/docs/metrics.md index 022f6246..00ab33bb 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -164,7 +164,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether `content` belongs to the annotation values configured in `key_list`. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks TC609 annotation metadata fields, types, and enumerated values. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `content` and `source`, and checks whether the HTTP or HTTPS `source` returns status 200. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | diff --git a/docs/rules.md b/docs/rules.md index ddbdfafa..ac2b0036 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -92,7 +92,7 @@ The specific rules for each quality metric are as follows: | Rule_TC609_0104_DocApplicationCompleteness | TC609_0104 | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0201_FormatCompliance | TC609_0201 | Combines existing NLP, SFT, image, audio, and video format rules. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Combines image-label overlap and visualization checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Uses HHEM consistency checking as partial evidence of authenticity. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0206_ContentConsistency | TC609_0206 | Combines structured-field and image-text consistency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 244477d1..f7fb1be1 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -387,50 +387,75 @@ def test_safety_compliance_has_usable_default_words(monkeypatch): assert "RuleUnsafeWords: 制作炸弹" in result.reason -def test_annotation_compliance_accepts_allowed_content(monkeypatch): - monkeypatch.setattr( - Rule_TC609_0203_AnnotationCompliance, - "dynamic_config", - EvaluatorRuleArgs(key_list=["positive", "negative"]), +def test_annotation_compliance_accepts_valid_metadata(): + result = Rule_TC609_0203_AnnotationCompliance.eval( + Data( + annotation={ + "label": [ + { + "iscrowd": 0, + "bbox": [20, 20, 20, 20], + "category": "human", + } + ], + "annotation_method": "人工标注", + "annotator": "普通标注员", + } + ) ) + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + +def test_annotation_compliance_accepts_none_for_unannotated_data(): result = Rule_TC609_0203_AnnotationCompliance.eval( - Data(content="positive") + Data(annotation=None) ) assert result.status is False assert result.label == [QualityLabel.QUALITY_GOOD] -def test_annotation_compliance_rejects_unknown_content(monkeypatch): - monkeypatch.setattr( - Rule_TC609_0203_AnnotationCompliance, - "dynamic_config", - EvaluatorRuleArgs(key_list=["positive", "negative"]), - ) +def test_annotation_compliance_declares_required_field(): + assert Rule_TC609_0203_AnnotationCompliance._required_fields == [ + RequiredField.ANNOTATION + ] + +def test_annotation_compliance_reports_invalid_nested_metadata(): result = Rule_TC609_0203_AnnotationCompliance.eval( - Data(content="neutral") + Data( + annotation={ + "label": [], + "annotation_method": "众包标注", + "annotator": 1, + } + ) ) assert result.status is True - assert result.label == [ - "QUALITY_BAD_TC609_0203.Rule_TC609_0203_AnnotationCompliance" - ] assert result.reason == [ - "content: value 'neutral' is not in dynamic_config.key_list" + "annotation.label: empty value is not allowed", + ( + "annotation.annotation_method: unsupported value '众包标注'; " + "allowed values: 人工标注, 其他, 半自动标注, 自动标注" + ), + "annotation.annotator: expected str or None, got int", ] -def test_annotation_compliance_requires_allowed_values(monkeypatch): - monkeypatch.setattr( - Rule_TC609_0203_AnnotationCompliance, - "dynamic_config", - EvaluatorRuleArgs(key_list=[]), +def test_annotation_compliance_requires_all_nested_fields(): + result = Rule_TC609_0203_AnnotationCompliance.eval( + Data(annotation={}) ) - with pytest.raises(ValueError, match="non-empty dynamic_config.key_list"): - Rule_TC609_0203_AnnotationCompliance.eval(Data(content="positive")) + assert result.status is True + assert result.reason == [ + "annotation.label: required field is missing", + "annotation.annotation_method: required field is missing", + "annotation.annotator: required field is missing", + ] def test_content_cleanliness_propagates_empty_watermark_config(monkeypatch): From 85eea670c4c887e4f8ce68e27fcb801196be1d98 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 14:30:30 +0800 Subject: [PATCH 62/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0204=5FStructuralC?= =?UTF-8?q?ompleteness=20=E9=BB=98=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 12 ++++++- .../model/rule/test_rule_tc609_quality.py | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index b5eaa8ef..398617ee 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -397,7 +397,17 @@ class Rule_TC609_0204_StructuralCompleteness(BaseRule): """Check required fields for missing, None, and empty values.""" dynamic_config = EvaluatorRuleArgs( - key_list=[], + key_list=[ + "id", + "data_content", + "original_time", + "last_modified_time", + "version", + "license", + "source", + "source_details", + "generated_data_indicator", + ], allow_none=False, allow_empty=False, ) diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index f7fb1be1..78c31452 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -527,6 +527,37 @@ def test_structural_completeness_accepts_present_values(monkeypatch): assert result.label == [QualityLabel.QUALITY_GOOD] +def test_structural_completeness_has_tc609_required_fields_by_default(): + assert Rule_TC609_0204_StructuralCompleteness.dynamic_config.key_list == [ + "id", + "data_content", + "original_time", + "last_modified_time", + "version", + "license", + "source", + "source_details", + "generated_data_indicator", + ] + + result = Rule_TC609_0204_StructuralCompleteness.eval( + Data( + id="dataset-id", + data_content=[{"media_type": "text", "content": "example"}], + original_time="2025-01-01", + last_modified_time="2025-01-01", + version="1.0.0", + license="其他", + source="互联网", + source_details="https://example.com/data", + generated_data_indicator=0, + ) + ) + + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] + + def test_structural_completeness_reports_missing_none_and_empty(monkeypatch): monkeypatch.setattr( Rule_TC609_0204_StructuralCompleteness, From d90869f93f0c62c184c7960ca4ad67121e8a21d9 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 15:03:40 +0800 Subject: [PATCH 63/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0205=5FContentAuth?= =?UTF-8?q?enticity=20=E9=87=8D=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + .../model/rule/guobiao/rule_tc609_quality.py | 81 ++++++------ docs/metrics.md | 2 +- docs/rules.md | 2 +- .../model/rule/test_rule_tc609_quality.py | 118 +++++------------- 5 files changed, 71 insertions(+), 133 deletions(-) diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index e050faca..dcf70d6e 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -10,4 +10,5 @@ class RequiredField(Enum): TYPE = "type" DT = "dt" SOURCE = "source" + SOURCE_DETAILS = "source_details" ANNOTATION = "annotation" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 398617ee..9579dd17 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -463,73 +463,68 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0205", ["guobiao_data"]) class Rule_TC609_0205_ContentAuthenticity(BaseRule): - """Check whether a record's HTTP or HTTPS source returns status 200.""" + """Check whether source metadata provides valid traceability information.""" - _required_fields = [RequiredField.CONTENT, RequiredField.SOURCE] - dynamic_config = EvaluatorRuleArgs(timeout=10) + _required_fields = [ + RequiredField.SOURCE, + RequiredField.SOURCE_DETAILS, + ] _metric_info = _tc609_metric_info( "0205", "Rule_TC609_0205_ContentAuthenticity", - "Checks whether source is an HTTP or HTTPS URL that returns status 200.", + "Checks source and source_details, including URL format when applicable.", "covered", ) @classmethod def eval(cls, input_data: Data) -> EvalDetail: - source = getattr(input_data, "source", None) res = EvalDetail(metric=cls.__name__) - if not ( - isinstance(source, str) - and source - and source.lower().startswith(("http://", "https://")) - ): + source = input_data.source + source_details = input_data.source_details + + if not isinstance(source, str) or not source.strip(): res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["source: expected a valid HTTP or HTTPS URL"] + res.reason = ["source: expected a non-empty string"] return res - import requests + if not isinstance(source_details, str) or not source_details.strip(): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["source_details: expected a non-empty string"] + return res - timeout = getattr(cls.dynamic_config, "timeout") - if ( - isinstance(timeout, bool) - or not isinstance(timeout, int) - or timeout <= 0 - ): - raise ValueError( - "Rule_TC609_0205_ContentAuthenticity requires " - "dynamic_config.timeout to be a positive integer" - ) - response = None - try: - response = requests.get( - source, - timeout=timeout, - allow_redirects=True, - stream=True, - ) - if response.status_code != 200: + if source.strip() == "互联网": + if not cls._is_valid_http_url(source_details): res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] res.reason = [ - f"source: URL returned HTTP status {response.status_code}, " - "expected 200" + "source_details: expected a valid HTTP or HTTPS URL" ] return res - except requests.RequestException as exc: - res.status = True - res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = [ - f"source: URL request failed: {type(exc).__name__}: {exc}" - ] - return res - finally: - if response is not None: - response.close() res.label = [QualityLabel.QUALITY_GOOD] return res + @staticmethod + def _is_valid_http_url(value): + from urllib.parse import urlparse + + if any(character.isspace() for character in value): + return False + try: + parsed = urlparse(value) + if ( + parsed.scheme.lower() not in {"http", "https"} + or not parsed.netloc + or not parsed.hostname + ): + return False + parsed.port + except ValueError: + return False + return True + @Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"]) class Rule_TC609_0206_ContentConsistency(BaseRule): diff --git a/docs/metrics.md b/docs/metrics.md index 00ab33bb..00c820c2 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -166,7 +166,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks TC609 annotation metadata fields, types, and enumerated values. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `content` and `source`, and checks whether the HTTP or HTTPS `source` returns status 200. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `source` and `source_details`; validates non-empty traceability information and HTTP/HTTPS URL syntax when applicable. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses a local multilingual NLI model to check semantic consistency among string fields configured in `key_list`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | diff --git a/docs/rules.md b/docs/rules.md index ac2b0036..3f57ae01 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -94,7 +94,7 @@ The specific rules for each quality metric are as follows: | Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Uses HHEM consistency checking as partial evidence of authenticity. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0206_ContentConsistency | TC609_0206 | Combines structured-field and image-text consistency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080101_TextPerplexity | TC609_02080101 | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | TC609-5-2025-04 High-quality dataset quality evaluation specification | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 78c31452..05f7b77c 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -618,136 +618,78 @@ def test_structural_completeness_requires_key_list(monkeypatch): @pytest.mark.parametrize( - "source", + "source_details", [ "https://example.com/data/1", "http://localhost:8080/record?id=1", ], ) -def test_content_authenticity_accepts_source_returning_200(source, monkeypatch): - class Response: - status_code = 200 - - def close(self): - pass - - monkeypatch.setattr("requests.get", lambda *args, **kwargs: Response()) +def test_content_authenticity_accepts_valid_internet_url(source_details): result = Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source=source) + Data(source="互联网", source_details=source_details) ) assert result.status is False assert result.label == [QualityLabel.QUALITY_GOOD] -def test_content_authenticity_rejects_source_not_returning_200(monkeypatch): - class Response: - status_code = 404 - - def close(self): - pass - - monkeypatch.setattr("requests.get", lambda *args, **kwargs: Response()) - result = Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source="https://example.com/missing") - ) - - assert result.status is True - assert result.reason == [ - "source: URL returned HTTP status 404, expected 200" - ] - - -def test_content_authenticity_rejects_request_failure(monkeypatch): - import requests - - def raise_timeout(*args, **kwargs): - raise requests.Timeout("timed out") - - monkeypatch.setattr("requests.get", raise_timeout) +def test_content_authenticity_accepts_non_url_source_details(): result = Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source="https://example.com/slow") - ) - - assert result.status is True - assert result.reason == [ - "source: URL request failed: Timeout: timed out" - ] - - -@pytest.mark.parametrize("timeout", [10.0, "10", 0, -1, True, None]) -def test_content_authenticity_requires_positive_integer_timeout( - timeout, monkeypatch -): - monkeypatch.setattr( - Rule_TC609_0205_ContentAuthenticity, - "dynamic_config", - EvaluatorRuleArgs(timeout=timeout), + Data( + source="图书", + source_details="ISBN 978-7-121-15535-2，第 10 页", + ) ) - with pytest.raises(ValueError, match="positive integer"): - Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source="https://example.com/data") - ) + assert result.status is False + assert result.label == [QualityLabel.QUALITY_GOOD] @pytest.mark.parametrize( - "source", + "source, source_details, reason", [ - None, - "", - "example.com/data/1", - "ftp://example.com/data/1", + (None, "detail", "source: expected a non-empty string"), + ("", "detail", "source: expected a non-empty string"), + ("图书", None, "source_details: expected a non-empty string"), + ("图书", "", "source_details: expected a non-empty string"), ], ) -def test_content_authenticity_rejects_source_without_http_prefix(source): +def test_content_authenticity_rejects_empty_source_metadata( + source, source_details, reason +): result = Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source=source) + Data(source=source, source_details=source_details) ) assert result.status is True - assert result.label == [ - "QUALITY_BAD_TC609_0205.Rule_TC609_0205_ContentAuthenticity" - ] - assert result.reason == [ - "source: expected a valid HTTP or HTTPS URL" - ] + assert result.reason == [reason] @pytest.mark.parametrize( - "source", + "source, source_details", [ - "https://", - "https://exa mple.com/data/1", - "https://example.com:invalid/data/1", + ("互联网", "example.com/data/1"), + ("互联网", "ftp://example.com/data/1"), + ("互联网", "https://"), + ("互联网", "https://exa mple.com/data/1"), + ("互联网", "https://example.com:invalid/data/1"), ], ) -def test_content_authenticity_handles_prefixed_url_request_failure( - source, monkeypatch -): - import requests - - def raise_invalid_url(*args, **kwargs): - raise requests.exceptions.InvalidURL("failed to parse URL") - - monkeypatch.setattr("requests.get", raise_invalid_url) +def test_content_authenticity_rejects_invalid_url(source, source_details): result = Rule_TC609_0205_ContentAuthenticity.eval( - Data(content="example", source=source) + Data(source=source, source_details=source_details) ) assert result.status is True - assert result.label == [ - "QUALITY_BAD_TC609_0205.Rule_TC609_0205_ContentAuthenticity" - ] assert result.reason == [ - "source: URL request failed: InvalidURL: failed to parse URL" + "source_details: expected a valid HTTP or HTTPS URL" ] def test_content_authenticity_declares_required_fields(): assert Rule_TC609_0205_ContentAuthenticity._required_fields == [ - RequiredField.CONTENT, RequiredField.SOURCE, + RequiredField.SOURCE_DETAILS, ] From d092d95311a61cb8efeff9882525f4588180613d Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 15:43:21 +0800 Subject: [PATCH 64/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0206=5FContentCons?= =?UTF-8?q?istency=20=E9=87=8D=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/io/input/required_field.py | 1 + .../model/rule/guobiao/rule_tc609_quality.py | 210 +++++++----------- .../rule/guobiao/rule_tc609_quality_base.py | 164 ++++++++++++++ docs/metrics.md | 2 +- docs/rules.md | 2 +- examples/guobiao/rule_content_consistency.py | 28 ++- .../model/rule/test_rule_tc609_quality.py | 171 ++++++++++---- 7 files changed, 399 insertions(+), 179 deletions(-) diff --git a/dingo/io/input/required_field.py b/dingo/io/input/required_field.py index dcf70d6e..60d070c7 100644 --- a/dingo/io/input/required_field.py +++ b/dingo/io/input/required_field.py @@ -12,3 +12,4 @@ class RequiredField(Enum): SOURCE = "source" SOURCE_DETAILS = "source_details" ANNOTATION = "annotation" + DATA_CONTENT = "data_content" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 9579dd17..c5950551 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -7,7 +7,13 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.base import BaseRule -from dingo.model.rule.guobiao.rule_tc609_quality_base import Rule_TC609_01_DocCompleteness, Rule_TC609_Composite, _tc609_metric_info, _TC609PlaceholderBase +from dingo.model.rule.guobiao.rule_tc609_quality_base import ( + Rule_TC609_01_DocCompleteness, + Rule_TC609_Composite, + _tc609_metric_info, + _TC609PlaceholderBase, + calculate_text_consistency, +) @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) @@ -528,158 +534,110 @@ def _is_valid_http_url(value): @Model.rule_register("QUALITY_BAD_TC609_0206", ["guobiao_data"]) class Rule_TC609_0206_ContentConsistency(BaseRule): - """Check semantic consistency among string fields configured in key_list.""" + """Check semantic consistency among text items in data_content.""" dynamic_config = EvaluatorRuleArgs( - key_list=[], threshold=0.5, - model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + model="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", device=-1, + batch_size=16, + max_length=512, + consensus_keep_ratio=0.8, ) _metric_info = _tc609_metric_info( "0206", "Rule_TC609_0206_ContentConsistency", - "Uses a local model to check semantic consistency among configured string fields.", - "covered", + "Checks semantic consistency among text items in data_content.", + "partial", ) - - _model_name = None - _model_device = None - _classifier = None - - @classmethod - def _get_classifier(cls, model_name, device): - if ( - cls._model_name == model_name - and cls._model_device == device - and cls._classifier is not None - ): - return cls._classifier - - required_packages = ("torch", "transformers") - missing_packages = [ - package - for package in required_packages - if importlib.util.find_spec(package) is None - ] - if missing_packages: - raise ImportError( - "Rule_TC609_0206_ContentConsistency requires optional packages: " - f"{', '.join(missing_packages)}. " - 'Install them with: pip install "dingo-python[hhem]"' - ) - - from transformers import pipeline - - cls._classifier = pipeline( - "zero-shot-classification", - model=model_name, - device=device, - ) - cls._model_name = model_name - cls._model_device = device - return cls._classifier + _required_fields = [RequiredField.DATA_CONTENT] @classmethod - def _calculate_consistency_score( - cls, reference, candidate, model_name, device - ): - classifier = cls._get_classifier(model_name, device) - result = classifier( - reference, - candidate_labels=[candidate], - hypothesis_template="这段文本与以下内容语义一致：{}", - multi_label=True, - truncation=True, - ) - labels = result.get("labels", []) - scores = result.get("scores", []) - if not labels or not scores or labels[0] != candidate: - raise RuntimeError("Zero-shot classifier returned an invalid result") + def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + data_content = input_data.data_content + if not isinstance(data_content, list) or not data_content: + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = ["data_content: expected a non-empty list"] + return res - score = float(scores[0]) - if not math.isfinite(score) or not 0.0 <= score <= 1.0: - raise RuntimeError( - f"Zero-shot classifier returned an invalid score: {score}" - ) - return score + texts = [] + text_indexes = [] + reasons = [] + for index, item in enumerate(data_content): + if not isinstance(item, dict): + res.status = True + reasons.append( + f"data_content[{index}]: expected dict, " + f"got {type(item).__name__}" + ) + continue - @classmethod - def eval(cls, input_data: Data) -> EvalDetail: - key_list = cls.dynamic_config.key_list - if not isinstance(key_list, list) or len(key_list) < 2: - raise ValueError( - "Rule_TC609_0206_ContentConsistency requires " - "dynamic_config.key_list to contain at least two fields" - ) - if ( - any(not isinstance(key, str) or not key for key in key_list) - or len(set(key_list)) != len(key_list) - ): - raise ValueError( - "Rule_TC609_0206_ContentConsistency requires " - "dynamic_config.key_list to contain unique non-empty strings" - ) + media_type = item.get("media_type") + if not isinstance(media_type, str) or not media_type.strip(): + res.status = True + reasons.append( + f"data_content[{index}].media_type: " + "expected a non-empty string" + ) + continue + if media_type.strip().lower() != "text": + continue - threshold = cls.dynamic_config.threshold - if ( - isinstance(threshold, bool) - or not isinstance(threshold, (int, float)) - or not 0 < threshold <= 1 - ): - raise ValueError( - "Rule_TC609_0206_ContentConsistency requires " - "dynamic_config.threshold to be in (0, 1]" - ) + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + res.status = True + reasons.append( + f"data_content[{index}].content: " + "expected a non-empty string for text media" + ) + continue + texts.append(content) + text_indexes.append(index) - record = input_data.model_dump() - invalid_fields = [ - field - for field in key_list - if field not in record or not isinstance(record[field], str) - ] - res = EvalDetail(metric=cls.__name__) - if invalid_fields: - res.status = True + if res.status: res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + return res + + if len(texts) < 2: + res.label = [QualityLabel.QUALITY_GOOD] res.reason = [ - f"{field}: field must exist and its value must be str" - for field in invalid_fields + "Fewer than two text items; consistency comparison is not needed" ] return res - reference_field = key_list[0] - pair_scores = [] - for candidate_field in key_list[1:]: - score = cls._calculate_consistency_score( - record[reference_field], - record[candidate_field], - cls.dynamic_config.model, - cls.dynamic_config.device, - ) - pair_scores.append((candidate_field, score)) - - minimum_score = min(score for _, score in pair_scores) - res.score = minimum_score - inconsistent_pairs = [ - (candidate_field, score) - for candidate_field, score in pair_scores - if score < threshold - ] - if inconsistent_pairs: + result = calculate_text_consistency( + texts=texts, + model_name=cls.dynamic_config.model, + device=cls.dynamic_config.device, + threshold=cls.dynamic_config.threshold, + batch_size=getattr(cls.dynamic_config, "batch_size", 16), + max_length=getattr(cls.dynamic_config, "max_length", 512), + consensus_keep_ratio=getattr( + cls.dynamic_config, + "consensus_keep_ratio", + 0.8, + ), + ) + res.score = result["score"] + if not result["is_consistent"]: res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] res.reason = [ - f"{reference_field} and {candidate_field} are inconsistent " - f"(score: {score:.4f}, threshold: {threshold:.4f})" - for candidate_field, score in inconsistent_pairs + "Text items in data_content are inconsistent " + f"(score: {res.score:.4f}, " + f"threshold: {cls.dynamic_config.threshold:.4f}, " + "outlier indexes: " + f"{[text_indexes[index] for index in result['outlier_indexes']]})" ] else: res.label = [QualityLabel.QUALITY_GOOD] res.reason = [ - f"Configured fields are consistent " - f"(minimum score: {minimum_score:.4f}, " - f"threshold: {threshold:.4f})" + "Text items in data_content are consistent " + f"(score: {res.score:.4f}, " + f"threshold: {cls.dynamic_config.threshold:.4f})" ] return res diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index bbb9f066..3da2ee30 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -25,6 +25,170 @@ def _tc609_metric_info(code, name, description, coverage): } +_text_embedding_components = {} + + +def _get_text_embedding_components(model_name, device): + """Load and cache a transformer model used for text embeddings.""" + missing_packages = [ + package + for package in ("torch", "transformers") + if importlib.util.find_spec(package) is None + ] + if missing_packages: + raise ImportError( + "Text consistency evaluation requires optional packages: " + f"{', '.join(missing_packages)}. " + 'Install them with: pip install "dingo-python[hhem]"' + ) + + import torch + from transformers import AutoModel, AutoTokenizer + + if device == -1: + torch_device = "cpu" + elif isinstance(device, int): + torch_device = f"cuda:{device}" + else: + torch_device = str(device) + + cache_key = (model_name, torch_device) + if cache_key not in _text_embedding_components: + tokenizer = AutoTokenizer.from_pretrained(model_name) + model = AutoModel.from_pretrained(model_name) + model.to(torch_device) + model.eval() + _text_embedding_components[cache_key] = ( + tokenizer, + model, + torch_device, + ) + return _text_embedding_components[cache_key] + + +def _encode_texts(texts, model_name, device, batch_size, max_length): + """Encode texts once in batches and return normalized sentence vectors.""" + import torch + import torch.nn.functional as functional + + tokenizer, model, torch_device = _get_text_embedding_components( + model_name, + device, + ) + embeddings = [] + for start in range(0, len(texts), batch_size): + batch = texts[start:start + batch_size] + encoded = tokenizer( + batch, + padding=True, + truncation=True, + max_length=max_length, + return_tensors="pt", + ) + encoded = { + key: value.to(torch_device) + for key, value in encoded.items() + } + with torch.inference_mode(): + hidden_state = model(**encoded).last_hidden_state + attention_mask = encoded["attention_mask"].unsqueeze(-1) + pooled = ( + (hidden_state * attention_mask).sum(dim=1) + / attention_mask.sum(dim=1).clamp(min=1) + ) + embeddings.append(functional.normalize(pooled, p=2, dim=1).cpu()) + return torch.cat(embeddings, dim=0) + + +def calculate_text_consistency( + texts, + model_name, + device=-1, + threshold=0.5, + batch_size=16, + max_length=512, + consensus_keep_ratio=0.8, +): + """Calculate semantic consistency for two or more texts. + + Two texts are compared directly. For three or more texts, every text is + compared with a robust semantic center so the calculation remains linear + in the number of texts rather than evaluating all text pairs. + """ + if ( + not isinstance(texts, list) + or len(texts) < 2 + or any(not isinstance(text, str) or not text.strip() for text in texts) + ): + raise ValueError( + "calculate_text_consistency requires at least two non-empty texts" + ) + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not 0 <= threshold <= 1 + ): + raise ValueError("threshold must be in [0, 1]") + if isinstance(batch_size, bool) or not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length <= 0: + raise ValueError("max_length must be a positive integer") + if not 0 < consensus_keep_ratio <= 1: + raise ValueError("consensus_keep_ratio must be in (0, 1]") + + import torch + import torch.nn.functional as functional + + normalized_texts = [text.strip() for text in texts] + embeddings = _encode_texts( + normalized_texts, + model_name, + device, + batch_size, + max_length, + ) + + if len(normalized_texts) == 2: + score = float(torch.sum(embeddings[0] * embeddings[1]).item()) + item_scores = [score, score] + else: + initial_center = functional.normalize( + embeddings.mean(dim=0), + p=2, + dim=0, + ) + initial_scores = embeddings @ initial_center + keep_count = max( + 2, + math.ceil(len(normalized_texts) * consensus_keep_ratio), + ) + keep_indexes = torch.topk(initial_scores, keep_count).indices + robust_center = functional.normalize( + embeddings[keep_indexes].mean(dim=0), + p=2, + dim=0, + ) + similarities = embeddings @ robust_center + score = float(torch.min(similarities).item()) + item_scores = [float(value) for value in similarities.tolist()] + + score = min(1.0, max(0.0, score)) + item_scores = [ + min(1.0, max(0.0, value)) + for value in item_scores + ] + return { + "score": score, + "is_consistent": score >= threshold, + "item_scores": item_scores, + "outlier_indexes": [ + index + for index, item_score in enumerate(item_scores) + if item_score < threshold + ], + } + + class Rule_TC609_Composite(BaseRule): """Base class for a TC609 metric composed from existing Dingo rules.""" diff --git a/docs/metrics.md b/docs/metrics.md index 00c820c2..d407dbe4 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -168,7 +168,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `source` and `source_details`; validates non-empty traceability information and HTTP/HTTPS URL syntax when applicable. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses a local multilingual NLI model to check semantic consistency among string fields configured in `key_list`. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses multilingual text embeddings to check consistency among text items in `data_content`; multiple texts use robust-center aggregation instead of all-pairs comparison. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | diff --git a/docs/rules.md b/docs/rules.md index 3f57ae01..260c6058 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -95,7 +95,7 @@ The specific rules for each quality metric are as follows: | Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | -| Rule_TC609_0206_ContentConsistency | TC609_0206 | Combines structured-field and image-text consistency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0206_ContentConsistency | TC609_0206 | Checks consistency among text items in `data_content` using multilingual embeddings and robust-center aggregation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080101_TextPerplexity | TC609_02080101 | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080102_KnowledgeInformationDensity | TC609_02080102 | Combines alphabetic-word, stop-word, and unique-word ratio checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | diff --git a/examples/guobiao/rule_content_consistency.py b/examples/guobiao/rule_content_consistency.py index 6cac94e1..e5edb4c1 100644 --- a/examples/guobiao/rule_content_consistency.py +++ b/examples/guobiao/rule_content_consistency.py @@ -1,4 +1,4 @@ -"""Evaluate string fields using the national-standard content-consistency rule. +"""Evaluate text items using the national-standard content-consistency rule. Optional dependencies: conda run -n dingo pip install "dingo-python[hhem]" @@ -15,15 +15,31 @@ def main(): data = Data( data_id="guobiao-content-consistency-example", - title="高血压患者的日常健康管理", - content="高血压患者应遵医嘱规律用药，并定期监测血压。", - summary="高血压患者需要规律服药和监测血压。", + data_content=[ + { + "media_type": "text", + "content": "高血压患者的日常健康管理", + }, + { + "media_type": "text", + "content": ( + "高血压患者应遵医嘱规律用药，并定期监测血压。" + "日常生活中还应注意低盐饮食和适量运动。" + ), + }, + { + "media_type": "image", + "content": "../data/images/blood-pressure.jpg", + }, + ], ) Rule_TC609_0206_ContentConsistency.dynamic_config = EvaluatorRuleArgs( - key_list=["title", "content", "summary"], threshold=0.5, - model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", + model=( + "sentence-transformers/" + "paraphrase-multilingual-MiniLM-L12-v2" + ), device=-1, ) result = Rule_TC609_0206_ContentConsistency.eval(data) diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 05f7b77c..90083c33 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -8,6 +8,7 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.guobiao import rule_tc609_quality +from dingo.model.rule.guobiao import rule_tc609_quality_base from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance, Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0206_ContentConsistency, Rule_TC609_0208_ContentCleanliness, Rule_TC609_0301_ContentDiversity) @@ -693,105 +694,185 @@ def test_content_authenticity_declares_required_fields(): ] -def test_content_consistency_accepts_consistent_string_fields(monkeypatch): +def test_content_consistency_accepts_consistent_text_items(monkeypatch): monkeypatch.setattr( Rule_TC609_0206_ContentConsistency, "dynamic_config", EvaluatorRuleArgs( - key_list=["title", "content", "summary"], threshold=0.5, model="test-model", device=-1, ), ) - scores = iter([0.9, 0.8]) monkeypatch.setattr( - Rule_TC609_0206_ContentConsistency, - "_calculate_consistency_score", - classmethod(lambda cls, *args: next(scores)), + rule_tc609_quality, + "calculate_text_consistency", + lambda **kwargs: { + "score": 0.85, + "is_consistent": True, + "item_scores": [0.9, 0.85, 0.8], + "outlier_indexes": [], + }, ) result = Rule_TC609_0206_ContentConsistency.eval( - Data(title="健康", content="健康知识", summary="健康摘要") + Data( + data_content=[ + {"media_type": "text", "content": "海底管道砂袋防护"}, + {"media_type": "image", "content": "pipeline.jpg"}, + {"media_type": "text", "content": "砂袋用于保护海底管道"}, + {"media_type": "text", "content": "模拟砂袋周围流场"}, + ] + ) ) assert result.status is False - assert result.score == 0.8 + assert result.score == 0.85 assert result.label == [QualityLabel.QUALITY_GOOD] -def test_content_consistency_rejects_inconsistent_string_fields(monkeypatch): +def test_content_consistency_rejects_inconsistent_text_items(monkeypatch): monkeypatch.setattr( Rule_TC609_0206_ContentConsistency, "dynamic_config", EvaluatorRuleArgs( - key_list=["title", "content"], threshold=0.5, model="test-model", device=-1, ), ) monkeypatch.setattr( - Rule_TC609_0206_ContentConsistency, - "_calculate_consistency_score", - classmethod(lambda cls, *args: 0.2), + rule_tc609_quality, + "calculate_text_consistency", + lambda **kwargs: { + "score": 0.2, + "is_consistent": False, + "item_scores": [0.9, 0.2], + "outlier_indexes": [1], + }, ) result = Rule_TC609_0206_ContentConsistency.eval( - Data(title="健康", content="金融市场") + Data( + data_content=[ + {"media_type": "text", "content": "健康知识"}, + {"media_type": "image", "content": "health.jpg"}, + {"media_type": "text", "content": "金融市场"}, + ] + ) ) assert result.status is True assert result.score == 0.2 assert result.reason == [ - "title and content are inconsistent " - "(score: 0.2000, threshold: 0.5000)" + "Text items in data_content are inconsistent " + "(score: 0.2000, threshold: 0.5000, outlier indexes: [2])" ] +def test_content_consistency_skips_comparison_for_one_text_item(): + result = Rule_TC609_0206_ContentConsistency.eval( + Data( + data_content=[ + {"media_type": "text", "content": "文本内容"}, + {"media_type": "image", "content": "image.jpg"}, + ] + ) + ) + + assert result.status is False + assert result.score is None + assert result.label == [QualityLabel.QUALITY_GOOD] + + @pytest.mark.parametrize( - "data", + "data, reason", [ - Data(title="健康"), - Data(title="健康", content=["健康知识"]), - Data(title="健康", content=None), + ( + Data(data_content=[]), + "data_content: expected a non-empty list", + ), + ( + Data(data_content=["text"]), + "data_content[0]: expected dict, got str", + ), + ( + Data(data_content=[{"media_type": ["text"], "content": "文本"}]), + "data_content[0].media_type: expected a non-empty string", + ), + ( + Data(data_content=[{"media_type": "text", "content": ""}]), + ( + "data_content[0].content: " + "expected a non-empty string for text media" + ), + ), ], ) -def test_content_consistency_requires_existing_string_fields(data, monkeypatch): +def test_content_consistency_rejects_invalid_data_content(data, reason): + result = Rule_TC609_0206_ContentConsistency.eval(data) + + assert result.status is True + assert result.reason == [reason] + + +def test_calculate_text_consistency_compares_two_texts_directly(monkeypatch): + import torch + monkeypatch.setattr( - Rule_TC609_0206_ContentConsistency, - "dynamic_config", - EvaluatorRuleArgs( - key_list=["title", "content"], - threshold=0.5, - model="test-model", - device=-1, + rule_tc609_quality_base, + "_encode_texts", + lambda *args, **kwargs: torch.tensor( + [[1.0, 0.0], [0.8, 0.6]] ), ) - result = Rule_TC609_0206_ContentConsistency.eval(data) + result = rule_tc609_quality_base.calculate_text_consistency( + ["标题", "摘要"], + model_name="test-model", + threshold=0.75, + ) + + assert result["score"] == pytest.approx(0.8) + assert result["is_consistent"] is True + assert result["outlier_indexes"] == [] - assert result.status is True - assert result.reason == [ - "content: field must exist and its value must be str" - ] +def test_calculate_text_consistency_uses_robust_center(monkeypatch): + import torch -@pytest.mark.parametrize("key_list", [[], ["content"], ["content", "content"]]) -def test_content_consistency_rejects_invalid_key_list(key_list, monkeypatch): + encoded = torch.tensor( + [ + [1.0, 0.0], + [0.99, 0.1], + [0.98, -0.1], + [0.97, 0.05], + [0.0, 1.0], + ] + ) + encoded = torch.nn.functional.normalize(encoded, p=2, dim=1) monkeypatch.setattr( - Rule_TC609_0206_ContentConsistency, - "dynamic_config", - EvaluatorRuleArgs( - key_list=key_list, - threshold=0.5, - model="test-model", - device=-1, - ), + rule_tc609_quality_base, + "_encode_texts", + lambda *args, **kwargs: encoded, + ) + + result = rule_tc609_quality_base.calculate_text_consistency( + ["文本1", "文本2", "文本3", "文本4", "离群文本"], + model_name="test-model", + threshold=0.5, + consensus_keep_ratio=0.8, ) - with pytest.raises(ValueError, match="key_list"): - Rule_TC609_0206_ContentConsistency.eval(Data(content="example")) + assert result["score"] < 0.5 + assert result["is_consistent"] is False + assert result["outlier_indexes"] == [4] + + +def test_content_consistency_declares_data_content_required(): + assert Rule_TC609_0206_ContentConsistency._required_fields == [ + RequiredField.DATA_CONTENT + ] def test_uncovered_rule_is_explicit_placeholder(): From 500dbaa6bbe75e1b42fea4e370ca6babe2da1141 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 16:35:07 +0800 Subject: [PATCH 65/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0207=5FDataTypeCon?= =?UTF-8?q?sistency=20=E9=87=8D=E5=86=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 155 +++++++++++------- .../rule/guobiao/rule_tc609_quality_base.py | 7 + docs/metrics.md | 2 +- docs/rules.md | 2 +- examples/guobiao/rule_type_consistency.py | 11 +- test/scripts/model/rule/test_rule_common.py | 72 ++++++-- 6 files changed, 175 insertions(+), 74 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index c5950551..89147a1e 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -10,6 +10,7 @@ from dingo.model.rule.guobiao.rule_tc609_quality_base import ( Rule_TC609_01_DocCompleteness, Rule_TC609_Composite, + TC609_DATASET_TYPE_DESCRIPTIONS, _tc609_metric_info, _TC609PlaceholderBase, calculate_text_consistency, @@ -644,30 +645,18 @@ def eval(cls, input_data: Data) -> EvalDetail: @Model.rule_register("QUALITY_BAD_TC609_0207", ["guobiao_data"]) class Rule_TC609_0207_DataTypeConsistency(BaseRule): - """Check whether content belongs to the type declared in ``input_data.type``. + """Check whether text content matches the configured dataset type.""" - A local zero-shot classifier evaluates the hypothesis ``这段文本属于{type}类型``. - The declared type may be any non-empty string, such as ``医疗`` or ``金融``. - """ - - _metric_info = { - "category": "National Standard Data Quality Metrics", - "quality_dimension": "TYPE_CONSISTENCY", - "metric_name": "Rule_TC609_0207_DataTypeConsistency", - "description": ( - "Uses a local zero-shot classifier to check whether content belongs " - "to the type declared in the record" - ), - "paper_title": "High-quality dataset quality evaluation specification", - "paper_url": "", - "paper_authors": "SAC/TC609", - "evaluation_results": "", - "standard_code": "0207", - "coverage": "partial", - } + _metric_info = _tc609_metric_info( + "0207", + "Rule_TC609_0207_DataTypeConsistency", + "Checks whether text content matches the configured dataset type.", + "partial", + ) - _required_fields = [RequiredField.CONTENT, RequiredField.TYPE] + _required_fields = [RequiredField.DATA_CONTENT] dynamic_config = EvaluatorRuleArgs( + dataset_type="通识数据集", threshold=0.5, model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", device=-1, @@ -711,71 +700,125 @@ def _get_classifier(cls, model_name, device): return cls._classifier @classmethod - def _calculate_match_score( - cls, content, declared_type, model_name, device - ): + def _classify_dataset_type(cls, content, model_name, device): classifier = cls._get_classifier(model_name, device) + dataset_types = list(TC609_DATASET_TYPE_DESCRIPTIONS) + descriptions = [ + TC609_DATASET_TYPE_DESCRIPTIONS[dataset_type] + for dataset_type in dataset_types + ] result = classifier( content, - candidate_labels=[declared_type], - hypothesis_template="这段文本属于{}类型。", - multi_label=True, + candidate_labels=descriptions, + hypothesis_template="这段文本符合以下数据集类型要求：{}", + multi_label=False, truncation=True, ) labels = result.get("labels", []) scores = result.get("scores", []) - if not labels or not scores or labels[0] != declared_type: + if len(labels) != len(descriptions) or len(scores) != len(descriptions): raise RuntimeError("Zero-shot classifier returned an invalid result") - score = float(scores[0]) - if not math.isfinite(score) or not 0.0 <= score <= 1.0: - raise RuntimeError( - f"Zero-shot classifier returned an invalid score: {score}" - ) - return score + + scores_by_type = {} + for label, score in zip(labels, scores): + score = float(score) + if ( + label not in descriptions + or not math.isfinite(score) + or not 0.0 <= score <= 1.0 + ): + raise RuntimeError( + "Zero-shot classifier returned an invalid result" + ) + dataset_type = dataset_types[descriptions.index(label)] + scores_by_type[dataset_type] = score + if len(scores_by_type) != len(dataset_types): + raise RuntimeError("Zero-shot classifier returned an invalid result") + predicted_type = dataset_types[descriptions.index(labels[0])] + return predicted_type, scores_by_type @classmethod def eval(cls, input_data: Data) -> EvalDetail: res = EvalDetail(metric=cls.__name__) - declared_type = getattr(input_data, "type", None) - content = getattr(input_data, "content", None) + dataset_type = cls.dynamic_config.dataset_type + if dataset_type not in TC609_DATASET_TYPE_DESCRIPTIONS: + raise ValueError( + "Rule_TC609_0207_DataTypeConsistency " + "dynamic_config.dataset_type must be one of: " + f"{', '.join(TC609_DATASET_TYPE_DESCRIPTIONS)}" + ) + + threshold = cls.dynamic_config.threshold + if threshold is None or not 0 < threshold <= 1: + raise ValueError( + "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]" + ) - if not isinstance(declared_type, str) or not declared_type.strip(): + texts = [] + reasons = [] + if not isinstance(input_data.data_content, list): res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Data type is missing or empty"] + res.reason = [ + "data_content: expected list, " + f"got {type(input_data.data_content).__name__}" + ] return res + for index, item in enumerate(input_data.data_content): + if not isinstance(item, dict): + res.status = True + reasons.append( + f"data_content[{index}]: expected dict, " + f"got {type(item).__name__}" + ) + continue + media_type = item.get("media_type") + if not isinstance(media_type, str) or not media_type.strip(): + res.status = True + reasons.append( + f"data_content[{index}].media_type: " + "expected non-empty str" + ) + continue + if media_type.strip().lower() != "text": + continue + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + res.status = True + reasons.append( + f"data_content[{index}].content: expected non-empty str" + ) + continue + texts.append(content.strip()) - if not isinstance(content, str) or not content.strip(): + if not texts: res.status = True + reasons.append("data_content: at least one text item is required") + if res.status: res.label = [f"{cls.metric_type}.{cls.__name__}"] - res.reason = ["Content is missing or empty"] + res.reason = reasons return res - threshold = cls.dynamic_config.threshold - if threshold is None or not 0 < threshold <= 1: - raise ValueError( - "Rule_TC609_0207_DataTypeConsistency dynamic_config.threshold must be in (0, 1]" - ) - - model_name = cls.dynamic_config.model - device = cls.dynamic_config.device - score = cls._calculate_match_score( - content, declared_type, model_name, device + predicted_type, scores_by_type = cls._classify_dataset_type( + "\n".join(texts), + cls.dynamic_config.model, + cls.dynamic_config.device, ) - res.score = score + res.score = scores_by_type[dataset_type] - if score >= threshold: + if predicted_type == dataset_type and res.score >= threshold: res.label = [QualityLabel.QUALITY_GOOD] res.reason = [ - f"Content matches declared type {declared_type} " - f"(score: {score:.4f}, threshold: {threshold:.4f})" + f"Text content matches dataset type {dataset_type} " + f"(score: {res.score:.4f}, threshold: {threshold:.4f})" ] else: res.status = True res.label = [f"{cls.metric_type}.{cls.__name__}"] res.reason = [ - f"Content does not match declared type {declared_type} " - f"(score: {score:.4f}, threshold: {threshold:.4f})" + f"Text content does not match dataset type {dataset_type}; " + f"predicted type: {predicted_type} " + f"(score: {res.score:.4f}, threshold: {threshold:.4f})" ] return res diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index 3da2ee30..21f1c566 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -9,6 +9,13 @@ from dingo.model.rule.base import BaseRule +TC609_DATASET_TYPE_DESCRIPTIONS = { + "通识数据集": "面向普通公众，内容属于跨行业普遍适用、无需特定行业背景即可理解的通用知识", + "行业通识数据集": "面向特定行业，内容属于该行业从业者普遍需要掌握的基础知识、通用规范或常见实践", + "行业专识数据集": "面向特定行业的专业人员，内容包含需要行业专业背景才能理解或应用的专业概念、方法、技术或经验", +} + + def _tc609_metric_info(code, name, description, coverage): """Build consistent documentation metadata for TC609 rules.""" return { diff --git a/docs/metrics.md b/docs/metrics.md index d407dbe4..e7e92468 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -116,7 +116,7 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Checks whether text items in `data_content` match the configured dataset type | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | diff --git a/docs/rules.md b/docs/rules.md index 260c6058..ceb1f1fc 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -96,7 +96,7 @@ The specific rules for each quality metric are as follows: | Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0206_ContentConsistency | TC609_0206 | Checks consistency among text items in `data_content` using multilingual embeddings and robust-center aggregation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Checks whether text items in `data_content` match the configured dataset type | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080101_TextPerplexity | TC609_02080101 | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080102_KnowledgeInformationDensity | TC609_02080102 | Combines alphabetic-word, stop-word, and unique-word ratio checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_02080103_RepeatedContent | TC609_02080103 | Combines document-text and formula repetition checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | diff --git a/examples/guobiao/rule_type_consistency.py b/examples/guobiao/rule_type_consistency.py index 86d5dff3..b07aa07b 100644 --- a/examples/guobiao/rule_type_consistency.py +++ b/examples/guobiao/rule_type_consistency.py @@ -1,4 +1,4 @@ -"""Evaluate one Chinese text using the national-standard type-consistency rule. +"""Evaluate text content using the national-standard type-consistency rule. Optional dependencies: conda run -n dingo pip install "dingo-python[hhem]" @@ -15,11 +15,16 @@ def main(): data = Data( data_id="guobiao-type-example", - type="医疗", - content="高血压患者应在医生指导下规律用药，并定期监测血压变化。", + data_content=[ + { + "media_type": "text", + "content": "高血压患者应在医生指导下规律用药，并定期监测血压变化。", + } + ], ) Rule_TC609_0207_DataTypeConsistency.dynamic_config = EvaluatorRuleArgs( + dataset_type="行业通识数据集", threshold=0.5, model="MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", device=-1, diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 73556546..21f1c595 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -213,32 +213,54 @@ def test_missing_dependencies_raise_clear_error(self, monkeypatch): class TestRule_TC609_0207_DataTypeConsistency: @staticmethod - def _mock_match_score(monkeypatch, score): + def _mock_classification(monkeypatch, predicted_type, scores): monkeypatch.setattr( Rule_TC609_0207_DataTypeConsistency, - "_calculate_match_score", - classmethod(lambda cls, *args: score), + "_classify_dataset_type", + classmethod(lambda cls, *args: (predicted_type, scores)), ) monkeypatch.setattr( Rule_TC609_0207_DataTypeConsistency, "dynamic_config", - EvaluatorRuleArgs(threshold=0.6, model="test-model", device=-1), + EvaluatorRuleArgs( + dataset_type="通识数据集", + threshold=0.6, + model="test-model", + device=-1, + ), ) - def test_content_matching_declared_type_is_good(self, monkeypatch): - self._mock_match_score(monkeypatch, 0.85) + def test_text_matching_dataset_type_is_good(self, monkeypatch): + self._mock_classification( + monkeypatch, + "通识数据集", + {"通识数据集": 0.85, "行业通识数据集": 0.1, "行业专识数据集": 0.05}, + ) result = Rule_TC609_0207_DataTypeConsistency.eval( - Data(data_id="type-match", type="medical", content="Clinical treatment") + Data( + data_id="type-match", + data_content=[ + {"media_type": "text", "content": "普通生活常识"}, + {"media_type": "image", "content": "image.png"}, + ], + ) ) assert result.status is False assert result.score == 0.85 assert result.label == [QualityLabel.QUALITY_GOOD] - def test_content_not_matching_declared_type_is_bad(self, monkeypatch): - self._mock_match_score(monkeypatch, 0.25) + def test_text_not_matching_dataset_type_is_bad(self, monkeypatch): + self._mock_classification( + monkeypatch, + "行业专识数据集", + {"通识数据集": 0.25, "行业通识数据集": 0.2, "行业专识数据集": 0.55}, + ) result = Rule_TC609_0207_DataTypeConsistency.eval( - Data(data_id="type-mismatch", type="medical", content="Stock prices") + Data( + data_id="type-mismatch", + data_content=[{"media_type": "text", "content": "专业行业知识"}], + ) ) assert result.status is True @@ -247,13 +269,37 @@ def test_content_not_matching_declared_type_is_bad(self, monkeypatch): "QUALITY_BAD_TC609_0207.Rule_TC609_0207_DataTypeConsistency" ] - def test_missing_type_is_bad(self): + def test_no_text_content_is_bad(self): result = Rule_TC609_0207_DataTypeConsistency.eval( - Data(data_id="type-missing", content="Ordinary text") + Data( + data_id="no-text", + data_content=[{"media_type": "image", "content": "image.png"}], + ) ) assert result.status is True - assert "missing or empty" in result.reason[0] + assert "at least one text item" in result.reason[0] + + def test_invalid_dataset_type_raises_value_error(self, monkeypatch): + monkeypatch.setattr( + Rule_TC609_0207_DataTypeConsistency, + "dynamic_config", + EvaluatorRuleArgs( + dataset_type="医疗数据集", + threshold=0.6, + model="test-model", + device=-1, + ), + ) + with pytest.raises(ValueError, match="dataset_type must be one of"): + Rule_TC609_0207_DataTypeConsistency.eval( + Data( + data_id="invalid-type", + data_content=[ + {"media_type": "text", "content": "医疗知识"} + ], + ) + ) class TestRule_TC609_0303_DataTimeRange: From 97092e627ee367b6be18972dc34767c370733148 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 16:44:15 +0800 Subject: [PATCH 66/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0208=5FContentClea?= =?UTF-8?q?nliness=20=E6=9B=B4=E6=96=B0=E8=BE=93=E5=85=A5=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 65 ++++++++++++++++++- .../model/rule/test_rule_tc609_quality.py | 57 +++++++++++++++- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 89147a1e..9ff72384 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -827,7 +827,15 @@ def eval(cls, input_data: Data) -> EvalDetail: class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): """0208: Content cleanliness, composed from available cleaning rules.""" - dynamic_config = EvaluatorRuleArgs(key_list=[]) + dynamic_config = EvaluatorRuleArgs( + key_list=[ + "版权所有", + "Copyright", + "未经授权不得转载", + "禁止转载", + "仅供学习交流", + ] + ) component_rules = ( "dingo.model.rule.rule_common.RuleAbnormalChar", "dingo.model.rule.rule_common.RuleAbnormalHtml", @@ -835,7 +843,7 @@ class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): "dingo.model.rule.rule_common.RuleContentNull", "dingo.model.rule.rule_common.RuleWatermark", ) - _required_fields = [RequiredField.CONTENT] + _required_fields = [RequiredField.DATA_CONTENT] _metric_info = _tc609_metric_info( "0208", "Rule_TC609_0208_ContentCleanliness", @@ -845,11 +853,62 @@ class Rule_TC609_0208_ContentCleanliness(Rule_TC609_Composite): @classmethod def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + reasons = [] + if not isinstance(input_data.data_content, list): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + "data_content: expected list, " + f"got {type(input_data.data_content).__name__}" + ] + return res + + texts = [] + for index, item in enumerate(input_data.data_content): + if not isinstance(item, dict): + res.status = True + reasons.append( + f"data_content[{index}]: expected dict, " + f"got {type(item).__name__}" + ) + continue + media_type = item.get("media_type") + if not isinstance(media_type, str) or not media_type.strip(): + res.status = True + reasons.append( + f"data_content[{index}].media_type: expected non-empty str" + ) + continue + if media_type.strip().lower() != "text": + continue + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + res.status = True + reasons.append( + f"data_content[{index}].content: expected non-empty str" + ) + continue + texts.append(content.strip()) + + if not texts: + res.status = True + reasons.append( + "data_content: at least one text item is required" + ) + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + return res + rule_watermark = cls._resolve_rule(cls.component_rules[-1]) rule_watermark.dynamic_config = EvaluatorRuleArgs( key_list=cls.dynamic_config.key_list or [], ) - return super().eval(input_data) + text_input = input_data.model_copy( + update={"content": "\n".join(texts)} + ) + return super().eval(text_input) @Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"]) diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index 90083c33..c27d6a81 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -470,7 +470,13 @@ def test_content_cleanliness_propagates_empty_watermark_config(monkeypatch): ValueError, match="RuleWatermark requires non-empty dynamic_config.key_list", ): - Rule_TC609_0208_ContentCleanliness.eval(Data(content="safe text")) + Rule_TC609_0208_ContentCleanliness.eval( + Data( + data_content=[ + {"media_type": "text", "content": "safe text"} + ] + ) + ) def test_content_cleanliness_passes_key_list_to_watermark(monkeypatch): @@ -486,7 +492,13 @@ def test_content_cleanliness_passes_key_list_to_watermark(monkeypatch): ) result = Rule_TC609_0208_ContentCleanliness.eval( - Data(content="text with DINGO-WATERMARK") + Data( + data_content=[ + {"media_type": "text", "content": "text with"}, + {"media_type": "image", "content": "DINGO-WATERMARK"}, + {"media_type": "text", "content": "DINGO-WATERMARK"}, + ] + ) ) assert RuleWatermark.dynamic_config.key_list == ["DINGO-WATERMARK"] @@ -502,13 +514,52 @@ def test_content_cleanliness_returns_good_without_watermark(monkeypatch): ) result = Rule_TC609_0208_ContentCleanliness.eval( - Data(content="ordinary clean text") + Data( + data_content=[ + {"media_type": "text", "content": "ordinary"}, + {"media_type": "text", "content": "clean text"}, + {"media_type": "image", "content": "DINGO-WATERMARK"}, + ] + ) ) assert result.status is False assert result.label == [QualityLabel.QUALITY_GOOD] +def test_content_cleanliness_has_usable_default_watermarks(monkeypatch): + assert Rule_TC609_0208_ContentCleanliness.dynamic_config.key_list + + result = Rule_TC609_0208_ContentCleanliness.eval( + Data( + data_content=[ + { + "media_type": "text", + "content": "本文版权所有，未经授权不得转载。", + } + ] + ) + ) + + assert result.status is True + assert "RuleWatermark: 版权所有" in result.reason + + +def test_content_cleanliness_requires_text_content(): + result = Rule_TC609_0208_ContentCleanliness.eval( + Data( + data_content=[ + {"media_type": "image", "content": "image.png"} + ] + ) + ) + + assert result.status is True + assert result.reason == [ + "data_content: at least one text item is required" + ] + + def test_structural_completeness_accepts_present_values(monkeypatch): monkeypatch.setattr( Rule_TC609_0204_StructuralCompleteness, From 522e23aa9e5727f6466f6fa6858ff20df9689df2 Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 16:50:47 +0800 Subject: [PATCH 67/80] =?UTF-8?q?feat:=20Rule=5FTC609=5F0202=5FSafetyCompl?= =?UTF-8?q?iance=20=E6=9B=B4=E6=96=B0=E8=BE=93=E5=85=A5=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 55 ++++++++++++++++++- docs/metrics.md | 2 +- docs/rules.md | 2 +- .../model/rule/test_rule_tc609_quality.py | 43 ++++++++++++++- 4 files changed, 95 insertions(+), 7 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 9ff72384..bb06a5ea 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -285,7 +285,7 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): "dingo.model.rule.rule_common.RulePIIDetection", "dingo.model.rule.rule_common.RuleIDCard", ) - _required_fields = [RequiredField.CONTENT] + _required_fields = [RequiredField.DATA_CONTENT] _metric_info = _tc609_metric_info( "0202", "Rule_TC609_0202_SafetyCompliance", @@ -295,6 +295,54 @@ class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite): @classmethod def eval(cls, input_data: Data) -> EvalDetail: + res = EvalDetail(metric=cls.__name__) + reasons = [] + if not isinstance(input_data.data_content, list): + res.status = True + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = [ + "data_content: expected list, " + f"got {type(input_data.data_content).__name__}" + ] + return res + + texts = [] + for index, item in enumerate(input_data.data_content): + if not isinstance(item, dict): + res.status = True + reasons.append( + f"data_content[{index}]: expected dict, " + f"got {type(item).__name__}" + ) + continue + media_type = item.get("media_type") + if not isinstance(media_type, str) or not media_type.strip(): + res.status = True + reasons.append( + f"data_content[{index}].media_type: expected non-empty str" + ) + continue + if media_type.strip().lower() != "text": + continue + content = item.get("content") + if not isinstance(content, str) or not content.strip(): + res.status = True + reasons.append( + f"data_content[{index}].content: expected non-empty str" + ) + continue + texts.append(content.strip()) + + if not texts: + res.status = True + reasons.append( + "data_content: at least one text item is required" + ) + if res.status: + res.label = [f"{cls.metric_type}.{cls.__name__}"] + res.reason = reasons + return res + rule_unsafe_words = cls._resolve_rule(cls.component_rules[0]) unsafe_words_config = EvaluatorRuleArgs( key_list=cls.dynamic_config.key_list or [], @@ -307,7 +355,10 @@ def eval(cls, input_data: Data) -> EvalDetail: rule_unsafe_words._unsafe_words_list = None rule_unsafe_words._unsafe_words_automaton = None rule_unsafe_words.dynamic_config = unsafe_words_config - return super().eval(input_data) + text_input = input_data.model_copy( + update={"content": "\n".join(texts)} + ) + return super().eval(text_input) @Model.rule_register("QUALITY_BAD_TC609_0203", ["guobiao_data"]) diff --git a/docs/metrics.md b/docs/metrics.md index e7e92468..e474dcd1 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -163,7 +163,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks TC609 annotation metadata fields, types, and enumerated values. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `source` and `source_details`; validates non-empty traceability information and HTTP/HTTPS URL syntax when applicable. | Internal Implementation | N/A | N/A | diff --git a/docs/rules.md b/docs/rules.md index ceb1f1fc..20122ff2 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -91,7 +91,7 @@ The specific rules for each quality metric are as follows: | Rule_TC609_0103_DocConstructionProcessCompleteness | TC609_0103 | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0104_DocApplicationCompleteness | TC609_0104 | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0201_FormatCompliance | TC609_0201 | Combines existing NLP, SFT, image, audio, and video format rules. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | | Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index c27d6a81..a46b3789 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -310,7 +310,10 @@ def eval(cls, input_data): ) result = Rule_TC609_0202_SafetyCompliance.eval( - Data(data_id="safety", content="test") + Data( + data_id="safety", + data_content=[{"media_type": "text", "content": "test"}], + ) ) assert result.status is True @@ -365,7 +368,13 @@ def eval(cls, input_data): ) result = Rule_TC609_0202_SafetyCompliance.eval( - Data(content="unsafe") + Data( + data_content=[ + {"media_type": "text", "content": "safe"}, + {"media_type": "image", "content": "unsafe"}, + {"media_type": "text", "content": "unsafe"}, + ] + ) ) assert UnsafeWordsRule.dynamic_config.key_list == ["unsafe"] @@ -381,13 +390,41 @@ def test_safety_compliance_has_usable_default_words(monkeypatch): monkeypatch.setattr(RuleUnsafeWords, "_unsafe_words_automaton", None) result = Rule_TC609_0202_SafetyCompliance.eval( - Data(content="该内容提供制作炸弹的具体步骤") + Data( + data_content=[ + { + "media_type": "text", + "content": "该内容提供制作炸弹的具体步骤", + } + ] + ) ) assert result.status is True assert "RuleUnsafeWords: 制作炸弹" in result.reason +def test_safety_compliance_requires_text_content(): + result = Rule_TC609_0202_SafetyCompliance.eval( + Data( + data_content=[ + {"media_type": "image", "content": "制作炸弹"} + ] + ) + ) + + assert result.status is True + assert result.reason == [ + "data_content: at least one text item is required" + ] + + +def test_safety_compliance_declares_data_content(): + assert Rule_TC609_0202_SafetyCompliance._required_fields == [ + RequiredField.DATA_CONTENT + ] + + def test_annotation_compliance_accepts_valid_metadata(): result = Rule_TC609_0203_AnnotationCompliance.eval( Data( From a5ed48f7a7b53482fd9384d2a82402750ea7535d Mon Sep 17 00:00:00 2001 From: shijin Date: Fri, 31 Jul 2026 18:52:31 +0800 Subject: [PATCH 68/80] =?UTF-8?q?feat:=20=E5=9B=BD=E6=A0=87=20LLM=200101-0?= =?UTF-8?q?104?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dingo/model/llm/guobiao/__init__.py | 1 + ..._tc609_0101_doc_basic_info_completeness.py | 46 +++++++++++++++ ...9_0102_doc_content_feature_completeness.py | 46 +++++++++++++++ ...3_doc_construction_process_completeness.py | 46 +++++++++++++++ ...tc609_0104_doc_application_completeness.py | 46 +++++++++++++++ docs/metrics.md | 9 +++ .../model/llm/test_tc609_doc_completeness.py | 56 +++++++++++++++++++ 7 files changed, 250 insertions(+) create mode 100644 dingo/model/llm/guobiao/__init__.py create mode 100644 dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py create mode 100644 dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py create mode 100644 dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py create mode 100644 dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py create mode 100644 test/scripts/model/llm/test_tc609_doc_completeness.py diff --git a/dingo/model/llm/guobiao/__init__.py b/dingo/model/llm/guobiao/__init__.py new file mode 100644 index 00000000..82e64589 --- /dev/null +++ b/dingo/model/llm/guobiao/__init__.py @@ -0,0 +1 @@ +"""LLM evaluators for TC609 high-quality dataset documentation.""" diff --git a/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py new file mode 100644 index 00000000..9ab77dcb --- /dev/null +++ b/dingo/model/llm/guobiao/llm_tc609_0101_doc_basic_info_completeness.py @@ -0,0 +1,46 @@ +from dingo.io.input import RequiredField +from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI + + +@Model.llm_register("LLM_TC609_0101_DocBasicInfoCompleteness") +class LLM_TC609_0101_DocBasicInfoCompleteness(BaseOpenAI): + """Evaluate completeness of basic dataset information.""" + + _required_fields = [RequiredField.CONTENT] + _metric_info = { + "category": "National Standard LLM Assessment Metrics", + "metric_name": "LLM_TC609_0101_DocBasicInfoCompleteness", + "description": ( + "Uses an LLM to assess dataset scale, format, file structure, " + "access channel, and technical support in dataset documentation." + ), + "paper_title": "TC609 high-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "examples": "examples/guobiao/example_doc3.py", + } + prompt = """ +# 角色 +你是高质量数据集说明文档审核专家。请评估文档的基本信息完整性（0101）。 + +# 检查事项 +1. 数据集规模：说明样本数量、记录数量、文件数量、存储规模等可核实的规模信息。 +2. 格式规范：说明文件格式、编码、字段结构或解析方式。 +3. 文件结构：说明目录、文件组成及其组织关系。 +4. 访问渠道：说明数据集的获取、下载或访问方式。 +5. 技术支持：说明问题反馈、维护渠道或技术支持方式。 + +# 判定规则 +逐项寻找明确证据。同义词、近义表达、表格、目录树和代码示例均可作为证据，不要求出现固定标题或关键词。 +只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。 +共5项；明确覆盖至少4项时score为1，否则为0。 +reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。 + +# 输出格式 +只输出合法JSON，不要输出Markdown或其他文字： +{"score": 0, "reason": "covered: ...; missing: ..."} + +# 待评估文档 +""" diff --git a/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py new file mode 100644 index 00000000..2314baba --- /dev/null +++ b/dingo/model/llm/guobiao/llm_tc609_0102_doc_content_feature_completeness.py @@ -0,0 +1,46 @@ +from dingo.io.input import RequiredField +from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI + + +@Model.llm_register("LLM_TC609_0102_DocContentFeatureCompleteness") +class LLM_TC609_0102_DocContentFeatureCompleteness(BaseOpenAI): + """Evaluate completeness of dataset content features.""" + + _required_fields = [RequiredField.CONTENT] + _metric_info = { + "category": "National Standard LLM Assessment Metrics", + "metric_name": "LLM_TC609_0102_DocContentFeatureCompleteness", + "description": ( + "Uses an LLM to assess modality, distribution, label statistics, " + "sample examples, and limitations in dataset documentation." + ), + "paper_title": "TC609 high-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "examples": "examples/guobiao/example_doc3.py", + } + prompt = """ +# 角色 +你是高质量数据集说明文档审核专家。请评估文档的内容特征完整性（0102）。 + +# 检查事项 +1. 模态类型：说明数据包含文本、图像、音频、视频或其他模态。 +2. 数据分布：说明领域、类别、语言、时间或其他维度的数据分布。 +3. 标签统计：说明标签类别及各类别的数量、占比或分布情况。 +4. 样本示例：给出能够代表实际记录结构和内容的数据样例。 +5. 局限性：说明覆盖范围、规模、偏差、适用边界或已知不足。 + +# 判定规则 +逐项寻找明确证据。同义词、近义表达、表格和示例均可作为证据，不要求出现固定标题或关键词。 +只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。 +共5项；明确覆盖至少4项时score为1，否则为0。 +reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。 + +# 输出格式 +只输出合法JSON，不要输出Markdown或其他文字： +{"score": 0, "reason": "covered: ...; missing: ..."} + +# 待评估文档 +""" diff --git a/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py new file mode 100644 index 00000000..f9127fcf --- /dev/null +++ b/dingo/model/llm/guobiao/llm_tc609_0103_doc_construction_process_completeness.py @@ -0,0 +1,46 @@ +from dingo.io.input import RequiredField +from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI + + +@Model.llm_register("LLM_TC609_0103_DocConstructionProcessCompleteness") +class LLM_TC609_0103_DocConstructionProcessCompleteness(BaseOpenAI): + """Evaluate completeness of the dataset construction process.""" + + _required_fields = [RequiredField.CONTENT] + _metric_info = { + "category": "National Standard LLM Assessment Metrics", + "metric_name": "LLM_TC609_0103_DocConstructionProcessCompleteness", + "description": ( + "Uses an LLM to assess source, collection, processing, annotation, " + "and version control in dataset documentation." + ), + "paper_title": "TC609 high-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "examples": "examples/guobiao/example_doc3.py", + } + prompt = """ +# 角色 +你是高质量数据集说明文档审核专家。请评估文档的数据集构建过程完整性（0103）。 + +# 检查事项 +1. 数据来源：说明数据来自何处，包括人工编写、公开数据、业务系统等来源。 +2. 采集方法：说明数据如何采集、收集、生成或选取。 +3. 加工处理流程：说明清洗、转换、去重、审核、质量控制等处理步骤。 +4. 标注规范：说明标签定义、标注方式、标注人员或一致性要求。 +5. 版本控制：说明版本号、发布日期、变更记录或版本管理方式。 + +# 判定规则 +逐项寻找明确证据。同义词、近义表达、流程列表和表格均可作为证据，不要求出现固定标题或关键词。 +只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。 +共5项；明确覆盖至少4项时score为1，否则为0。 +reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。 + +# 输出格式 +只输出合法JSON，不要输出Markdown或其他文字： +{"score": 0, "reason": "covered: ...; missing: ..."} + +# 待评估文档 +""" diff --git a/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py b/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py new file mode 100644 index 00000000..3f3d2084 --- /dev/null +++ b/dingo/model/llm/guobiao/llm_tc609_0104_doc_application_completeness.py @@ -0,0 +1,46 @@ +from dingo.io.input import RequiredField +from dingo.model import Model +from dingo.model.llm.base_openai import BaseOpenAI + + +@Model.llm_register("LLM_TC609_0104_DocApplicationCompleteness") +class LLM_TC609_0104_DocApplicationCompleteness(BaseOpenAI): + """Evaluate completeness of dataset application information.""" + + _required_fields = [RequiredField.CONTENT] + _metric_info = { + "category": "National Standard LLM Assessment Metrics", + "metric_name": "LLM_TC609_0104_DocApplicationCompleteness", + "description": ( + "Uses an LLM to assess license, target scenarios, evaluation " + "method, benchmark results, and typical cases." + ), + "paper_title": "TC609 high-quality dataset quality evaluation specification", + "paper_url": "", + "paper_authors": "SAC/TC609", + "evaluation_results": "", + "examples": "examples/guobiao/example_doc3.py", + } + prompt = """ +# 角色 +你是高质量数据集说明文档审核专家。请评估文档的应用说明完整性（0104）。 + +# 检查事项 +1. 使用许可：说明许可证、授权协议或使用限制。 +2. 目标应用场景：说明适用和不适用的任务、用户或业务场景。 +3. 评估方法：说明如何评测该数据集或如何验证其质量。 +4. 基准结果：给出评测结果、基线结果，或明确说明不提供基准及其原因。 +5. 典型应用案例：说明一个具体使用流程、示例任务或实际应用案例。 + +# 判定规则 +逐项寻找明确证据。同义词、近义表达、操作步骤和示例均可作为证据，不要求出现固定标题或关键词。 +只能依据输入文档判断，不得根据常识补全。仅出现字段名称但没有说明实际数据集情况，不算完整。 +共5项；明确覆盖至少4项时score为1，否则为0。 +reason必须简洁列出已覆盖事项及证据，以及缺失或说明不足的事项。 + +# 输出格式 +只输出合法JSON，不要输出Markdown或其他文字： +{"score": 0, "reason": "covered: ...; missing: ..."} + +# 待评估文档 +""" diff --git a/docs/metrics.md b/docs/metrics.md index e474dcd1..4e2bf864 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -30,6 +30,15 @@ This document provides comprehensive information about all quality metrics used | `LLMTextQualityV5` | LLMTextQualityV5 | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | | `LLMTextTable` | LLMTextTable | Impact-driven text quality evaluation for LLM pretraining, focusing on structural completeness, readability, diversit... | [WanJuanSiLu: A High-Quality Open-Source Webtext Dataset for Low-Resource Languages](https://arxiv.org/abs/2501.14506) (Yu et al., 2025) | [📊 See Results](eval/prompt/redpajama_data_evaluated_by_prompt.md) | [📝 View Example](../examples/llm_and_rule/llm_local.py) | +### National Standard LLM Assessment Metrics + +| Type | Metric | Description | Source | Evaluation Results | Examples | +|------|--------|-------------|--------|-------------------|----------| +| `LLM_TC609_0101_DocBasicInfoCompleteness` | LLM_TC609_0101_DocBasicInfoCompleteness | Uses an LLM to assess dataset scale, format, file structure, access channel, and technical support in dataset documentation. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) | +| `LLM_TC609_0102_DocContentFeatureCompleteness` | LLM_TC609_0102_DocContentFeatureCompleteness | Uses an LLM to assess modality, distribution, label statistics, sample examples, and limitations. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) | +| `LLM_TC609_0103_DocConstructionProcessCompleteness` | LLM_TC609_0103_DocConstructionProcessCompleteness | Uses an LLM to assess source, collection, processing, annotation, and version control. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) | +| `LLM_TC609_0104_DocApplicationCompleteness` | LLM_TC609_0104_DocApplicationCompleteness | Uses an LLM to assess license, target scenarios, evaluation method, benchmark results, and typical cases. | TC609 | N/A | [📝 View Example](../examples/guobiao/example_doc3.py) | + ### SFT Data Assessment Metrics | Type | Metric | Description | Paper Source | Evaluation Results | Examples | diff --git a/test/scripts/model/llm/test_tc609_doc_completeness.py b/test/scripts/model/llm/test_tc609_doc_completeness.py new file mode 100644 index 00000000..8b6a7942 --- /dev/null +++ b/test/scripts/model/llm/test_tc609_doc_completeness.py @@ -0,0 +1,56 @@ +from dingo.io.input import Data +from dingo.model.llm.guobiao.llm_tc609_0101_doc_basic_info_completeness import ( + LLM_TC609_0101_DocBasicInfoCompleteness, +) +from dingo.model.llm.guobiao.llm_tc609_0102_doc_content_feature_completeness import ( + LLM_TC609_0102_DocContentFeatureCompleteness, +) +from dingo.model.llm.guobiao.llm_tc609_0103_doc_construction_process_completeness import ( + LLM_TC609_0103_DocConstructionProcessCompleteness, +) +from dingo.model.llm.guobiao.llm_tc609_0104_doc_application_completeness import ( + LLM_TC609_0104_DocApplicationCompleteness, +) + + +TC609_LLM_CLASSES = [ + LLM_TC609_0101_DocBasicInfoCompleteness, + LLM_TC609_0102_DocContentFeatureCompleteness, + LLM_TC609_0103_DocConstructionProcessCompleteness, + LLM_TC609_0104_DocApplicationCompleteness, +] + + +def test_tc609_doc_llm_prompts_define_binary_json_output(): + for evaluator in TC609_LLM_CLASSES: + assert '"score": 0' in evaluator.prompt + assert "至少 4 项" in evaluator.prompt + assert "同义词、近义表达" in evaluator.prompt + assert evaluator._required_fields + + +def test_tc609_doc_llm_build_messages_contains_document(): + document = "这是一份数据集说明文档。" + for evaluator in TC609_LLM_CLASSES: + messages = evaluator.build_messages(Data(content=document)) + assert messages == [ + {"role": "user", "content": evaluator.prompt + document} + ] + + +def test_tc609_doc_llm_process_response_pass_and_fail(): + evaluator = LLM_TC609_0101_DocBasicInfoCompleteness + + passed = evaluator.process_response( + '{"score": 1, "reason": "covered: 5项; missing: 无"}' + ) + assert passed.status is False + assert passed.reason == ["covered: 5项; missing: 无"] + + failed = evaluator.process_response( + '{"score": 0, "reason": "covered: 3项; missing: 访问渠道、技术支持"}' + ) + assert failed.status is True + assert failed.reason == [ + "covered: 3项; missing: 访问渠道、技术支持" + ] From a8ee1c07cefb7f3e7bd7a67c35fce505070c5c09 Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 31 Jul 2026 19:57:52 +0800 Subject: [PATCH 69/80] =?UTF-8?q?fix:=20=E7=A7=BB=E9=99=A4=E5=85=A8?= =?UTF-8?q?=E5=B1=80=20HF=20=E9=95=9C=E5=83=8F=E6=B3=A8=E5=85=A5=EF=BC=8C?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=96=87=E6=A1=A3=E6=8C=89=E9=9C=80=E6=8C=87?= =?UTF-8?q?=E5=AF=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 之前 dingo/__init__.py 用 os.environ.setdefault 把 HF_ENDPOINT 默认 指向 hf-mirror.com，会对所有未显式设置该变量的环境生效，导致 CI （美国 runner）被强制走中国镜像、加载 HF 数据集 chupei/format-text 时跨境连接超时，报 LocalEntryNotFoundError 使 Integration Test 失败。 改为在幻觉检测中文文档中指导用户按需 export HF_ENDPOINT，不侵入全局 环境，既满足国内用户走镜像的需求，也不影响能直连官网的环境（如 CI）。 Co-Authored-By: Claude Opus 4.8 --- dingo/__init__.py | 7 ------- docs/hallucination_guide.md | 13 +++++++++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dingo/__init__.py b/dingo/__init__.py index 0ec17bd2..e69de29b 100644 --- a/dingo/__init__.py +++ b/dingo/__init__.py @@ -1,7 +0,0 @@ -import os - -# 为无法访问 huggingface.co 的环境提供默认镜像。 -# 必须在任何 huggingface_hub / transformers 被导入之前设置， -# 因为 HF_ENDPOINT 在 huggingface_hub 导入时即被固定读取。 -# 使用 setdefault 保证用户可通过外部环境变量覆盖为其他镜像或官方地址。 -os.environ.setdefault("HF_ENDPOINT", "https://hf-mirror.com") diff --git a/docs/hallucination_guide.md b/docs/hallucination_guide.md index 2ca58899..521a12a4 100644 --- a/docs/hallucination_guide.md +++ b/docs/hallucination_guide.md @@ -67,6 +67,19 @@ pip install transformers torch pip install -r requirements/hhem_integration.txt ``` +#### 模型下载与镜像 + +首次运行会自动从 Hugging Face 下载 HHEM-2.1-Open 模型（约 400MB），之后从本地缓存加载，无需重复下载。 + +如果无法访问 `huggingface.co`（如国内网络），可在运行前设置镜像环境变量，让下载走 [hf-mirror.com](https://hf-mirror.com)： + +```bash +# 设置镜像后再运行 dingo（对 huggingface_hub / transformers / datasets 全部生效） +export HF_ENDPOINT=https://hf-mirror.com +``` + +> 说明：Dingo 不会强制修改该变量，以免影响能直连官网的环境（如 CI）；是否使用镜像由你自行控制。也可用 `huggingface-cli download vectara/hallucination_evaluation_model` 提前手动下载。 + #### 基本使用 ```python From 6590bec4b94db4e9e2333ba0b273aa4554020880 Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 31 Jul 2026 20:34:24 +0800 Subject: [PATCH 70/80] =?UTF-8?q?fix:=20HHEM=20=E5=9B=9E=E5=A1=AB=20score?= =?UTF-8?q?=20=E5=B9=B6=E4=BF=AE=E5=A4=8D=E7=A4=BA=E4=BE=8B=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=20None=20=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RuleHallucinationHHEM.eval 里 result.score 赋值一直是注释状态，导致 EvalDetail.score 恒为 None；而示例脚本用 {getattr(result,'score','N/A'):.3f} 格式化——因 score 属性存在只是值为 None，getattr 默认值不生效，最终 None:.3f 触发 "unsupported format string passed to NoneType.__format__"。 - rule_hallucination_hhem.py: 取消注释 result.score = avg_hallucination_score （位于 if/else 之前，HALLUCINATION_DETECTED 与 NO_HALLUCINATION 两分支都回填） - sdk_rule_hhem_detection.py: 6 处格式化改为对 None 安全（有值 .3f，None 显示 N/A） Co-Authored-By: Claude Opus 4.8 --- dingo/model/rule/rule_hallucination_hhem.py | 2 +- examples/hallucination/sdk_rule_hhem_detection.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dingo/model/rule/rule_hallucination_hhem.py b/dingo/model/rule/rule_hallucination_hhem.py index a9fd3ff8..6e61ec17 100644 --- a/dingo/model/rule/rule_hallucination_hhem.py +++ b/dingo/model/rule/rule_hallucination_hhem.py @@ -163,7 +163,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # Create result result = EvalDetail(metric=cls.__name__) - # result.score = avg_hallucination_score + result.score = avg_hallucination_score # Determine if hallucination detected based on threshold if avg_hallucination_score > cls.dynamic_config.threshold: diff --git a/examples/hallucination/sdk_rule_hhem_detection.py b/examples/hallucination/sdk_rule_hhem_detection.py index 576fbdc6..070720f5 100644 --- a/examples/hallucination/sdk_rule_hhem_detection.py +++ b/examples/hallucination/sdk_rule_hhem_detection.py @@ -35,7 +35,7 @@ def example_1_basic_rule_hhem_detection(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}") + print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print(f"Threshold: {RuleHallucinationHHEM.dynamic_config.threshold}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") @@ -61,7 +61,7 @@ def example_2_no_hallucination_rule(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}") + print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") print() @@ -89,7 +89,7 @@ def example_3_complex_scenario_rule(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {getattr(result, 'score', 'N/A'):.3f}") + print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") print() @@ -150,7 +150,7 @@ def example_5_batch_evaluation_rule(): print("Batch Rule-based Evaluation Results:") for i, result in enumerate(results): - print(f" Item {i + 1}: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}") + print(f" Item {i + 1}: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}") print() @@ -176,7 +176,7 @@ def example_6_threshold_comparison_rule(): RuleHallucinationHHEM.dynamic_config.threshold = threshold result = RuleHallucinationHHEM.eval(data) - print(f"Threshold {threshold}: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}") + print(f"Threshold {threshold}: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}") # Restore original threshold RuleHallucinationHHEM.dynamic_config.threshold = original_threshold @@ -205,7 +205,7 @@ def example_7_performance_benchmark_rule(): end_time = time.time() print(f"Rule-based HHEM Inference Time: {end_time - start_time:.3f} seconds") - print(f"Result: Error={result.status}, Score={getattr(result, 'score', 'N/A'):.3f}") + print(f"Result: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}") print(f"Model Info: Local HHEM-2.1-Open (Rule-based)") print() From b2aa6bc9cedf307bbaf6c43344cfd7ea046f4d5f Mon Sep 17 00:00:00 2001 From: chupei Date: Fri, 31 Jul 2026 20:36:56 +0800 Subject: [PATCH 71/80] =?UTF-8?q?fix:=20=E9=9A=94=E7=A6=BB=20lmdeploy=20?= =?UTF-8?q?=E5=B9=B6=E9=99=90=E5=88=B6=20transformers<4.49=20=E4=BB=A5?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20HHEM=20=E5=8A=A0=E8=BD=BD=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dingo-saas 运行 RuleHallucinationHHEM 报 "'HHEMv2ForSequenceClassification' object has no attribute 'all_tied_weights_keys'"。 根因：lmdeploy 硬性要求 transformers>=4.56，被并入 [all]/optional 后拉高了 transformers 版本，而 HHEM 官方 remote code 为旧版 transformers 编写， 4.49+ 的加载流程会访问 all_tied_weights_keys（旧 remote code 未实现）。 - setup.py: lmdeploy 单独成 extra（lmdeploy），不再并入 optional/all， 避免同一环境内拉高 transformers 导致 HHEM 无法加载 - optional.txt: 移除 lmdeploy 行 - hhem_integration.txt: transformers 上限收紧为 <4.49，并注明原因 - base_lmdeploy_apiclient.py: create_client 内惰性 import lmdeploy， 缺失时抛清晰 ImportError 指引 pip install dingo-python[lmdeploy] Co-Authored-By: Claude Opus 4.8 --- dingo/model/llm/base_lmdeploy_apiclient.py | 8 +++++++- requirements/hhem_integration.txt | 5 ++++- requirements/optional.txt | 1 - setup.py | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dingo/model/llm/base_lmdeploy_apiclient.py b/dingo/model/llm/base_lmdeploy_apiclient.py index 7cc10eb9..2ac5a7b4 100644 --- a/dingo/model/llm/base_lmdeploy_apiclient.py +++ b/dingo/model/llm/base_lmdeploy_apiclient.py @@ -23,7 +23,13 @@ class BaseLmdeployApiClient(BaseLLM): @classmethod def create_client(cls): - from lmdeploy.serve.openai.api_client import APIClient + try: + from lmdeploy.serve.openai.api_client import APIClient + except ImportError as exc: + raise ImportError( + "lmdeploy is required for the lmdeploy API client. " + "Install with: pip install dingo-python[lmdeploy] (or pip install lmdeploy)" + ) from exc if not cls.dynamic_config.api_url: raise ValueError("api_url cannot be empty in llm config.") diff --git a/requirements/hhem_integration.txt b/requirements/hhem_integration.txt index 8efa88f2..01a849fa 100644 --- a/requirements/hhem_integration.txt +++ b/requirements/hhem_integration.txt @@ -2,7 +2,10 @@ # Required for Vectara HHEM-2.1-Open hallucination detection model # Core transformers library for HHEM model -transformers>=4.30.0 +# 上限 <4.49：HHEM 官方 remote code (modeling_hhem_v2.py) 为旧版 transformers 编写， +# transformers 4.49+ 的加载流程会访问 all_tied_weights_keys 属性，旧 remote code 未实现， +# 导致 "'HHEMv2ForSequenceClassification' object has no attribute 'all_tied_weights_keys'"。 +transformers>=4.30.0,<4.49 # PyTorch (CPU version should be sufficient for HHEM) torch>=1.12.0 diff --git a/requirements/optional.txt b/requirements/optional.txt index bd51970a..d0198546 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -4,7 +4,6 @@ ftfy imagededup google-api-python-client -lmdeploy opencv-python-headless pyiqa pyspark==3.4.1 diff --git a/setup.py b/setup.py index 307e9e31..d4635707 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,9 @@ def _read_requirements(path): hhem_requirements = _read_requirements("./requirements/hhem_integration.txt") litellm_requirements = ["litellm>=1.80.0,<1.87.0"] retrieval_requirements = _read_requirements("./requirements/retrieval.txt") +# lmdeploy 单独成组：它硬性要求 transformers>=4.56，与 HHEM 需要的 transformers<4.49 冲突， +# 因此不并入 optional/all，避免同一环境内 HHEM 无法加载。需要时单独 pip install dingo-python[lmdeploy]。 +lmdeploy_requirements = ["lmdeploy"] extras_require = { @@ -24,6 +27,7 @@ def _read_requirements(path): 'hhem': hhem_requirements, 'litellm': litellm_requirements, 'retrieval': retrieval_requirements, + 'lmdeploy': lmdeploy_requirements, 'all': optional_requirements + hhem_requirements + agent_requirements + litellm_requirements + retrieval_requirements, } From 42bd3d5823f74019d4eb19232418054f6053852b Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 31 Jul 2026 12:38:11 +0000 Subject: [PATCH 72/80] =?UTF-8?q?=F0=9F=93=9A=20Auto-update=20metrics=20do?= =?UTF-8?q?cumentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 022f6246..10d27c94 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -115,7 +115,6 @@ This document provides comprehensive information about all quality metrics used | `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Uses a local zero-shot classifier to check whether content belongs to the type declared in the record | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | @@ -163,12 +162,12 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and their types against a user-provided schema. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether `content` belongs to the annotation values configured in `key_list`. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `content` and `source`, and checks whether the HTTP or HTTPS `source` returns status 200. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses a local multilingual NLI model to check semantic consistency among string fields configured in `key_list`. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks whether content belongs to a user-provided annotation value list. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks configured fields for missing, None, and empty values. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Checks whether source is an HTTP or HTTPS URL that returns status 200. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses a local model to check semantic consistency among configured string fields. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | From 2423152d7e56a66e028b57199777cb529565089d Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 3 Aug 2026 15:43:23 +0800 Subject: [PATCH 73/80] =?UTF-8?q?feat:=20rule=20md=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=EF=BC=8Ctc609=20=E6=9C=AA=E6=94=AF=E6=8C=81=EF=BC=8C=E5=88=99?= =?UTF-8?q?=E4=B8=8D=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../model/rule/guobiao/rule_tc609_quality.py | 64 +++++++++---------- docs/metrics.md | 44 ++----------- docs/rules.md | 40 ------------ docs/rules_tc609.md | 14 ++++ 4 files changed, 50 insertions(+), 112 deletions(-) create mode 100644 docs/rules_tc609.md diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index bb06a5ea..6383ba7e 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -17,7 +17,7 @@ ) -@Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) +# @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness): """0101: Basic information completeness in dataset documentation.""" @@ -51,7 +51,7 @@ class Rule_TC609_0101_DocBasicInfoCompleteness(Rule_TC609_01_DocCompleteness): ) -@Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao_doc"]) +# @Model.rule_register("QUALITY_BAD_TC609_0102", ["guobiao_doc"]) class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompleteness): """0102: Content feature completeness in dataset documentation.""" @@ -85,7 +85,7 @@ class Rule_TC609_0102_DocContentFeatureCompleteness(Rule_TC609_01_DocCompletenes ) -@Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao_doc"]) +# @Model.rule_register("QUALITY_BAD_TC609_0103", ["guobiao_doc"]) class Rule_TC609_0103_DocConstructionProcessCompleteness( Rule_TC609_01_DocCompleteness ): @@ -122,7 +122,7 @@ class Rule_TC609_0103_DocConstructionProcessCompleteness( ) -@Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao_doc"]) +# @Model.rule_register("QUALITY_BAD_TC609_0104", ["guobiao_doc"]) class Rule_TC609_0104_DocApplicationCompleteness(Rule_TC609_01_DocCompleteness): """0104: Application-description completeness in dataset documentation.""" @@ -962,7 +962,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return super().eval(text_input) -@Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080101", ["pretrain", "guobiao_text"]) class Rule_TC609_02080101_TextPerplexity(BaseRule): """Check whether text perplexity exceeds the configured threshold.""" @@ -1143,7 +1143,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080102", ["guobiao_text"]) class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAlphaWords", @@ -1159,7 +1159,7 @@ class Rule_TC609_02080102_KnowledgeInformationDensity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080103", ["guobiao_text"]) class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleDocRepeat", @@ -1174,7 +1174,7 @@ class Rule_TC609_02080103_RepeatedContent(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080104", ["guobiao_text"]) class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", @@ -1191,7 +1191,7 @@ class Rule_TC609_02080104_TextCompleteness(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080105", ["guobiao_text"]) class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleContentNull", @@ -1208,7 +1208,7 @@ class Rule_TC609_02080105_InformationMissing(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080106", ["guobiao_text"]) class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleAbnormalChar", @@ -1226,7 +1226,7 @@ class Rule_TC609_02080106_TextPurity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao_text"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080107", ["guobiao_text"]) class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_common.RuleNoPunc", @@ -1244,7 +1244,7 @@ class Rule_TC609_02080107_TextCoherence(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao_image"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080201", ["guobiao_image"]) class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageSizeValid",) _required_fields = [RequiredField.IMAGE] @@ -1256,7 +1256,7 @@ class Rule_TC609_02080201_ImageResolution(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao_image"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080202", ["guobiao_image"]) class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageRepeat",) _required_fields = [RequiredField.CONTENT] @@ -1268,7 +1268,7 @@ class Rule_TC609_02080202_ImageDuplication(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao_image"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080203", ["guobiao_image"]) class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite): component_rules = ("dingo.model.rule.rule_image.RuleImageQuality",) _required_fields = [RequiredField.IMAGE] @@ -1280,7 +1280,7 @@ class Rule_TC609_02080203_ImageSignalNoiseRatio(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao_image"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080204", ["guobiao_image"]) class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite): component_rules = ( "dingo.model.rule.rule_image.RuleImageValid", @@ -1295,7 +1295,7 @@ class Rule_TC609_02080204_ImageClarity(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080301", ["guobiao_video"]) class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080301", "Rule_TC609_02080301_VideoResolution", @@ -1303,7 +1303,7 @@ class Rule_TC609_02080301_VideoResolution(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080302", ["guobiao_video"]) class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080302", "Rule_TC609_02080302_VideoDuplication", @@ -1311,7 +1311,7 @@ class Rule_TC609_02080302_VideoDuplication(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080303", ["guobiao_video"]) class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080303", "Rule_TC609_02080303_VideoFrameRate", @@ -1319,7 +1319,7 @@ class Rule_TC609_02080303_VideoFrameRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080304", ["guobiao_video"]) class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080304", "Rule_TC609_02080304_VideoDuration", @@ -1327,7 +1327,7 @@ class Rule_TC609_02080304_VideoDuration(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080305", ["guobiao_video"]) class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080305", "Rule_TC609_02080305_VideoClarity", @@ -1335,7 +1335,7 @@ class Rule_TC609_02080305_VideoClarity(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_video"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080306", ["guobiao_video"]) class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080306", "Rule_TC609_02080306_VideoDynamicRange", @@ -1343,7 +1343,7 @@ class Rule_TC609_02080306_VideoDynamicRange(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080401", ["guobiao_audio"]) class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite): # Existing RuleAudioDuration currently contains the SNR implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioDuration",) @@ -1356,7 +1356,7 @@ class Rule_TC609_02080401_AudioSignalNoiseRatio(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080402", ["guobiao_audio"]) class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080402", "Rule_TC609_02080402_SignalDistortionRatio", @@ -1364,7 +1364,7 @@ class Rule_TC609_02080402_SignalDistortionRatio(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080403", ["guobiao_audio"]) class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080403", "Rule_TC609_02080403_AudioSampleRate", @@ -1372,7 +1372,7 @@ class Rule_TC609_02080403_AudioSampleRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080404", ["guobiao_audio"]) class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080404", "Rule_TC609_02080404_AudioBitDepth", @@ -1380,7 +1380,7 @@ class Rule_TC609_02080404_AudioBitDepth(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080405", ["guobiao_audio"]) class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): _metric_info = _tc609_metric_info( "02080405", "Rule_TC609_02080405_AudioBitRate", @@ -1388,7 +1388,7 @@ class Rule_TC609_02080405_AudioBitRate(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao_audio"]) +# @Model.rule_register("QUALITY_BAD_TC609_02080406", ["guobiao_audio"]) class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite): # Existing RuleAudioSnrQuality currently contains the duration implementation. component_rules = ("dingo.model.rule.rule_audio.RuleAudioSnrQuality",) @@ -1401,7 +1401,7 @@ class Rule_TC609_02080406_AudioDuration(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_model"]) +# @Model.rule_register("QUALITY_BAD_TC609_0301", ["guobiao_model"]) class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): """0301: Placeholder for content diversity.""" @@ -1413,7 +1413,7 @@ class Rule_TC609_0301_ContentDiversity(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_model"]) +# @Model.rule_register("QUALITY_BAD_TC609_0302", ["guobiao_model"]) class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): """0302: Placeholder for scale completeness.""" @@ -1425,7 +1425,7 @@ class Rule_TC609_0302_ScaleCompleteness(_TC609PlaceholderBase): ) -@Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao_model"]) +# @Model.rule_register("QUALITY_BAD_TC609_0303", ["guobiao_model"]) class Rule_TC609_0303_DataTimeRange(BaseRule): """Check whether creation/update time fields are within configured ranges.""" @@ -1560,7 +1560,7 @@ def eval(cls, input_data: Data) -> EvalDetail: return res -@Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao_model"]) +# @Model.rule_register("QUALITY_BAD_TC609_0304", ["guobiao_model"]) class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite): """0304: Annotation accuracy, partially covered by label checks.""" @@ -1577,7 +1577,7 @@ class Rule_TC609_0304_AnnotationAccuracy(Rule_TC609_Composite): ) -@Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_model"]) +# @Model.rule_register("QUALITY_BAD_TC609_0305", ["guobiao_model"]) class Rule_TC609_0305_ModelAdaptability(_TC609PlaceholderBase): """0305: Placeholder for model adaptability.""" diff --git a/docs/metrics.md b/docs/metrics.md index 4e2bf864..2d5ac248 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -116,19 +116,6 @@ This document provides comprehensive information about all quality metrics used | `LLMMetaRaterReadability` | LLMMetaRaterReadability | Evaluates the clarity and coherence of text using appropriate vocabulary and sentence structures on a 5-point scale | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | | `LLMMetaRaterReasoning` | LLMMetaRaterReasoning | Evaluates the reasoning complexity and logical depth of text content, from simple logical judgments to complex multid... | [Meta-rater: A Multi-dimensional Data Selection Method for Pre-training Language Models](https://arxiv.org/pdf/2504.14194) (Zhuang et al., 2025) | N/A | N/A | -### National Standard Data Quality Metrics - -| Type | Metric | Description | Paper Source | Evaluation Results | Examples | -|------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_TC609_0101` | Rule_TC609_0101_DocBasicInfoCompleteness | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and s... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0102` | Rule_TC609_0102_DocContentFeatureCompleteness | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples,... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0103` | Rule_TC609_0103_DocConstructionProcessCompleteness | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing,... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0104` | Rule_TC609_0104_DocApplicationCompleteness | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchm... | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Checks whether text items in `data_content` match the configured dataset type | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080101` | Rule_TC609_02080101_TextPerplexity | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0303` | Rule_TC609_0303_DataTimeRange | Checks whether created and updated timestamps are within configured time ranges | Internal Implementation | N/A | N/A | - ### OCR Eval Metric | Type | Metric | Description | Paper Source | Evaluation Results | Examples | @@ -170,41 +157,18 @@ This document provides comprehensive information about all quality metrics used ### SAC/TC609 High-quality Dataset Metrics +Only the following eight TC609 rule metrics are currently registered. Other rule implementations remain in the source code with their registration decorators commented out. + | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| +| `QUALITY_BAD_TC609_0201` | Rule_TC609_0201_FormatCompliance | Checks required fields and types against `field_schema`. Extra fields are allowed by default and can be rejected with `allow_extra=false`. Supports `str`, `int`, `float`, `bool`, `list`, `dict`, and nullable `Optional[...]` variants. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0202` | Rule_TC609_0202_SafetyCompliance | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0203` | Rule_TC609_0203_AnnotationCompliance | Checks TC609 annotation metadata fields, types, and enumerated values. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Checks fields configured in `key_list` for missing values. `allow_none` and `allow_empty` control whether `None` and empty strings/lists/dicts are accepted. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0205` | Rule_TC609_0205_ContentAuthenticity | Requires `source` and `source_details`; validates non-empty traceability information and HTTP/HTTPS URL syntax when applicable. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0204` | Rule_TC609_0204_StructuralCompleteness | Combines null-content and short-content checks. | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0206` | Rule_TC609_0206_ContentConsistency | Uses multilingual text embeddings to check consistency among text items in `data_content`; multiple texts use robust-center aggregation instead of all-pairs comparison. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080102` | Rule_TC609_02080102_KnowledgeInformationDensity | Combines alphabetic-word, stop-word, and unique-word ratio checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080103` | Rule_TC609_02080103_RepeatedContent | Combines document-text and formula repetition checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080104` | Rule_TC609_02080104_TextCompleteness | Combines null, short, ellipsis-ending, and terminal-ending checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080105` | Rule_TC609_02080105_InformationMissing | Uses content length and sentence/word counts as partial missing-information checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080106` | Rule_TC609_02080106_TextPurity | Combines abnormal HTML, character, invisible-content, and watermark checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080107` | Rule_TC609_02080107_TextCoherence | Combines punctuation, word-boundary, and line-break fluency checks. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080201` | Rule_TC609_02080201_ImageResolution | Uses image aspect-ratio validation as partial resolution coverage. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080202` | Rule_TC609_02080202_ImageDuplication | Uses PHash and CNN duplicate-image detection. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080203` | Rule_TC609_02080203_ImageSignalNoiseRatio | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080204` | Rule_TC609_02080204_ImageClarity | Combines image validity and NIMA quality as partial clarity coverage. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080301` | Rule_TC609_02080301_VideoResolution | Placeholder: video resolution is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080302` | Rule_TC609_02080302_VideoDuplication | Placeholder: duplicate-video detection is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080303` | Rule_TC609_02080303_VideoFrameRate | Placeholder: video FPS validation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080304` | Rule_TC609_02080304_VideoDuration | Placeholder: video duration validation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080305` | Rule_TC609_02080305_VideoClarity | Placeholder: video clarity evaluation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080306` | Rule_TC609_02080306_VideoDynamicRange | Placeholder: video dynamic-range evaluation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080401` | Rule_TC609_02080401_AudioSignalNoiseRatio | Uses the existing Welch power-spectrum SNR implementation. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080402` | Rule_TC609_02080402_SignalDistortionRatio | Placeholder: signal distortion ratio is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080403` | Rule_TC609_02080403_AudioSampleRate | Placeholder: sample-rate quality validation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080404` | Rule_TC609_02080404_AudioBitDepth | Placeholder: audio bit-depth validation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080405` | Rule_TC609_02080405_AudioBitRate | Placeholder: audio bit-rate validation is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_02080406` | Rule_TC609_02080406_AudioDuration | Uses the existing WAV duration implementation. | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_TC609_0207` | Rule_TC609_0207_DataTypeConsistency | Checks whether text items in `data_content` match the configured dataset type | Internal Implementation | N/A | N/A | | `QUALITY_BAD_TC609_0208` | Rule_TC609_0208_ContentCleanliness | Combines available text cleanliness checks; modality coverage is partial. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0301` | Rule_TC609_0301_ContentDiversity | Placeholder: target-scenario distribution coverage is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0302` | Rule_TC609_0302_ScaleCompleteness | Placeholder: dataset scale versus model requirements is not implemented. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0304` | Rule_TC609_0304_AnnotationAccuracy | Uses image annotation checks as partial evidence of annotation accuracy. | Internal Implementation | N/A | N/A | -| `QUALITY_BAD_TC609_0305` | Rule_TC609_0305_ModelAdaptability | Placeholder: before/after model performance comparison is not implemented. | Internal Implementation | N/A | N/A | ### SFT Data Assessment Metrics - Agent-Enhanced diff --git a/docs/rules.md b/docs/rules.md index 20122ff2..a325657c 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -86,46 +86,6 @@ The specific rules for each quality metric are as follows: | RuleSpecialCharacter | RELEVANCE | check whether content has special characters. | | | RuleStopWord | EFFECTIVENESS | check whether the ratio of stop word > 0.06 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | | RuleSymbolWordRatio | EFFECTIVENESS | check whether the ratio of symbol / word is > 0.4 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [Gopher](https://arxiv.org/abs/2112.11446) [Dolma](https://arxiv.org/abs/2402.00159) | -| Rule_TC609_0101_DocBasicInfoCompleteness | TC609_0101 | Checks whether dataset documentation covers basic information aspects such as scale, format, structure, access, and support | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0102_DocContentFeatureCompleteness | TC609_0102 | Checks whether dataset documentation covers content-feature aspects such as modality, distribution, labels, examples, and limitations | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0103_DocConstructionProcessCompleteness | TC609_0103 | Checks whether dataset documentation covers construction-process aspects such as data source, collection, processing, annotation, and version control | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0104_DocApplicationCompleteness | TC609_0104 | Checks whether dataset documentation covers application aspects such as license, scenarios, evaluation method, benchmark, and cases | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0201_FormatCompliance | TC609_0201 | Combines existing NLP, SFT, image, audio, and video format rules. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0202_SafetyCompliance | TC609_0202 | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0203_AnnotationCompliance | TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | -| Rule_TC609_0204_StructuralCompleteness | TC609_0204 | Combines null-content and short-content checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0205_ContentAuthenticity | TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | -| Rule_TC609_0206_ContentConsistency | TC609_0206 | Checks consistency among text items in `data_content` using multilingual embeddings and robust-center aggregation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0207_DataTypeConsistency | TC609_0207 | Checks whether text items in `data_content` match the configured dataset type | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080101_TextPerplexity | TC609_02080101 | Calculates text perplexity with a causal language model and flags text whose PPL exceeds the configured threshold | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080102_KnowledgeInformationDensity | TC609_02080102 | Combines alphabetic-word, stop-word, and unique-word ratio checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080103_RepeatedContent | TC609_02080103 | Combines document-text and formula repetition checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080104_TextCompleteness | TC609_02080104 | Combines null, short, ellipsis-ending, and terminal-ending checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080105_InformationMissing | TC609_02080105 | Uses content length and sentence/word counts as partial missing-information checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080106_TextPurity | TC609_02080106 | Combines abnormal HTML, character, invisible-content, and watermark checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080107_TextCoherence | TC609_02080107 | Combines punctuation, word-boundary, and line-break fluency checks. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080201_ImageResolution | TC609_02080201 | Uses image aspect-ratio validation as partial resolution coverage. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080202_ImageDuplication | TC609_02080202 | Uses PHash and CNN duplicate-image detection. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080203_ImageSignalNoiseRatio | TC609_02080203 | Uses NIMA image quality as partial evidence; it is not a true SNR metric. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080204_ImageClarity | TC609_02080204 | Combines image validity and NIMA quality as partial clarity coverage. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080301_VideoResolution | TC609_02080301 | Placeholder: video resolution is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080302_VideoDuplication | TC609_02080302 | Placeholder: duplicate-video detection is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080303_VideoFrameRate | TC609_02080303 | Placeholder: video FPS validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080304_VideoDuration | TC609_02080304 | Placeholder: video duration validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080305_VideoClarity | TC609_02080305 | Placeholder: video clarity evaluation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080306_VideoDynamicRange | TC609_02080306 | Placeholder: video dynamic-range evaluation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080401_AudioSignalNoiseRatio | TC609_02080401 | Uses the existing Welch power-spectrum SNR implementation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080402_SignalDistortionRatio | TC609_02080402 | Placeholder: signal distortion ratio is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080403_AudioSampleRate | TC609_02080403 | Placeholder: sample-rate quality validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080404_AudioBitDepth | TC609_02080404 | Placeholder: audio bit-depth validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080405_AudioBitRate | TC609_02080405 | Placeholder: audio bit-rate validation is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_02080406_AudioDuration | TC609_02080406 | Uses the existing WAV duration implementation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0208_ContentCleanliness | TC609_0208 | Combines available text cleanliness checks; modality coverage is partial. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0301_ContentDiversity | TC609_0301 | Placeholder: target-scenario distribution coverage is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0302_ScaleCompleteness | TC609_0302 | Placeholder: dataset scale versus model requirements is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0303_DataTimeRange | TC609_0303 | Checks whether created and updated timestamps are within configured time ranges | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0304_AnnotationAccuracy | TC609_0304 | Uses image annotation checks as partial evidence of annotation accuracy. | TC609-5-2025-04 High-quality dataset quality evaluation specification | -| Rule_TC609_0305_ModelAdaptability | TC609_0305 | Placeholder: before/after model performance comparison is not implemented. | TC609-5-2025-04 High-quality dataset quality evaluation specification | | RuleUniqueWords | UNDERSTANDABILITY | check whether the ratio of unique words > 0.1 | [Redpajama](https://www.together.ai/blog/redpajama-data-v2) [MAP-en](https://arxiv.org/abs/2405.19327) | | RuleUnsafeWords | SECURITY | Check whether content contains unsafe words. | | | RuleVedioDataFormat | EFFECTIVENESS | Check whether video data has the expected format. | | diff --git a/docs/rules_tc609.md b/docs/rules_tc609.md new file mode 100644 index 00000000..442233ef --- /dev/null +++ b/docs/rules_tc609.md @@ -0,0 +1,14 @@ +# SAC/TC609 High-quality Dataset Rules + +The following eight TC609 rules are currently registered. Rules whose registration decorators are commented out are not included. + +| Rule | Metric Type | Description | Standard Source | +|------|-------------|-------------|-----------------| +| Rule_TC609_0201_FormatCompliance | QUALITY_BAD_TC609_0201 | Checks required fields and types against the configured field schema. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0202_SafetyCompliance | QUALITY_BAD_TC609_0202 | Combines unsafe-word, PII, and identity-card detection for text items in `data_content`. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0203_AnnotationCompliance | QUALITY_BAD_TC609_0203 | Checks annotation metadata fields, types, and enumerated values. | TC609-5-2025-02 High-quality dataset format requirements | +| Rule_TC609_0204_StructuralCompleteness | QUALITY_BAD_TC609_0204 | Checks configured fields for missing values. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0205_ContentAuthenticity | QUALITY_BAD_TC609_0205 | Checks source traceability metadata and validates URL syntax when applicable. | TC609-5-2025-02 High-quality dataset format requirements | +| Rule_TC609_0206_ContentConsistency | QUALITY_BAD_TC609_0206 | Checks consistency among text items in `data_content` using multilingual embeddings and robust-center aggregation. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0207_DataTypeConsistency | QUALITY_BAD_TC609_0207 | Checks whether text items in `data_content` match the configured dataset type. | TC609-5-2025-04 High-quality dataset quality evaluation specification | +| Rule_TC609_0208_ContentCleanliness | QUALITY_BAD_TC609_0208 | Combines available text cleanliness checks; modality coverage is partial. | TC609-5-2025-04 High-quality dataset quality evaluation specification | From b90faf2a00c406334ec0a1a20f236f4eea22bcaf Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 3 Aug 2026 15:46:36 +0800 Subject: [PATCH 74/80] feat: lint --- dingo/model/rule/guobiao/rule_tc609_quality.py | 11 +++-------- .../rule/guobiao/rule_tc609_quality_base.py | 1 - .../model/llm/test_tc609_doc_completeness.py | 17 ++++------------- .../model/rule/test_rule_tc609_quality.py | 3 +-- 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/dingo/model/rule/guobiao/rule_tc609_quality.py b/dingo/model/rule/guobiao/rule_tc609_quality.py index 6383ba7e..aa418263 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality.py @@ -7,14 +7,8 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model from dingo.model.rule.base import BaseRule -from dingo.model.rule.guobiao.rule_tc609_quality_base import ( - Rule_TC609_01_DocCompleteness, - Rule_TC609_Composite, - TC609_DATASET_TYPE_DESCRIPTIONS, - _tc609_metric_info, - _TC609PlaceholderBase, - calculate_text_consistency, -) +from dingo.model.rule.guobiao.rule_tc609_quality_base import (TC609_DATASET_TYPE_DESCRIPTIONS, Rule_TC609_01_DocCompleteness, Rule_TC609_Composite, _tc609_metric_info, _TC609PlaceholderBase, + calculate_text_consistency) # @Model.rule_register("QUALITY_BAD_TC609_0101", ["guobiao_doc"]) @@ -450,6 +444,7 @@ def eval(cls, input_data: Data) -> EvalDetail: res.label = [QualityLabel.QUALITY_GOOD] return res + @Model.rule_register("QUALITY_BAD_TC609_0204", ["guobiao_data"]) class Rule_TC609_0204_StructuralCompleteness(BaseRule): """Check required fields for missing, None, and empty values.""" diff --git a/dingo/model/rule/guobiao/rule_tc609_quality_base.py b/dingo/model/rule/guobiao/rule_tc609_quality_base.py index 21f1c566..44c9479d 100644 --- a/dingo/model/rule/guobiao/rule_tc609_quality_base.py +++ b/dingo/model/rule/guobiao/rule_tc609_quality_base.py @@ -8,7 +8,6 @@ from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.rule.base import BaseRule - TC609_DATASET_TYPE_DESCRIPTIONS = { "通识数据集": "面向普通公众，内容属于跨行业普遍适用、无需特定行业背景即可理解的通用知识", "行业通识数据集": "面向特定行业，内容属于该行业从业者普遍需要掌握的基础知识、通用规范或常见实践", diff --git a/test/scripts/model/llm/test_tc609_doc_completeness.py b/test/scripts/model/llm/test_tc609_doc_completeness.py index 8b6a7942..ba31131a 100644 --- a/test/scripts/model/llm/test_tc609_doc_completeness.py +++ b/test/scripts/model/llm/test_tc609_doc_completeness.py @@ -1,17 +1,8 @@ from dingo.io.input import Data -from dingo.model.llm.guobiao.llm_tc609_0101_doc_basic_info_completeness import ( - LLM_TC609_0101_DocBasicInfoCompleteness, -) -from dingo.model.llm.guobiao.llm_tc609_0102_doc_content_feature_completeness import ( - LLM_TC609_0102_DocContentFeatureCompleteness, -) -from dingo.model.llm.guobiao.llm_tc609_0103_doc_construction_process_completeness import ( - LLM_TC609_0103_DocConstructionProcessCompleteness, -) -from dingo.model.llm.guobiao.llm_tc609_0104_doc_application_completeness import ( - LLM_TC609_0104_DocApplicationCompleteness, -) - +from dingo.model.llm.guobiao.llm_tc609_0101_doc_basic_info_completeness import LLM_TC609_0101_DocBasicInfoCompleteness +from dingo.model.llm.guobiao.llm_tc609_0102_doc_content_feature_completeness import LLM_TC609_0102_DocContentFeatureCompleteness +from dingo.model.llm.guobiao.llm_tc609_0103_doc_construction_process_completeness import LLM_TC609_0103_DocConstructionProcessCompleteness +from dingo.model.llm.guobiao.llm_tc609_0104_doc_application_completeness import LLM_TC609_0104_DocApplicationCompleteness TC609_LLM_CLASSES = [ LLM_TC609_0101_DocBasicInfoCompleteness, diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index a46b3789..a799802d 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -7,8 +7,7 @@ from dingo.io.input import RequiredField from dingo.io.output.eval_detail import EvalDetail, QualityLabel from dingo.model.model import Model -from dingo.model.rule.guobiao import rule_tc609_quality -from dingo.model.rule.guobiao import rule_tc609_quality_base +from dingo.model.rule.guobiao import rule_tc609_quality, rule_tc609_quality_base from dingo.model.rule.guobiao.rule_tc609_quality import (Rule_TC609_0201_FormatCompliance, Rule_TC609_0202_SafetyCompliance, Rule_TC609_0203_AnnotationCompliance, Rule_TC609_0204_StructuralCompleteness, Rule_TC609_0205_ContentAuthenticity, Rule_TC609_0206_ContentConsistency, Rule_TC609_0208_ContentCleanliness, Rule_TC609_0301_ContentDiversity) From a325a8def19ce4a45a60066c6ac5ad403b2c7211 Mon Sep 17 00:00:00 2001 From: chupei Date: Mon, 3 Aug 2026 15:53:23 +0800 Subject: [PATCH 75/80] feat: add perspective check example --- examples/security/sdk_perspective.py | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 examples/security/sdk_perspective.py diff --git a/examples/security/sdk_perspective.py b/examples/security/sdk_perspective.py new file mode 100644 index 00000000..70974ad7 --- /dev/null +++ b/examples/security/sdk_perspective.py @@ -0,0 +1,49 @@ +"""Run LLMPerspective against the Google Perspective API. + +Before running this example: + + pip install google-api-python-client + export PERSPECTIVE_API_KEY="your-google-api-key" + python examples/security/sdk_perspective.py +""" + +import os + +from dingo.config.input_args import EvaluatorLLMArgs +from dingo.io.input import Data +from dingo.model.llm.llm_perspective import LLMPerspective + + +def main() -> None: + api_key = os.getenv("PERSPECTIVE_API_KEY") or os.getenv("GOOGLE_API_KEY") + if not api_key: + raise SystemExit( + "Please set PERSPECTIVE_API_KEY (or GOOGLE_API_KEY) before running this example." + ) + + LLMPerspective.dynamic_config = EvaluatorLLMArgs( + key=api_key, + api_url=os.getenv( + "PERSPECTIVE_API_URL", + "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1", + ), + ) + LLMPerspective.client = None + + samples = [ + Data(data_id="perspective-good", content="Thank you for your thoughtful answer."), + Data(data_id="perspective-toxic", content="You are stupid and I hate you."), + ] + + for sample in samples: + result = LLMPerspective.eval(sample) + print(f"data_id: {sample.data_id}") + print(f"content: {sample.content}") + print(f"status: {result.status} # True means a quality issue was detected") + print(f"label: {result.label}") + print(f"reason: {result.reason}") + print() + + +if __name__ == "__main__": + main() From 444929e869dbdfd7d3752e57cd156827954d5af7 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 3 Aug 2026 16:17:15 +0800 Subject: [PATCH 76/80] feat: update test --- .../model/llm/test_tc609_doc_completeness.py | 2 +- test/scripts/model/rule/test_rule_common.py | 11 ------- .../model/rule/test_rule_tc609_quality.py | 33 +++++-------------- 3 files changed, 9 insertions(+), 37 deletions(-) diff --git a/test/scripts/model/llm/test_tc609_doc_completeness.py b/test/scripts/model/llm/test_tc609_doc_completeness.py index ba31131a..2e155d21 100644 --- a/test/scripts/model/llm/test_tc609_doc_completeness.py +++ b/test/scripts/model/llm/test_tc609_doc_completeness.py @@ -15,7 +15,7 @@ def test_tc609_doc_llm_prompts_define_binary_json_output(): for evaluator in TC609_LLM_CLASSES: assert '"score": 0' in evaluator.prompt - assert "至少 4 项" in evaluator.prompt + assert "至少4项" in evaluator.prompt assert "同义词、近义表达" in evaluator.prompt assert evaluator._required_fields diff --git a/test/scripts/model/rule/test_rule_common.py b/test/scripts/model/rule/test_rule_common.py index 21f1c595..b22dc7fe 100644 --- a/test/scripts/model/rule/test_rule_common.py +++ b/test/scripts/model/rule/test_rule_common.py @@ -148,7 +148,6 @@ def test_high_perplexity_is_bad(self, monkeypatch): ) assert res.status is True - assert res.label == ["QUALITY_BAD_TC609_02080101.Rule_TC609_02080101_TextPerplexity"] assert "125.5000" in res.reason[0] assert "test-model" in res.reason[0] @@ -191,7 +190,6 @@ def fail_if_called(cls, model_name): res = Rule_TC609_02080101_TextPerplexity.eval(Data(data_id="ppl-empty", content=" ")) assert res.status is True - assert res.label == ["QUALITY_BAD_TC609_02080101.Rule_TC609_02080101_TextPerplexity"] assert "empty content" in res.reason[0] def test_missing_dependencies_raise_clear_error(self, monkeypatch): @@ -339,7 +337,6 @@ def test_created_time_out_of_range_is_bad(self, monkeypatch): ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "earlier than allowed start" in result.reason[0] def test_dt_invalid_format_is_bad(self, monkeypatch): @@ -359,7 +356,6 @@ def test_dt_invalid_format_is_bad(self, monkeypatch): ) ) assert result.status is True - assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "unsupported datetime format" in result.reason[0] def test_missing_time_field_is_bad(self, monkeypatch): @@ -374,7 +370,6 @@ def test_missing_time_field_is_bad(self, monkeypatch): result = Rule_TC609_0303_DataTimeRange.eval(Data(data_id="time-missing")) assert result.status is True - assert result.label == ["QUALITY_BAD_TC609_0303.Rule_TC609_0303_DataTimeRange"] assert "dt is missing" in result.reason[0] @@ -614,9 +609,6 @@ def test_basic_info_completeness_bad(self, monkeypatch): Data(data_id="doc-basic-bad", content=content) ) assert res.status is True - assert res.label == [ - "QUALITY_BAD_TC609_0101.Rule_TC609_0101_DocBasicInfoCompleteness" - ] assert res.score < 0.8 def test_content_feature_completeness_good(self, monkeypatch): @@ -660,7 +652,4 @@ def test_empty_content_is_bad(self): Data(data_id="doc-empty", content=" ") ) assert res.status is True - assert res.label == [ - "QUALITY_BAD_TC609_0104.Rule_TC609_0104_DocApplicationCompleteness" - ] assert "missing or empty" in res.reason[0] diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index a799802d..da4b65a9 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -14,7 +14,7 @@ from dingo.model.rule.rule_common import RuleWatermark -def test_tc609_quality_defines_all_standard_metrics(): +def test_unsupported_tc609_quality_metrics_are_not_registered(): rule_classes = { name: cls for name, cls in inspect.getmembers( @@ -26,7 +26,7 @@ def test_tc609_quality_defines_all_standard_metrics(): } assert len(rule_classes) == 40 - assert all(name in Model.rule_name_map for name in rule_classes) + assert all(name not in Model.rule_name_map for name in rule_classes) expected_primary_codes = { "0101", "0102", "0103", "0104", @@ -41,28 +41,11 @@ def test_tc609_quality_defines_all_standard_metrics(): assert actual_codes == expected_primary_codes -def test_tc609_rules_are_grouped_by_evaluation_object(): - expected_group_sizes = { - "guobiao_doc": 4, - "guobiao_data": 8, - "guobiao_text": 7, - "guobiao_image": 4, - "guobiao_video": 6, - "guobiao_audio": 6, - "guobiao_model": 5, - } - - for group_name, expected_size in expected_group_sizes.items(): - tc609_rules = [ - rule - for rule in Model.rule_groups[group_name] - if rule.__name__.startswith("Rule_TC609_") - ] - assert len(tc609_rules) == expected_size - +def test_unsupported_tc609_rules_are_not_grouped(): assert not any( rule.__name__.startswith("Rule_TC609_") - for rule in Model.rule_groups.get("guobiao", []) + for rules in Model.rule_groups.values() + for rule in rules ) @@ -904,7 +887,7 @@ def test_content_consistency_rejects_invalid_data_content(data, reason): def test_calculate_text_consistency_compares_two_texts_directly(monkeypatch): - import torch + torch = pytest.importorskip("torch") monkeypatch.setattr( rule_tc609_quality_base, @@ -926,7 +909,7 @@ def test_calculate_text_consistency_compares_two_texts_directly(monkeypatch): def test_calculate_text_consistency_uses_robust_center(monkeypatch): - import torch + torch = pytest.importorskip("torch") encoded = torch.tensor( [ @@ -963,7 +946,7 @@ def test_content_consistency_declares_data_content_required(): def test_uncovered_rule_is_explicit_placeholder(): - assert Rule_TC609_0301_ContentDiversity.group == ["guobiao_model"] + assert Rule_TC609_0301_ContentDiversity.group == [] with pytest.raises(NotImplementedError, match="placeholder"): Rule_TC609_0301_ContentDiversity.eval( Data(data_id="diversity", content="test") From 85993b0299b23a76aed17191b3f171f6dee803a1 Mon Sep 17 00:00:00 2001 From: shijin Date: Mon, 3 Aug 2026 16:24:41 +0800 Subject: [PATCH 77/80] feat: update test --- .../model/rule/test_rule_tc609_quality.py | 59 ++++++++++++++++--- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/test/scripts/model/rule/test_rule_tc609_quality.py b/test/scripts/model/rule/test_rule_tc609_quality.py index da4b65a9..06d84c66 100644 --- a/test/scripts/model/rule/test_rule_tc609_quality.py +++ b/test/scripts/model/rule/test_rule_tc609_quality.py @@ -14,7 +14,7 @@ from dingo.model.rule.rule_common import RuleWatermark -def test_unsupported_tc609_quality_metrics_are_not_registered(): +def test_only_supported_tc609_quality_metrics_are_registered(): rule_classes = { name: cls for name, cls in inspect.getmembers( @@ -26,7 +26,24 @@ def test_unsupported_tc609_quality_metrics_are_not_registered(): } assert len(rule_classes) == 40 - assert all(name not in Model.rule_name_map for name in rule_classes) + expected_registered = { + f"Rule_TC609_020{index}_{suffix}" + for index, suffix in enumerate( + ( + "FormatCompliance", + "SafetyCompliance", + "AnnotationCompliance", + "StructuralCompleteness", + "ContentAuthenticity", + "ContentConsistency", + "DataTypeConsistency", + "ContentCleanliness", + ), + start=1, + ) + } + actual_registered = set(rule_classes) & set(Model.rule_name_map) + assert actual_registered == expected_registered expected_primary_codes = { "0101", "0102", "0103", "0104", @@ -41,12 +58,38 @@ def test_unsupported_tc609_quality_metrics_are_not_registered(): assert actual_codes == expected_primary_codes -def test_unsupported_tc609_rules_are_not_grouped(): - assert not any( - rule.__name__.startswith("Rule_TC609_") - for rules in Model.rule_groups.values() - for rule in rules - ) +def test_only_supported_tc609_rules_are_grouped_as_data_rules(): + actual_groups = { + group_name: { + rule.__name__ + for rule in rules + if rule.__name__.startswith("Rule_TC609_") + } + for group_name, rules in Model.rule_groups.items() + } + actual_groups = { + group_name: rules + for group_name, rules in actual_groups.items() + if rules + } + assert actual_groups == { + "guobiao_data": { + f"Rule_TC609_020{index}_{suffix}" + for index, suffix in enumerate( + ( + "FormatCompliance", + "SafetyCompliance", + "AnnotationCompliance", + "StructuralCompleteness", + "ContentAuthenticity", + "ContentConsistency", + "DataTypeConsistency", + "ContentCleanliness", + ), + start=1, + ) + } + } def test_format_compliance_accepts_matching_record(monkeypatch): From db594fb5933790dd4eacd384cd36523cdbaab783 Mon Sep 17 00:00:00 2001 From: chupei Date: Tue, 4 Aug 2026 10:40:21 +0800 Subject: [PATCH 78/80] =?UTF-8?q?feat:=20RuleHallucinationHHEM=20=E5=BA=95?= =?UTF-8?q?=E5=B1=82=E6=A8=A1=E5=9E=8B=E6=94=B9=E7=94=A8=20MiniCheck-Flan-?= =?UTF-8?q?T5-Large?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将本地幻觉检测模型从 Vectara HHEM-2.1-Open 换为 lytang/MiniCheck-Flan-T5-Large（保留类名与注册 id 以向后兼容）。 - 标准 T5 模型，去掉 transformers<4.49 版本限制 - LLM-AggreFact 准确率更高（75.0 vs 71.8） - 忠实复刻官方 flan-t5 推理（predict 前缀、单步 decoder、 label token [3,209] softmax、按 chunk 取 max） - 同步更新依赖、示例与中英文幻觉检测指南、自动生成的 metrics.md Co-Authored-By: Claude Opus 4.8 --- dingo/model/rule/rule_hallucination_hhem.py | 214 ++++++++++++------ docs/hallucination_detection_guide.md | 61 +++-- docs/hallucination_guide.md | 56 ++--- docs/metrics.md | 4 +- docs/rules.md | 2 +- .../hallucination/sdk_rule_hhem_detection.py | 52 +++-- requirements/hhem_integration.txt | 21 +- setup.py | 6 +- 8 files changed, 253 insertions(+), 163 deletions(-) diff --git a/dingo/model/rule/rule_hallucination_hhem.py b/dingo/model/rule/rule_hallucination_hhem.py index 6e61ec17..229b0abb 100644 --- a/dingo/model/rule/rule_hallucination_hhem.py +++ b/dingo/model/rule/rule_hallucination_hhem.py @@ -1,14 +1,30 @@ """ -HHEM-2.1-Open Hallucination Detection Rule - -This module provides integration with Vectara's HHEM-2.1-Open model as a rule-based -hallucination detection tool for efficient local inference without API costs. - -Key advantages of HHEM-2.1-Open: -- Superior performance compared to GPT-3.5/GPT-4 on benchmarks -- Local inference with <600MB RAM usage -- Fast processing (~1.5s for 2k tokens on modern CPU) -- No API costs or rate limits +MiniCheck Hallucination Detection Rule + +This module provides local, API-free hallucination (ungrounded-claim) detection +for RAG-style data by checking whether a response is supported by its context. + +Model: `lytang/MiniCheck-Flan-T5-Large` (EMNLP 2024, arXiv:2404.10774). + +Why MiniCheck instead of Vectara HHEM-2.1-Open (the original backing model): +- Stronger grounding accuracy: MiniCheck-Flan-T5-Large scores 75.0 vs HHEM's + 71.8 on the LLM-AggreFact benchmark (llm-aggrefact.github.io). +- Robust across transformers versions: MiniCheck is a *standard* + `T5ForConditionalGeneration` (config `model_type: t5`, no `auto_map` / + `trust_remote_code`). HHEM shipped custom remote code that breaks on + transformers >= 4.49 (AttributeError: `all_tied_weights_keys`), which forced + a `<4.49` pin. MiniCheck removes that constraint. +- Still efficient and CPU-friendly (0.8B params), no API costs or rate limits. + +The class name is kept as `RuleHallucinationHHEM` for backward compatibility +(the registered rule id `QUALITY_BAD_HALLUCINATION` and existing configs are +unaffected). + +Inference is replicated faithfully from the official MiniCheck source +(Liyan06/MiniCheck): input `"predict: " + doc + + claim`, a single-step +decoder forward, then a 2-way softmax over label token ids [3, 209] where +index 1 is P(supported). Long documents are split into word chunks and the +support probability is aggregated by max. """ import json @@ -26,13 +42,17 @@ @Model.rule_register("QUALITY_BAD_HALLUCINATION", ["hallucination", "rag"]) class RuleHallucinationHHEM(BaseRule): """ - HHEM-2.1-Open hallucination detection rule. + MiniCheck-based hallucination detection rule. - Provides efficient local hallucination detection with: - - Superior performance than GPT models on benchmarks - - Low resource usage (<600MB RAM) - - Fast inference (~1.5s for 2k tokens on modern CPU) - - No API costs or rate limits + Detects ungrounded claims by checking whether the response (content) is + supported by the provided context, using `lytang/MiniCheck-Flan-T5-Large`: + - Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8) + - Standard T5 model -> no transformers version pin, no remote code + - Local inference, CPU-friendly, no API costs or rate limits + + Note: the class is still named `RuleHallucinationHHEM` for backward + compatibility; the underlying model was upgraded from Vectara HHEM-2.1-Open + to MiniCheck. """ # Metadata for documentation generation @@ -40,66 +60,131 @@ class RuleHallucinationHHEM(BaseRule): "category": "SFT Data Assessment Metrics", "quality_dimension": "HALLUCINATION", "metric_name": "RuleHallucinationHHEM", - "description": "Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and context", - "paper_title": "HHEM-2.1-Open", - "paper_url": "https://huggingface.co/vectara/hallucination_evaluation_model", - "paper_authors": "Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch" + "description": "Uses the MiniCheck-Flan-T5-Large model for local hallucination " + "detection by checking whether the response is grounded in the context", + "paper_title": "MiniCheck: Efficient Fact-Checking of LLMs on Grounding Documents", + "paper_url": "https://arxiv.org/abs/2404.10774", + "paper_authors": "Liyan Tang, Philippe Laban, Greg Durrett" } + # CONTENT = the response/claim to verify; CONTEXT = the grounding document. + # These are exactly the two inputs MiniCheck needs (claim vs. document), so + # they remain the most fitting required fields. _required_fields = [RequiredField.CONTENT, RequiredField.CONTEXT] dynamic_config = EvaluatorRuleArgs(threshold=0.5) model = None + tokenizer = None _load_lock = Lock() - _model_repo_id = "vectara/hallucination_evaluation_model" + _model_repo_id = "lytang/MiniCheck-Flan-T5-Large" + # MiniCheck flan-t5 inference config (from Liyan06/MiniCheck) + _chunk_size = 500 # words per document chunk before max-aggregation + _max_input_length = 2048 # tokenizer truncation length @classmethod def load_model(cls): - """Load HHEM-2.1-Open model""" + """Load the MiniCheck-Flan-T5-Large model and tokenizer.""" if cls.model is None: with cls._load_lock: if cls.model is not None: return try: - from huggingface_hub import snapshot_download - from transformers import AutoModelForSequenceClassification + from transformers import AutoModelForSeq2SeqLM, AutoTokenizer - log.info("Loading HHEM-2.1-Open model...") + log.info("Loading MiniCheck-Flan-T5-Large model...") + # MiniCheck is a standard T5 model: no trust_remote_code + # needed, and it loads on any modern transformers version. try: - model_path = snapshot_download( - repo_id=cls._model_repo_id, - repo_type="model", - local_files_only=True, + # Prefer offline / cached load first + cls.model = AutoModelForSeq2SeqLM.from_pretrained( + cls._model_repo_id, local_files_only=True, + ) + cls.tokenizer = AutoTokenizer.from_pretrained( + cls._model_repo_id, local_files_only=True, ) except Exception: - model_path = snapshot_download( - repo_id=cls._model_repo_id, - repo_type="model", + # Fall back to downloading from the Hub + cls.model = AutoModelForSeq2SeqLM.from_pretrained( + cls._model_repo_id, ) - - cls.model = AutoModelForSequenceClassification.from_pretrained( - model_path, - trust_remote_code=True, - local_files_only=True, - ) - log.info("✅ HHEM-2.1-Open model loaded successfully") + cls.tokenizer = AutoTokenizer.from_pretrained( + cls._model_repo_id, + ) + cls.model.eval() + log.info("✅ MiniCheck-Flan-T5-Large model loaded successfully") except ImportError: raise ImportError( - "transformers and huggingface_hub are required for HHEM model. " - "Install with: pip install transformers huggingface_hub" + "transformers is required for the MiniCheck model. " + "Install with: pip install transformers torch sentencepiece" ) except Exception as e: raise RuntimeError( - "Failed to load HHEM model. " + "Failed to load MiniCheck model. " "The first run requires network access to download " f"'{cls._model_repo_id}', or a populated Hugging Face cache. " f"Original error: {e}" ) from e + @staticmethod + def _chunk_document(document: str, chunk_size: int) -> List[str]: + """Split a document into consecutive chunks of ~chunk_size words. + + Lightweight, dependency-free approximation of MiniCheck's sentence-based + chunking (the official code uses nltk sent_tokenize). RAG contexts + usually fit in a single chunk, so this rarely changes behavior; long + documents are still split so no content is silently truncated. + """ + text = document.strip() + if not text: + return [] + words = text.split() + if len(words) <= chunk_size: + return [text] + return [" ".join(words[i:i + chunk_size]) + for i in range(0, len(words), chunk_size)] + + @classmethod + def _support_prob(cls, document: str, claim: str) -> float: + """Probability that `claim` is supported by `document` (0=unsupported, 1=supported). + + Faithful replication of the official MiniCheck flan-t5 inference: + input = "predict: " + doc_chunk + tokenizer.eos_token + claim + forward = model(input_ids, attention_mask, decoder_input_ids=zeros(B,1)) + logits = outputs.logits.squeeze(1) + probs = softmax(logits[:, [3, 209]]) # 3=no support, 209=support + support = probs[:, 1] + Aggregated by max over document chunks. + """ + import torch + + chunks = cls._chunk_document(document, cls._chunk_size) or [""] + texts = ["predict: " + cls.tokenizer.eos_token.join([chunk, claim]) + for chunk in chunks] + enc = cls.tokenizer( + texts, + max_length=cls._max_input_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + decoder_input_ids = torch.zeros( + (enc["input_ids"].size(0), 1), dtype=torch.long) + with torch.no_grad(): + logits = cls.model( + input_ids=enc["input_ids"], + attention_mask=enc["attention_mask"], + decoder_input_ids=decoder_input_ids, + ).logits.squeeze(1) + # Label token ids from the official MiniCheck code: 3=no support, 209=support + label_probs = torch.nn.functional.softmax( + logits[:, torch.tensor([3, 209])], dim=-1) + support_probs = label_probs[:, 1] + return float(support_probs.max().item()) + @classmethod def eval(cls, input_data: Data) -> EvalDetail: """ - Evaluate hallucination using HHEM-2.1-Open model. + Evaluate hallucination using the MiniCheck-Flan-T5-Large model. Args: input_data: Data object containing content and context @@ -118,9 +203,9 @@ def eval(cls, input_data: Data) -> EvalDetail: result.status = True # result.type = cls.metric_type # result.name = "MISSING_CONTEXT" - # result.reason = ["Context is required for HHEM hallucination detection but was not provided"] + # result.reason = ["Context is required for hallucination detection but was not provided"] result.label = [f"{cls.metric_type}.MISSING_CONTEXT"] - result.reason = ["Context is required for HHEM hallucination detection but was not provided"] + result.reason = ["Context is required for hallucination detection but was not provided"] return result else: contexts = input_data.context @@ -142,20 +227,15 @@ def eval(cls, input_data: Data) -> EvalDetail: response = input_data.content - # Create premise-hypothesis pairs for HHEM evaluation - # Format: (premise, hypothesis) where premise=context, hypothesis=response - pairs = [(context, response) for context in context_list] - try: - # Use HHEM model's official predict() method - # This returns consistency scores (0=hallucinated, 1=consistent) - scores = cls.model.predict(pairs) - - # Convert to list if tensor - consistency_scores = scores.tolist() if hasattr(scores, 'tolist') else list(scores) - - # HHEM returns consistency scores (0=hallucinated, 1=consistent) - # We convert to hallucination scores (1=hallucinated, 0=consistent) + # Score each context with MiniCheck: P(response supported by context). + # support prob in [0,1], 1 = fully grounded / consistent. + consistency_scores = [ + cls._support_prob(context, response) for context in context_list + ] + + # Convert support probabilities to hallucination scores + # (1 = hallucinated / ungrounded, 0 = consistent) hallucination_scores = [1.0 - score for score in consistency_scores] # Average hallucination score across all contexts @@ -174,7 +254,7 @@ def eval(cls, input_data: Data) -> EvalDetail: # Generate detailed analysis analysis_parts = [ - f"🔍 HHEM-2.1-Open 幻觉检测分析", + "🔍 MiniCheck 幻觉检测分析", f"📊 平均幻觉分数: {avg_hallucination_score:.3f} (阈值: {cls.dynamic_config.threshold})", f"📝 评估上下文数量: {len(context_list)}" ] @@ -209,7 +289,7 @@ def eval(cls, input_data: Data) -> EvalDetail: f"🚨 结论: 检测到幻觉 (分数 {avg_hallucination_score:.3f} > 阈值 {cls.dynamic_config.threshold})", " 回答与提供的上下文存在显著矛盾", "", - "💡 模型信息: 使用 Vectara HHEM-2.1-Open (本地推理)" + "💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理)" ]) # result.reason = ["\n".join(analysis_parts)] @@ -222,11 +302,11 @@ def eval(cls, input_data: Data) -> EvalDetail: # Generate analysis for non-hallucination case analysis = ( - f"✅ HHEM-2.1-Open 幻觉检测分析\n" + f"✅ MiniCheck 幻觉检测分析\n" f"📊 平均幻觉分数: {avg_hallucination_score:.3f} (阈值: {cls.dynamic_config.threshold})\n" f"📝 评估上下文数量: {len(context_list)}\n" f"🎉 结论: 未检测到幻觉，回答与上下文基本一致\n" - f"💡 模型信息: 使用 Vectara HHEM-2.1-Open (本地推理)" + f"💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理)" ) # result.reason = [analysis] result.reason = [analysis] @@ -238,10 +318,10 @@ def eval(cls, input_data: Data) -> EvalDetail: result = EvalDetail(metric=cls.__name__) result.status = True # result.type = cls.metric_type - # result.name = "HHEM_ERROR" - # result.reason = [f"HHEM model inference failed: {str(e)}"] - result.label = [f"{cls.metric_type}.HHEM_ERROR"] - result.reason = [f"HHEM model inference failed: {str(e)}"] + # result.name = "MINICHECK_ERROR" + # result.reason = [f"MiniCheck model inference failed: {str(e)}"] + result.label = [f"{cls.metric_type}.MINICHECK_ERROR"] + result.reason = [f"MiniCheck model inference failed: {str(e)}"] return result @classmethod @@ -261,7 +341,7 @@ def evaluate_with_detailed_output(cls, input_data: Data) -> dict: # "assessment_type": result.type, # "assessment_name": result.name, "analysis": result.reason[0] if result.reason else "", - "model_info": "HHEM-2.1-Open (Vectara)" + "model_info": "MiniCheck-Flan-T5-Large" } @classmethod diff --git a/docs/hallucination_detection_guide.md b/docs/hallucination_detection_guide.md index 4da27e0d..8e44fb3a 100644 --- a/docs/hallucination_detection_guide.md +++ b/docs/hallucination_detection_guide.md @@ -1,6 +1,8 @@ # Dingo Hallucination Detection - Complete Guide -This guide introduces how to use integrated hallucination detection features in Dingo, supporting two detection methods: **HHEM-2.1-Open local model** (recommended) and **GPT-based cloud detection**. +This guide introduces how to use integrated hallucination detection features in Dingo, supporting two detection methods: **MiniCheck local model** (recommended) and **GPT-based cloud detection**. + +> Note: the local rule is still named `RuleHallucinationHHEM` for backward compatibility, but its underlying model was upgraded from Vectara HHEM-2.1-Open to `lytang/MiniCheck-Flan-T5-Large` (a standard T5 grounding checker, arXiv:2404.10774). ## 🎯 Feature Overview @@ -57,7 +59,7 @@ context = {"passages": ["Context 1", "Context 2"]} ## 🚀 Quick Start -### Method 1: HHEM-2.1-Open Local Model (Recommended ⭐) +### Method 1: MiniCheck Local Model (Recommended ⭐) **Advantages**: - ✅ Fast speed @@ -72,7 +74,7 @@ context = {"passages": ["Context 1", "Context 2"]} pip install dingo-python[hhem] # Or install dependencies manually -pip install sentence-transformers torch +pip install transformers torch sentencepiece ``` **Usage**: @@ -82,7 +84,7 @@ from dingo.config.input_args import EvaluatorRuleArgs from dingo.io.input import Data from dingo.model.rule.rule_hallucination_hhem import RuleHallucinationHHEM -# Configure (first run will auto-download model ~400MB) +# Configure (first run will auto-download model ~3GB, flan-t5-large) RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs( threshold=0.5 # Hallucination threshold, higher = stricter ) @@ -208,7 +210,7 @@ print(f"Pass rate: {summary.score}%") ### Threshold Adjustment ```python -# Method 1: Rule-based (HHEM) +# Method 1: Rule-based (MiniCheck) RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs( threshold=0.5 # Range: 0.0-1.0 ) @@ -227,38 +229,33 @@ LLMHallucination.dynamic_config = EvaluatorLLMArgs( - **General scenarios** (Q&A systems): 0.5-0.6 - **Loose scenarios** (creative content): 0.7-0.8 -### Device Selection (HHEM Only) - -```python -# Auto-select (default: uses GPU if available) -RuleHallucinationHHEM.dynamic_config = EvaluatorRuleArgs() +### Device (MiniCheck) -# Force CPU -import torch -RuleHallucinationHHEM.device = "cpu" +By default the MiniCheck model is loaded and runs on **CPU**, which is +sufficient for the 0.8B flan-t5-large checkpoint. If you want to run inference +on GPU, move the loaded model to your device after the first load: -# Force GPU -RuleHallucinationHHEM.device = "cuda" - -# Specific GPU -RuleHallucinationHHEM.device = "cuda:0" +```python +# Trigger a load, then move to GPU +RuleHallucinationHHEM.load_model() +RuleHallucinationHHEM.model = RuleHallucinationHHEM.model.to("cuda") ``` ## 📈 Performance Comparison -| Feature | HHEM-2.1-Open | GPT-based | +| Feature | MiniCheck (local) | GPT-based | |---------|---------------|-----------| -| **Speed** | Fast (~50ms/sample) | Slower (~1-2s/sample) | +| **Speed** | Fast (~200-500ms/sample on CPU) | Slower (~1-2s/sample) | | **Cost** | Free | API costs | -| **Accuracy** | High (F1: 0.84) | Very High | +| **Accuracy** | High (75.0 balanced acc. on LLM-AggreFact) | Very High | | **Privacy** | Local, secure | Data sent to API | -| **Deployment** | Needs model download (~400MB) | Needs API key | +| **Deployment** | Needs model download (~3GB) | Needs API key | | **Offline** | ✅ Supported | ❌ Requires network | **Recommendations**: -- **Production environment**: HHEM-2.1-Open (fast, free, private) +- **Production environment**: MiniCheck (fast, free, private) - **High-precision scenarios**: GPT-based (highest accuracy) -- **Offline scenarios**: HHEM-2.1-Open (can run completely offline) +- **Offline scenarios**: MiniCheck (can run completely offline) ## 🌟 Best Practices @@ -321,21 +318,23 @@ data = Data( ## ❓ FAQ -### Q1: HHEM vs GPT-based, which to choose? +### Q1: MiniCheck vs GPT-based, which to choose? -- **Production/large-scale**: HHEM (fast, free, private) +- **Production/large-scale**: MiniCheck (fast, free, private) - **High-precision evaluation**: GPT-based (highest accuracy, but has costs) -- **Offline scenarios**: HHEM (can run completely offline) +- **Offline scenarios**: MiniCheck (can run completely offline) -### Q2: Why does HHEM download model on first run? +### Q2: Why does the local rule download a model on first run? -HHEM uses Sentence-Transformers model (~400MB), auto-downloads and caches on first run. Subsequent runs load directly from cache, no re-download needed. +The local rule uses the `lytang/MiniCheck-Flan-T5-Large` model (~3GB), +auto-downloads and caches on first run. Subsequent runs load directly from +cache, no re-download needed. ### Q3: What if model download fails? ```bash # Manually download -huggingface-cli download vectara/hallucination_evaluation_model --local-dir ~/.cache/huggingface/hub/models--vectara--hallucination_evaluation_model +hf download lytang/MiniCheck-Flan-T5-Large # Or use mirror export HF_ENDPOINT=https://hf-mirror.com @@ -352,7 +351,7 @@ export HF_ENDPOINT=https://hf-mirror.com - [RAG Evaluation Metrics Guide](rag_evaluation_metrics.md) - [Factuality Assessment Guide](factuality_assessment_guide.md) -- [HHEM Paper](https://arxiv.org/abs/2406.09053) +- [MiniCheck Paper](https://arxiv.org/abs/2404.10774) ## 📝 Example Scenarios diff --git a/docs/hallucination_guide.md b/docs/hallucination_guide.md index 521a12a4..d546f1bc 100644 --- a/docs/hallucination_guide.md +++ b/docs/hallucination_guide.md @@ -1,6 +1,8 @@ # Dingo 幻觉检测功能完整指南 -本指南介绍如何在 Dingo 中使用集成的幻觉检测功能，支持两种检测方案：**HHEM-2.1-Open 本地模型**（推荐）和 **GPT-based 云端检测**。 +本指南介绍如何在 Dingo 中使用集成的幻觉检测功能，支持两种检测方案：**MiniCheck 本地模型**（推荐）和 **GPT-based 云端检测**。 + +> 说明：本地规则仍沿用类名 `RuleHallucinationHHEM`（保持向后兼容），但底层模型已从 Vectara HHEM-2.1-Open 升级为 `lytang/MiniCheck-Flan-T5-Large`（标准 T5 事实核查模型，arXiv:2404.10774）。相比 HHEM，MiniCheck 在 LLM-AggreFact 基准上更强（75.0 vs 71.8），且为标准 T5，不再受 transformers 版本限制。 ## 🎯 功能概述 @@ -58,18 +60,18 @@ context = "单个参考上下文" ## 🚀 快速开始 -### 方法一：HHEM-2.1-Open 本地模型（推荐） +### 方法一：MiniCheck 本地模型（推荐） #### 安装依赖 ```bash -pip install transformers torch +pip install transformers torch sentencepiece # 或使用专门的依赖文件 pip install -r requirements/hhem_integration.txt ``` #### 模型下载与镜像 -首次运行会自动从 Hugging Face 下载 HHEM-2.1-Open 模型（约 400MB），之后从本地缓存加载，无需重复下载。 +首次运行会自动从 Hugging Face 下载 `lytang/MiniCheck-Flan-T5-Large` 模型（约 3GB，flan-t5-large），之后从本地缓存加载，无需重复下载。 如果无法访问 `huggingface.co`（如国内网络），可在运行前设置镜像环境变量，让下载走 [hf-mirror.com](https://hf-mirror.com)： @@ -78,7 +80,7 @@ pip install -r requirements/hhem_integration.txt export HF_ENDPOINT=https://hf-mirror.com ``` -> 说明：Dingo 不会强制修改该变量，以免影响能直连官网的环境（如 CI）；是否使用镜像由你自行控制。也可用 `huggingface-cli download vectara/hallucination_evaluation_model` 提前手动下载。 +> 说明：Dingo 不会强制修改该变量，以免影响能直连官网的环境（如 CI）；是否使用镜像由你自行控制。也可用 `hf download lytang/MiniCheck-Flan-T5-Large` 提前手动下载。 #### 基本使用 @@ -140,7 +142,7 @@ print(f"详细原因: {result.reason[0]}") # 包含幻觉分数等详细信息 ## 📊 批量数据集评估 -### 使用 HHEM-2.1-Open（本地，免费） +### 使用 MiniCheck 本地模型（本地，免费） ```python from dingo.config import InputArgs @@ -159,7 +161,7 @@ input_data = { } }, "executor": { - "rule_list": ["RuleHallucinationHHEM"], # Use HHEM rule instead of LLM + "rule_list": ["RuleHallucinationHHEM"], # 使用本地 MiniCheck 规则替代 LLM "result_save": { "bad": True, "good": True # Also save good examples for comparison @@ -178,7 +180,7 @@ input_args = InputArgs(**input_data) executor = Executor.exec_map["local"](input_args) result = executor.execute() -print(f"HHEM 幻觉检测完成: 发现 {result.bad_count}/{result.total_count} 个问题") +print(f"MiniCheck 幻觉检测完成: 发现 {result.bad_count}/{result.total_count} 个问题") ``` ### 使用 GPT（在线，需要 API） @@ -231,7 +233,7 @@ print(f"GPT 幻觉检测完成: 发现 {result.bad_count}/{result.total_count} ```python # 方式1: 直接设置类属性 -RuleHallucinationHHEM.dynamic_config.threshold = 0.3 # HHEM 更严格的检测 +RuleHallucinationHHEM.dynamic_config.threshold = 0.3 # MiniCheck 更严格的检测 LLMHallucination.threshold = 0.3 # GPT 更严格的检测 # 方式2: 通过配置文件 @@ -261,7 +263,7 @@ LLMHallucination.threshold = 0.3 # GPT 更严格的检测 ### 性能优化配置 ```python -# HHEM 批量处理优化 +# MiniCheck 批量处理优化 RuleHallucinationHHEM.load_model() # 预加载模型 results = RuleHallucinationHHEM.batch_evaluate(data_list) # 批量更高效 @@ -319,16 +321,19 @@ result.reason # List[str]: 详细分析原因（包含幻觉分数信 ### 典型输出示例 -#### HHEM 输出示例 +#### MiniCheck 输出示例 ``` -HHEM 幻觉分数: 0.650 (阈值: 0.500) -处理了 2 个上下文对： +🔍 MiniCheck 幻觉检测分析 +📊 平均幻觉分数: 0.350 (阈值: 0.500) +📝 评估上下文数量: 2 +❌ 发现 1 个潜在矛盾: + 1. 上下文: "爱因斯坦在1921年获得诺贝尔奖。" + 一致性分数: 0.350, 幻觉分数: 0.650 +✅ 1 个上下文与回答一致: 1. 上下文: "爱因斯坦因发现光电效应获得诺贝尔奖。" - 一致性: 0.95 → 幻觉分数: 0.05 - 2. 上下文: "爱因斯坦在1921年获得诺贝尔奖。" - 一致性: 0.35 → 幻觉分数: 0.65 -平均幻觉分数: 0.350 -❌ 检测到幻觉: 超过阈值 0.500 + 一致性分数: 0.950, 幻觉分数: 0.050 +🚨 结论: 检测到幻觉 (分数 0.350 > 阈值 0.500) +💡 模型信息: 使用 MiniCheck-Flan-T5-Large (本地推理) ``` #### GPT 输出示例 @@ -366,7 +371,7 @@ HHEM 幻觉分数: 0.650 (阈值: 0.500) ### 1. RAG 系统质量监控 ```python -# 实时基于RAG监控回答质量（使用本地HHEM） +# 实时基于RAG监控回答质量（使用本地 MiniCheck） def monitor_rag_response(question, generated_answer, retrieved_docs): data = Data( data_id=f"rag_{timestamp}", @@ -376,7 +381,6 @@ def monitor_rag_response(question, generated_answer, retrieved_docs): ) result = RuleHallucinationHHEM.eval(data) # 本地、快速、免费 - if result.status: logger.warning(f"检测到幻觉: {result.reason[0]}") # 触发人工审核或回答重生成 @@ -385,7 +389,7 @@ def monitor_rag_response(question, generated_answer, retrieved_docs): ### 2. SFT 数据集预处理 ```python -# 训练前检查SFT数据质量（批量处理使用HHEM） +# 训练前检查SFT数据质量（批量处理使用 MiniCheck） input_data = { "input_path": "sft_training_data.jsonl", "custom_config": { @@ -404,7 +408,7 @@ def filter_hallucinated_responses(responses_with_context): for item in responses_with_context: data = Data(**item) - # 使用本地HHEM进行快速检测 + # 使用本地 MiniCheck 进行快速检测 result = RuleHallucinationHHEM.eval(data) if not result.status: # 无幻觉 @@ -424,7 +428,7 @@ class RAGWithHallucinationDetection: self.retriever = retriever self.llm = llm self.detector = hallucination_detector - # 预加载HHEM模型以提高性能 + # 预加载 MiniCheck 模型以提高性能 self.detector.load_model() def generate_answer(self, question): @@ -486,13 +490,13 @@ dingo/ │ ├── llm/ │ │ └── llm_hallucination.py # GPT-based 检测（DeepEval风格） │ ├── rule/ -│ │ └── rule_hallucination_hhem.py # HHEM-2.1-Open 集成 +│ │ └── rule_hallucination_hhem.py # MiniCheck-Flan-T5-Large 集成 │ ├── prompt/prompt_hallucination.py # GPT 提示词模板 │ └── response/response_hallucination.py # 响应数据结构 ├── io/input/Data.py # 扩展Data类支持context ├── examples/hallucination/ # 使用示例 -│ ├── sdk_rule_hhem_detection.py # Rule-based HHEM 使用示例 +│ ├── sdk_rule_hhem_detection.py # Rule-based MiniCheck 使用示例 │ ├── sdk_hallucination_detection.py # GPT 使用示例 │ └── dataset_hallucination_evaluation.py # 批量评估示例 -└── requirements/hhem_integration.txt # HHEM 依赖 +└── requirements/hhem_integration.txt # MiniCheck 依赖 ``` diff --git a/docs/metrics.md b/docs/metrics.md index 10d27c94..40ef8d29 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -41,7 +41,7 @@ This document provides comprehensive information about all quality metrics used | `LLMText3HHarmless` | LLMText3HHarmless | Checks if responses avoid harmful content, discriminatory language, and dangerous assistance | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `LLMText3HHelpful` | LLMText3HHelpful | Assesses if responses address questions directly and follow instructions appropriately | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | | `LLMText3HHonest` | LLMText3HHonest | Evaluates if responses provide accurate information without fabrication or deception | [Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback](https://arxiv.org/pdf/2204.05862) (Bai et al., 2022) | [📊 See Results](eval/prompt/qa_data_evaluated_by_3h.md) | N/A | -| `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses Vectara's HHEM-2.1-Open model for local hallucination detection by evaluating consistency between response and c... | [HHEM-2.1-Open](https://huggingface.co/vectara/hallucination_evaluation_model) (Forrest Bao, Miaoran Li, Rogger Luo, Ofer Mendelevitch) | N/A | N/A | +| `QUALITY_BAD_HALLUCINATION` | RuleHallucinationHHEM | Uses the MiniCheck-Flan-T5-Large model for local hallucination detection by checking whether the response is grounded... | [MiniCheck: Efficient Fact-Checking of LLMs on Grounding Documents](https://arxiv.org/abs/2404.10774) (Liyan Tang, Philippe Laban, Greg Durrett) | N/A | N/A | ### Classification Metrics @@ -144,7 +144,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleSourceFieldValidation, RuleQuanliangFieldValidation, RuleAuthorFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex source fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics diff --git a/docs/rules.md b/docs/rules.md index ddbdfafa..65789533 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -28,7 +28,7 @@ The specific rules for each quality metric are as follows: | RuleEnterAndSpace | EFFECTIVENESS | Check abnormal combinations of line breaks and spaces. | | | RuleEnterMore | EFFECTIVENESS | Check whether content has excessive consecutive line breaks. | | | RuleEnterRatioMore | EFFECTIVENESS | Check whether the line-break ratio is excessive. | | -| RuleHallucinationHHEM | HALLUCINATION | Detect hallucinations with HHEM-2.1-Open. | | +| RuleHallucinationHHEM | HALLUCINATION | Detect ungrounded claims with MiniCheck-Flan-T5-Large (checks whether the response is supported by the context). | | | RuleHeadWordAr | RELEVANCE | Check Arabic content for irrelevant source information. | | | RuleHeadWordCs | RELEVANCE | Check Czech content for irrelevant source information. | | | RuleHeadWordHu | RELEVANCE | Check Hungarian content for irrelevant source information. | | diff --git a/examples/hallucination/sdk_rule_hhem_detection.py b/examples/hallucination/sdk_rule_hhem_detection.py index 070720f5..1d2832dd 100644 --- a/examples/hallucination/sdk_rule_hhem_detection.py +++ b/examples/hallucination/sdk_rule_hhem_detection.py @@ -1,15 +1,19 @@ """ -Dingo Rule-based HHEM-2.1-Open Hallucination Detection Example +Dingo Rule-based MiniCheck Hallucination Detection Example -This example demonstrates how to use the HHEM-2.1-Open model as a rule-based -hallucination detection tool for efficient local inference without API costs. +This example demonstrates how to use the RuleHallucinationHHEM rule (backed by +the MiniCheck-Flan-T5-Large model) as a local, API-free tool for detecting +ungrounded claims — i.e. whether a response is supported by its context. -Rule-based HHEM offers: +Rule-based MiniCheck offers: - Better architecture fit (rules vs LLM for deterministic local models) -- Superior performance than GPT-3.5 and GPT-4 on benchmarks -- Local inference with <600MB RAM usage -- Fast processing (~1.5s for 2k tokens on modern CPU) -- No API costs or rate limits +- Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8) +- Standard T5 model — no transformers version pin, no remote code +- Local inference, no API costs or rate limits + +Note: the rule/class is still named RuleHallucinationHHEM for backward +compatibility; the underlying model was upgraded from Vectara HHEM-2.1-Open +to MiniCheck. """ from dingo.io.input import Data @@ -35,7 +39,7 @@ def example_1_basic_rule_hhem_detection(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") + print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print(f"Threshold: {RuleHallucinationHHEM.dynamic_config.threshold}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") @@ -61,7 +65,7 @@ def example_2_no_hallucination_rule(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") + print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") print() @@ -89,7 +93,7 @@ def example_3_complex_scenario_rule(): print(f"Error Status: {result.status}") # True = hallucination detected, False = no hallucination print(f"Label: {result.label}") - print(f"HHEM Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") + print(f"Hallucination Score: {f'{result.score:.3f}' if result.score is not None else 'N/A'}") print("\nDetailed Analysis:") print(result.reason[0] if result.reason else "N/A") print() @@ -204,27 +208,26 @@ def example_7_performance_benchmark_rule(): result = RuleHallucinationHHEM.eval(data) end_time = time.time() - print(f"Rule-based HHEM Inference Time: {end_time - start_time:.3f} seconds") + print(f"Rule-based MiniCheck Inference Time: {end_time - start_time:.3f} seconds") print(f"Result: Error={result.status}, Score={f'{result.score:.3f}' if result.score is not None else 'N/A'}") - print(f"Model Info: Local HHEM-2.1-Open (Rule-based)") + print(f"Model Info: Local MiniCheck-Flan-T5-Large (Rule-based)") print() if __name__ == "__main__": - print("🔍 Dingo Rule-based HHEM-2.1-Open Hallucination Detection Examples") + print("🔍 Dingo Rule-based MiniCheck Hallucination Detection Examples") print("=" * 70) print() - print("💡 Rule-based HHEM-2.1-Open Advantages:") + print("💡 Rule-based MiniCheck Advantages:") print("- Better architecture: Rules for deterministic local models") print("- Local inference (no API costs)") - print("- High performance (better than GPT-3.5/GPT-4 on benchmarks)") - print("- Low resource usage (<600MB RAM)") - print("- Fast inference (~1.5s for 2k tokens)") + print("- Strong grounding accuracy (75.0 on LLM-AggreFact, > HHEM's 71.8)") + print("- Standard T5 model (no transformers version pin, no remote code)") print() - print("⚠️ First run will download the model (~400MB)") - print("⚠️ Requires: pip install transformers") + print("⚠️ First run will download the model (~3GB, flan-t5-large)") + print("⚠️ Requires: pip install transformers torch sentencepiece") print() try: @@ -236,11 +239,11 @@ def example_7_performance_benchmark_rule(): example_6_threshold_comparison_rule() example_7_performance_benchmark_rule() - print("🎉 All Rule-based HHEM examples completed successfully!") + print("🎉 All Rule-based MiniCheck examples completed successfully!") except ImportError as e: print(f"❌ Import Error: {e}") - print("Please install required dependencies: pip install transformers") + print("Please install required dependencies: pip install transformers torch sentencepiece") except Exception as e: print(f"❌ Error: {e}") print("Please check your setup and try again") @@ -248,8 +251,7 @@ def example_7_performance_benchmark_rule(): print() print("📈 Rule-based vs LLM-based Comparison:") print("- Architecture: Rule-based (✓) vs LLM-based (for API models)") - print("- Performance: HHEM-2.1 > GPT-4 > GPT-3.5") + print("- Accuracy: MiniCheck competitive with much larger LLM fact-checkers") print("- Cost: Rule-based (Free) vs LLM-based (API costs)") - print("- Speed: Rule-based (~1.5s) vs LLM-based (3-10s)") print("- Privacy: Rule-based (Local) vs LLM-based (Cloud)") - print("- Resource: Rule-based (<600MB) vs LLM-based (API dependency)") + print("- Dependency: Rule-based (Local model) vs LLM-based (API dependency)") diff --git a/requirements/hhem_integration.txt b/requirements/hhem_integration.txt index 01a849fa..2158f203 100644 --- a/requirements/hhem_integration.txt +++ b/requirements/hhem_integration.txt @@ -1,16 +1,19 @@ -# HHEM-2.1-Open Integration Dependencies -# Required for Vectara HHEM-2.1-Open hallucination detection model +# Hallucination Detection Dependencies (MiniCheck-Flan-T5-Large) +# Required for the RuleHallucinationHHEM rule, which detects ungrounded claims +# by checking whether a response is supported by its context. +# +# Model: lytang/MiniCheck-Flan-T5-Large (arXiv:2404.10774). +# It is a standard T5ForConditionalGeneration checkpoint (no custom remote +# code), so it loads on any modern transformers version — the previous +# <4.49 pin (needed by Vectara HHEM's remote code) is no longer required. -# Core transformers library for HHEM model -# 上限 <4.49：HHEM 官方 remote code (modeling_hhem_v2.py) 为旧版 transformers 编写， -# transformers 4.49+ 的加载流程会访问 all_tied_weights_keys 属性，旧 remote code 未实现， -# 导致 "'HHEMv2ForSequenceClassification' object has no attribute 'all_tied_weights_keys'"。 -transformers>=4.30.0,<4.49 +# Core transformers library +transformers>=4.30.0 -# PyTorch (CPU version should be sufficient for HHEM) +# PyTorch (CPU version is sufficient) torch>=1.12.0 -# Additional dependencies for model operations +# Tokenizer dependencies for flan-t5 (SentencePiece) sentencepiece>=0.1.99 tokenizers>=0.13.0 diff --git a/setup.py b/setup.py index d4635707..a3ca2be4 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,10 @@ def _read_requirements(path): hhem_requirements = _read_requirements("./requirements/hhem_integration.txt") litellm_requirements = ["litellm>=1.80.0,<1.87.0"] retrieval_requirements = _read_requirements("./requirements/retrieval.txt") -# lmdeploy 单独成组：它硬性要求 transformers>=4.56，与 HHEM 需要的 transformers<4.49 冲突， -# 因此不并入 optional/all，避免同一环境内 HHEM 无法加载。需要时单独 pip install dingo-python[lmdeploy]。 +# lmdeploy 单独成组：它硬性要求 transformers>=4.56 并拉入大量重依赖，作为可选推理后端不并入 +# optional/all，保持默认环境轻量。需要时单独 pip install dingo-python[lmdeploy]。 +# （注：幻觉检测模型已从 Vectara HHEM 换为标准 T5 的 MiniCheck，不再有 transformers<4.49 上限， +# 故 lmdeploy 与幻觉检测不再存在版本冲突；此处隔离仅出于依赖体量考虑。） lmdeploy_requirements = ["lmdeploy"] From 9260291ebf2f23f421af523a0e0d3ef3e0229016 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Tue, 4 Aug 2026 02:41:49 +0000 Subject: [PATCH 79/80] =?UTF-8?q?=F0=9F=93=9A=20Auto-update=20metrics=20do?= =?UTF-8?q?cumentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics.md b/docs/metrics.md index 40ef8d29..6c8739de 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -144,7 +144,7 @@ This document provides comprehensive information about all quality metrics used | Type | Metric | Description | Paper Source | Evaluation Results | Examples | |------|--------|-------------|--------------|-------------------|----------| -| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleSourceFieldValidation, RuleQuanliangFieldValidation, RuleAuthorFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex source fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | +| `QUALITY_BAD_EFFECTIVENESS` | RuleMetadataSimilarity, RuleAuthorFieldValidation, RuleQuanliangFieldValidation, RuleSourceFieldValidation | 检查元数据字段与基准数据的相似度匹配，阈值默认为0.6; Validate OpenAlex author fields and report invalid fields; Validate Quanliang metadata f... | Internal Implementation | N/A | N/A | ### Rule-Based RESUME Quality Metrics From e212cf0874e45876575d40a234a1b62c09181f1c Mon Sep 17 00:00:00 2001 From: sjshailab Date: Tue, 4 Aug 2026 18:50:18 +0800 Subject: [PATCH 80/80] feat: update setup v2.5.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a3ca2be4..b5e7b6ea 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ def _read_requirements(path): setup( name="dingo-python", - version="2.4.0", + version="2.5.0", author="Dingo", description="A Comprehensive AI Data Quality Evaluation Tool for Large Models", long_description=long_description,