feat: v2.5.0

.github/workflows/lint.yml

@@ -2,6 +2,9 @@ name: Lint
 
 on: [push, pull_request]
 
+permissions:
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -29,33 +32,5 @@ jobs:
         run: |
           python .github/scripts/check_imports.py
 
-      - name: Run pre-commit (auto-fix)
-        id: pre_commit_auto_fix
-        run: |
-          # 运行 pre-commit，允许自动修复，不因修复而失败
-          pre-commit run --all-files || true
-
-      - name: Check for changes
-        id: check_changes
-        run: |
-          if [[ -n $(git status --porcelain) ]]; then
-            echo "changed=true" >> $GITHUB_OUTPUT
-            echo "📝 Files were modified by pre-commit auto-fix"
-          else
-            echo "changed=false" >> $GITHUB_OUTPUT
-            echo "✅ No auto-fix changes"
-          fi
-
-      - name: Commit auto-fix changes
-        if: steps.check_changes.outputs.changed == 'true' && github.event_name == 'push'
-        run: |
-          git config --local user.email "action@github.com"
-          git config --local user.name "GitHub Action"
-          git add -A
-          git commit -m "🎨 Auto-format code with pre-commit"
-          git push
-
-      - name: Run pre-commit (final check)
-        run: |
-          # 再次运行 pre-commit，这次如果有错误就真的失败
-          pre-commit run --all-files
+      - name: Run pre-commit
+        run: pre-commit run --all-files

AGENTS.md

@@ -49,6 +49,8 @@ dingo/
 │   │   ├── rule/            ← Rule-based evaluators (80+ built-in)
 │   │   │   ├── base.py      ← BaseRule
 │   │   │   ├── rule_common.py ← Common rules (text quality, format, PII, etc.)
+│   │   │   ├── guobiao/
+│   │   │   │   └── rule_tc609_quality.py ← TC609 quality metrics and placeholders
 │   │   │   ├── rule_search_ranking.py ← IR ranking metrics (NDCG, MRR, Recall, Precision, MAP, HitRate)
 │   │   │   └── utils/       ← Shared utilities (normalize, ngrams, etc.)
 │   │   └── llm/             ← LLM-based evaluators

dingo/config/input_args.py

@@ -25,6 +25,7 @@ class DatasetSqlArgs(BaseModel):
     port: str = ''
     database: str = ''
     connect_args: str = ''  # 连接参数，如 ?charset=utf8mb4
+    engine_args: str = ''  # SQLAlchemy 引擎参数，如 pool_recycle=1800&pool_pre_ping=true
 
 
 class DatasetExcelArgs(BaseModel):
@@ -193,6 +194,3 @@ class InputArgs(BaseModel):
     dataset: DatasetArgs = DatasetArgs()
     executor: ExecutorArgs = ExecutorArgs()
     evaluator: List[EvalPipline] = []
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)

dingo/data/converter/base.py

@@ -238,6 +238,23 @@ def _convert(raw: Union[str, Dict]):
         return _convert
 
 
+@BaseConverter.register("md")
+class MarkdownConverter(BaseConverter):
+    """Markdown file converter."""
+
+    def __init__(self):
+        super().__init__()
+
+    @classmethod
+    def convertor(cls, input_args: InputArgs) -> Callable:
+        def _convert(raw: Union[str, Dict]):
+            if isinstance(raw, Dict):
+                return Data(**raw)
+            return Data(**{"id": "", "content": str(raw)})
+
+        return _convert
+
+
 @BaseConverter.register("jsonl")
 class JsonLineConverter(BaseConverter):
     """Json line file converter."""

dingo/data/datasource/local.py

@@ -32,7 +32,7 @@ def to_dict(self) -> Dict[str, Any]:
             "config_name": self.config_name,
         }
 
-    def load(self, **kwargs) -> Generator[str, None, None]:
+    def load(self, **kwargs) -> Generator[Any, None, None]:
         """Load the local file dataset based on `LocalDataSource`.
         Args:
             kwargs: Additional keyword arguments used for loading the dataset.
@@ -51,10 +51,39 @@ def _find_all_files(self, path: str, file_list: List[str]):
         for _f in os.listdir(path):
             f = os.path.join(path, _f)
             if os.path.isfile(f):
-                file_list.append(f)
+                if getattr(self.input_args.dataset, "format", None) == "jsonl":
+                    if f.lower().endswith(".jsonl"):
+                        file_list.append(f)
+                elif getattr(self.input_args.dataset, "format", None) == "json":
+                    if f.lower().endswith(".json"):
+                        file_list.append(f)
+                elif getattr(self.input_args.dataset, "format", None) == "md":
+                    if f.lower().endswith(".md"):
+                        file_list.append(f)
+                else:
+                    file_list.append(f)
             if os.path.isdir(f):
                 self._find_all_files(f, file_list)
 
+    @staticmethod
+    def _load_md_file(path: str) -> Dict[str, str]:
+        try:
+            with open(path, "r", encoding="utf-8") as md_file:
+                return {
+                    "id": os.path.basename(path),
+                    "content": md_file.read(),
+                }
+        except UnicodeDecodeError as decode_error:
+            raise RuntimeError(
+                f'Failed to read markdown file "{path}": Unsupported file encoding. '
+                f'Markdown files must be UTF-8 encoded. Original error: {str(decode_error)}'
+            )
+        except Exception as e:
+            raise RuntimeError(
+                f'Unexpected error reading markdown file "{path}": {str(e)}. '
+                f'Please check if the file exists and is readable.'
+            )
+
     def _load_excel_file_xlsx(self, path: str) -> Generator[str, None, None]:
         """
         Load an .xlsx Excel file and return its contents row by row as JSON strings.
@@ -377,7 +406,7 @@ def _load_excel_file_xls(self, path: str) -> Generator[str, None, None]:
             if wb:
                 wb.release_resources()
 
-    def _load_local_file(self) -> Generator[str, None, None]:
+    def _load_local_file(self) -> Generator[Any, None, None]:
         """
         Load a local file and return its contents.
 
@@ -390,14 +419,27 @@ def _load_local_file(self) -> Generator[str, None, None]:
             raise RuntimeError(f'"{self.path}" is not a valid path')
 
         f_list = []
+        input_is_file = os.path.isfile(self.path)
         if os.path.exists(self.path) and os.path.isfile(self.path):
             f_list = [self.path]
         elif os.path.exists(self.path) and os.path.isdir(self.path):
             self._find_all_files(self.path, f_list)
 
-        by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2"]
+        by_line = self.input_args.dataset.format not in ["json", "listjson", "mineru", "mineru_v2", "md"]
 
         for f in f_list:
+            if self.input_args.dataset.format == "md":
+                if f.lower().endswith(".md"):
+                    yield self._load_md_file(f)
+                elif input_is_file:
+                    raise RuntimeError(
+                        f'Input file "{self.path}" is not a markdown file. '
+                        f'Please provide a ".md" file or a directory containing ".md" files when dataset.format is "md".'
+                    )
+                else:
+                    continue
+                continue
+
             # Check if file is CSV
             if f.endswith('.csv'):
                 if self.input_args.dataset.format != 'csv':

dingo/data/datasource/sql.py

@@ -1,14 +1,23 @@
 from typing import Any, Dict, Generator, Optional
+from urllib.parse import parse_qsl
 
 from sqlalchemy import create_engine, text
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import URL, Engine
 
 from dingo.config import InputArgs
 from dingo.data.datasource.base import DataSource
 
 
 @DataSource.register()
 class SqlDataSource(DataSource):
+    _ENGINE_ARG_TYPES = {
+        "pool_pre_ping": "bool",
+        "pool_recycle": "int",
+        "pool_size": "int",
+        "max_overflow": "int",
+        "pool_timeout": "int",
+    }
+
     def __init__(
         self,
         input_args: InputArgs = None,
@@ -33,37 +42,120 @@ def _get_engine(sql_config) -> Engine:
                 "must be set when using SQL datasource."
             )
 
-        # 构建数据库连接URL
-        # SQLite 格式: sqlite:///path/to/database.db
-        # 其他数据库格式: dialect+driver://username:password@host:port/database
-        if sql_config.dialect.lower() == "sqlite":
-            driver_part = f"+{sql_config.driver}" if sql_config.driver else ""
-            connection_url = f"{sql_config.dialect}{driver_part}:///{sql_config.database}"
-        else:
-            # 对于非 SQLite 数据库，需要用户名、密码和主机
-            if not sql_config.username or not sql_config.host:
+        dialect = sql_config.dialect.lower()
+        query_args = SqlDataSource._parse_connect_args(sql_config.connect_args)
+
+        connection_url = SqlDataSource._build_connection_url(sql_config, query_args)
+
+        engine_kwargs: Dict[str, Any] = {"pool_pre_ping": True}
+        if dialect in {"mysql", "mariadb"}:
+            engine_kwargs["pool_recycle"] = 1800
+        engine_kwargs.update(SqlDataSource._parse_engine_args(sql_config.engine_args))
+
+        engine = create_engine(connection_url, **engine_kwargs)
+        return engine
+
+    @staticmethod
+    def _build_driver_name(sql_config) -> str:
+        return (
+            f"{sql_config.dialect}+{sql_config.driver}"
+            if sql_config.driver
+            else sql_config.dialect
+        )
+
+    @staticmethod
+    def _parse_connect_args(connect_args: str) -> Dict[str, str]:
+        return SqlDataSource._parse_query_arg_string(connect_args)
+
+    @staticmethod
+    def _parse_query_arg_string(raw_arg_string: str) -> Dict[str, str]:
+        if not raw_arg_string:
+            return {}
+        normalized = raw_arg_string.strip()
+        if normalized.startswith("?"):
+            normalized = normalized[1:]
+        if not normalized:
+            return {}
+        return {
+            key: value
+            for key, value in parse_qsl(normalized, keep_blank_values=False)
+            if key
+        }
+
+    @staticmethod
+    def _parse_bool_value(raw_value: str, key: str) -> bool:
+        normalized = raw_value.strip().lower()
+        if normalized == "true":
+            return True
+        if normalized == "false":
+            return False
+        raise RuntimeError(
+            f"SQL engine arg '{key}' expects 'true' or 'false', got: {raw_value}."
+        )
+
+    @staticmethod
+    def _parse_engine_args(engine_args: str) -> Dict[str, Any]:
+        raw_engine_args = SqlDataSource._parse_query_arg_string(engine_args)
+        if not raw_engine_args:
+            return {}
+
+        parsed_engine_args: Dict[str, Any] = {}
+        for key, raw_value in raw_engine_args.items():
+            expected_type = SqlDataSource._ENGINE_ARG_TYPES.get(key)
+            if expected_type is None:
+                allowed = ", ".join(sorted(SqlDataSource._ENGINE_ARG_TYPES.keys()))
                 raise RuntimeError(
-                    f"For {sql_config.dialect}, username and host must be set."
+                    f"Unsupported SQL engine arg '{key}'. Allowed keys: {allowed}."
                 )
 
-            driver_part = f"+{sql_config.driver}" if sql_config.driver else ""
-            port_part = f":{sql_config.port}" if sql_config.port else ""
-            password_part = f":{sql_config.password}" if sql_config.password else ""
+            if expected_type == "int":
+                try:
+                    parsed_engine_args[key] = int(raw_value)
+                except ValueError as exc:
+                    raise RuntimeError(
+                        f"SQL engine arg '{key}' expects an integer value, got: {raw_value}."
+                    ) from exc
+            elif expected_type == "bool":
+                parsed_engine_args[key] = SqlDataSource._parse_bool_value(raw_value, key)
+
+        return parsed_engine_args
+
+    @staticmethod
+    def _parse_port(port: str) -> Optional[int]:
+        if not port:
+            return None
+        try:
+            return int(port)
+        except ValueError as exc:
+            raise RuntimeError("SQL connection parameter 'port' must be an integer.") from exc
 
-            connection_url = (
-                f"{sql_config.dialect}{driver_part}://"
-                f"{sql_config.username}{password_part}@"
-                f"{sql_config.host}{port_part}/{sql_config.database}"
+    @staticmethod
+    def _build_connection_url(sql_config, query_args: Dict[str, str]) -> URL:
+        driver_name = SqlDataSource._build_driver_name(sql_config)
+        query = query_args or None
+
+        if sql_config.dialect.lower() == "sqlite":
+            return URL.create(
+                drivername=driver_name,
+                database=sql_config.database,
+                query=query,
             )
 
-        # 添加连接参数（如 ?charset=utf8mb4）
-        if sql_config.connect_args:
-            # 确保参数以 ? 开头
-            args_part = sql_config.connect_args if sql_config.connect_args.startswith('?') else f"?{sql_config.connect_args}"
-            connection_url = f"{connection_url}{args_part}"
+        # 对于非 SQLite 数据库，需要用户名、密码和主机
+        if not sql_config.username or not sql_config.host:
+            raise RuntimeError(
+                f"For {sql_config.dialect}, username and host must be set."
+            )
 
-        engine = create_engine(connection_url)
-        return engine
+        return URL.create(
+            drivername=driver_name,
+            username=sql_config.username,
+            password=sql_config.password or None,
+            host=sql_config.host,
+            port=SqlDataSource._parse_port(sql_config.port),
+            database=sql_config.database,
+            query=query,
+        )
 
     @staticmethod
     def get_source_type() -> str: