Skip to content

feat: update text metrics calculate #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
e06084 merged 1 commit into from
Jul 31, 2025
Merged

feat: update text metrics calculate #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@ WebMainBench 是一个专门用于端到端评测网页正文抽取质量的基

#### 指标详细说明

| 指标名称 | 中文名称 | 取值范围 | 说明 |
| 指标名称 | 计算方式 | 取值范围 | 说明 |
|---------|----------|----------|------|
| `overall` | 综合得分 | 0.0-1.0 | 所有指标的平均值，反映整体抽取质量 |
| `text_edit` | 文本编辑距离 | 0.0-1.0 | 衡量文本内容差异的指标，基于编辑距离计算 |
| `code_edit` | 代码编辑距离 | 0.0-1.0 | 衡量代码内容差异的指标，基于编辑距离计算 |
| `table_TEDS` | 表格编辑距离 | 0.0-1.0 | 表格结构和内容抽取准确性，使用TEDS算法 |
| `table_edit` | 表格编辑距离 | 0.0-1.0 | 衡量表格内容差异的指标，基于编辑距离计算 |
| `formula_edit` | 公式编辑距离 | 0.0-1.0 | 衡量公式内容差异的指标，基于编辑距离计算，包括行内和行间公式 |
| `overall` | 所有成功指标的平均值 | 0.0-1.0 | 综合质量评分，分数越高质量越好 |
| `text_edit` | `1 - (编辑距离 / 最大文本长度)` | 0.0-1.0 | 纯文本相似度，分数越高质量越好 |
| `code_edit` | `1 - (编辑距离 / 最大代码长度)` | 0.0-1.0 | 代码内容相似度，分数越高质量越好 |
| `table_TEDS` | `1 - (树编辑距离 / 最大节点数)` | 0.0-1.0 | 表格结构相似度，分数越高质量越好 |
| `table_edit` | `1 - (编辑距离 / 最大表格长度)` | 0.0-1.0 | 表格内容相似度，分数越高质量越好 |
| `formula_edit` | `1 - (编辑距离 / 最大公式长度)` | 0.0-1.0 | 公式内容相似度，分数越高质量越好 |


### 🏗️ **系统架构**
Expand Down
8 changes: 4 additions & 4 deletions tests/test_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -368,14 +368,14 @@ def test_formula_sample_edit_distance(self):
# 验证公式编辑距离（符号转义导致的固定低分）
self.assertIn("formula_edit", results)
self.assertTrue(results["formula_edit"].success)
self.assertAlmostEqual(results["formula_edit"].score, 0.122807, places=5,
msg=f"formula_edit分数应该是0.122807，实际: {results['formula_edit'].score}")
self.assertAlmostEqual(results["formula_edit"].score, 0.000000, places=5,
msg=f"formula_edit分数应该是0.000000，实际: {results['formula_edit'].score}")

# 验证文本编辑距离（去除公式后的纯文本，也受符号转义影响）
self.assertIn("text_edit", results)
self.assertTrue(results["text_edit"].success)
self.assertAlmostEqual(results["text_edit"].score, 0.372093, places=5,
msg=f"text_edit分数应该是0.372093，实际: {results['text_edit'].score}")
self.assertAlmostEqual(results["text_edit"].score, 0.320000, places=5,
msg=f"text_edit分数应该是0.320000，实际: {results['text_edit'].score}")

def test_overall_score_calculation(self):
"""测试综合分数计算"""
Expand Down
176 changes: 176 additions & 0 deletions webmainbench/metrics/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from dataclasses import dataclass
from typing import Dict, Any, List, Optional, Union
import traceback
import re


@dataclass
Expand Down Expand Up @@ -121,6 +122,181 @@ def batch_calculate(self, predicted_list: List[Any],
results.append(result)
return results

@staticmethod
def split_content(text: str, content_list: List[Dict[str, Any]] = None) -> Dict[str, str]:
"""
统一的内容分割方法，将文本分为代码、公式、表格和剩余文本4个部分。

Args:
text: 原始markdown文本
content_list: 结构化内容列表（来自llm-webkit等）

Returns:
Dict with keys: 'code', 'formula', 'table', 'text'
"""
# 优先从content_list中提取
if content_list:
extracted_content = BaseMetric._extract_from_content_list(content_list)
if any(extracted_content.values()):
return extracted_content

# 从markdown文本中提取
return BaseMetric._extract_from_markdown(text or "")

@staticmethod
def _extract_from_content_list(content_list: List[Dict[str, Any]]) -> Dict[str, str]:
"""从content_list中递归提取各种类型的内容"""
extracted = {
'code': [],
'formula': [],
'table': [],
'text': []
}

def _recursive_extract(items):
if not isinstance(items, list):
return

for item in items:
if not isinstance(item, dict):
continue

item_type = item.get('type', '').lower()
content = item.get('content', '').strip()

# 根据类型分类内容
if item_type in ['code', 'code_block', 'inline_code']:
if content:
extracted['code'].append(content)
elif item_type in ['formula', 'math', 'equation', 'latex']:
if content:
extracted['formula'].append(content)
elif item_type in ['table', 'table_content', 'html_table', 'table_row', 'table_cell']:
if content:
extracted['table'].append(content)
elif item_type in ['text', 'paragraph', 'heading']:
if content:
extracted['text'].append(content)

# 递归处理子元素
for child_key in ['children', 'items', 'content_list']:
if child_key in item and isinstance(item[child_key], list):
_recursive_extract(item[child_key])

_recursive_extract(content_list)

# 将列表转换为字符串
return {
'code': '\n'.join(extracted['code']),
'formula': '\n'.join(extracted['formula']),
'table': '\n'.join(extracted['table']),
'text': '\n'.join(extracted['text'])
}

@staticmethod
def _extract_from_markdown(text: str) -> Dict[str, str]:
"""从markdown文本中提取各种类型的内容"""
if not text:
return {'code': '', 'formula': '', 'table': '', 'text': ''}

# 收集所有需要移除的内容片段
extracted_segments = []

# 提取代码
code_parts = []
# 代码块 ```code```
for match in re.finditer(r'```[\s\S]*?```', text):
code_block = match.group(0)
extracted_segments.append(code_block)
code_parts.append(code_block.strip('`').strip())

# 行内代码 `code`
for match in re.finditer(r'`([^`]+)`', text):
inline_code_full = match.group(0) # 包含反引号的完整匹配
inline_code_content = match.group(1) # 只是内容
extracted_segments.append(inline_code_full)
code_parts.append(inline_code_content)

# 提取公式
formula_parts = []
# 统一的公式提取模式
latex_patterns = [
r'(?<!\\)\$\$([^$]+)\$\$(?!\\)', # Display math (not escaped)
r'(?<!\\)\$([^$\n]+)\$(?![\\\$])', # Inline math (not escaped)
# r'\\begin\{equation\*?\}(.*?)\\end\{equation\*?\}', # Equation environment
# r'\\begin\{align\*?\}(.*?)\\end\{align\*?\}', # Align environment
# r'\\begin\{gather\*?\}(.*?)\\end\{gather\*?\}', # Gather environment
# r'\\begin\{eqnarray\*?\}(.*?)\\end\{eqnarray\*?\}', # Eqnarray environment
# r'\\begin\{multline\*?\}(.*?)\\end\{multline\*?\}', # Multline environment
# r'\\begin\{split\}(.*?)\\end\{split\}', # Split environment
]

for pattern in latex_patterns:
for match in re.finditer(pattern, text, re.DOTALL):
formula_full = match.group(0) # 完整匹配（包含$符号）
formula_content = match.group(1) # 只是公式内容
extracted_segments.append(formula_full)
if formula_content.strip():
formula_parts.append(formula_content.strip())

# 提取表格
table_parts = []

# 1. 提取HTML表格
html_table_pattern = r'<table[^>]*>.*?</table>'
for match in re.finditer(html_table_pattern, text, re.DOTALL | re.IGNORECASE):
html_table = match.group(0)
extracted_segments.append(html_table)
table_parts.append(html_table)

# 2. 提取Markdown表格
lines = text.split('\n')
table_lines = []
in_markdown_table = False

for line in lines:
if '|' in line and line.strip():
table_lines.append(line)
in_markdown_table = True
elif in_markdown_table and line.strip() == '':
# Markdown表格结束（空行）
if table_lines:
md_table = '\n'.join(table_lines)
extracted_segments.append(md_table)
table_parts.append(md_table)
table_lines = []
in_markdown_table = False
elif in_markdown_table:
# 表格内的非表格行，Markdown表格结束
if table_lines:
md_table = '\n'.join(table_lines)
extracted_segments.append(md_table)
table_parts.append(md_table)
table_lines = []
in_markdown_table = False

# 处理文档末尾的Markdown表格
if table_lines:
md_table = '\n'.join(table_lines)
extracted_segments.append(md_table)
table_parts.append(md_table)

# 提取剩余文本（移除所有已提取的内容片段）
clean_text = text
for segment in extracted_segments:
clean_text = clean_text.replace(segment, '', 1)

# 清理多余的空行
clean_text = re.sub(r'\n\s*\n', '\n\n', clean_text)
clean_text = clean_text.strip()

return {
'code': '\n'.join(code_parts),
'formula': '\n'.join(formula_parts),
'table': '\n'.join(table_parts),
'text': clean_text
}

def aggregate_results(self, results: List[MetricResult]) -> MetricResult:
"""
Aggregate multiple metric results.
Expand Down
39 changes: 3 additions & 36 deletions webmainbench/metrics/formula_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,42 +37,9 @@ def _calculate_score(self, predicted: str, groundtruth: str,

def _extract_formula_content(self, text: str, content_list: List[Dict[str, Any]] = None) -> str:
"""从文本和content_list中提取公式内容"""
formula_parts = []

# 优先从content_list中递归提取
if content_list:
formula_parts = self._extract_formulas_from_content_list(content_list)

# 如果content_list中有公式，直接返回
if formula_parts:
return '\n'.join(formula_parts)

# 只有当content_list中没有公式时，才从文本中提取（使用与_extract_formulas一致的逻辑）
if text:
# 使用增强的公式提取模式
latex_patterns = [
r'\$\$([^$]+)\$\$', # Display math
r'(?<!\$)\$([^$\n]+)\$(?!\$)', # Inline math (improved)
r'\\begin\{equation\*?\}(.*?)\\end\{equation\*?\}', # Equation environment
r'\\begin\{align\*?\}(.*?)\\end\{align\*?\}', # Align environment
r'\\begin\{gather\*?\}(.*?)\\end\{gather\*?\}', # Gather environment
r'\\begin\{eqnarray\*?\}(.*?)\\end\{eqnarray\*?\}', # Eqnarray environment
r'\\begin\{multline\*?\}(.*?)\\end\{multline\*?\}', # Multline environment
r'\\begin\{split\}(.*?)\\end\{split\}', # Split environment
]

for pattern in latex_patterns:
matches = re.findall(pattern, text, re.DOTALL)
formula_parts.extend(matches)

# Clean and filter formulas before joining
cleaned_formulas = []
for formula in formula_parts:
formula = formula.strip()
if formula and len(formula) > 1:
cleaned_formulas.append(formula)

return '\n'.join(cleaned_formulas)
# 使用统一的内容分割方法
content_parts = self.split_content(text, content_list)
return content_parts.get('formula', '')

def _extract_formulas_from_content_list(self, content_list: List[Dict[str, Any]]) -> List[str]:
"""递归从content_list中提取公式内容"""
Expand Down
47 changes: 6 additions & 41 deletions webmainbench/metrics/table_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,44 +38,9 @@ def _calculate_score(self, predicted: str, groundtruth: str,

def _extract_table_content(self, text: str, content_list: List[Dict[str, Any]] = None) -> str:
"""从文本和content_list中提取表格内容"""
table_parts = []

# 优先从content_list中递归提取
if content_list:
table_parts = self._extract_tables_from_content_list(content_list)

# 如果content_list中有表格，直接返回
if table_parts:
return '\n'.join(table_parts)

# 只有当content_list中没有表格时，才从markdown文本中提取
if text:
lines = text.split('\n')
table_lines = []
in_table = False

for line in lines:
if '|' in line:
table_lines.append(line)
in_table = True
elif in_table and line.strip() == '':
# 表格结束
if table_lines:
table_parts.append('\n'.join(table_lines))
table_lines = []
in_table = False
elif in_table:
# 表格内的非表格行，表格结束
if table_lines:
table_parts.append('\n'.join(table_lines))
table_lines = []
in_table = False

# 处理文档末尾的表格
if table_lines:
table_parts.append('\n'.join(table_lines))

return '\n'.join(table_parts)
# 使用统一的内容分割方法
content_parts = self.split_content(text, content_list)
return content_parts.get('table', '')

def _extract_tables_from_content_list(self, content_list: List[Dict[str, Any]]) -> List[str]:
"""递归从content_list中提取表格内容"""
Expand Down Expand Up @@ -138,6 +103,6 @@ def _calculate_score(self, predicted: str, groundtruth: str,

def _extract_table_content(self, text: str, content_list: List[Dict[str, Any]] = None) -> str:
"""从文本和content_list中提取表格内容"""
# 复用TableEditMetric的表格提取逻辑
table_edit_metric = TableEditMetric("temp")
return table_edit_metric._extract_table_content(text, content_list)
# 使用统一的内容分割方法
content_parts = self.split_content(text, content_list)
return content_parts.get('table', '')
Loading