def quicksort(arr):
if len(arr) <= 1:
return arr
-
+
pivot = arr[len(arr) // 2]
left = [x for x in arr if x < pivot]
middle = [x for x in arr if x == pivot]
right = [x for x in arr if x > pivot]
-
+
return quicksort(left) + middle + quicksort(right)| Location | % Asian | % Black | % Hispanic | % White |
|---|---|---|---|---|
| Alabama | 2 | 25 | 6 | 63 |
| Alaska | 7 | 3 | 6 | 56 |
| Arizona | 4 | 3 | 19 | 51 |
| Arkansas | 2 | 14 | 7 | 76 |
| California | 17 | 4 | 31 | 36 |
| Colorado | 3 | 4 | 13 | 75 |
| Connecticut | 5 | 8 | 15 | 65 |
| Delaware | 4 | 17 | 10 | 61 |
| District of Columbia | 6 | 45 | 14 | 49 |
| Florida | 9 | 33 | 53 | |
| Georgia | 6 | 27 | 9 | 53 |
| Hawaii | 54 | 1 | 26 | |
| Idaho | 2 | 1 | 11 | 82 |
| Illinois | 7 | 11 | 15 | 63 |
| Indiana | 3 | 7 | 7 | 80 |
| Iowa | 2 | 2 | 5 | 93 |
| Kansas | 3 | 4 | 12 | 75 |
| Kentucky | 2 | 7 | 81 | |
| Louisiana | 3 | 31 | 7 | 58 |
| Maine | 2 | 2 | 2 | 82 |
| Maryland | 7 | 27 | 10 | 54 |
| Massachusetts | 8 | 7 | 10 | 73 |
| Michigan | 4 | 10 | 5 | 73 |
| Minnesota | 6 | 5 | 5 | 82 |
| Mississippi | 2 | 38 | 3 | 56 |
| Missouri | 3 | 11 | 5 | 86 |
| Nevada | 10 | 6 | 27 | 39 |
| New Hampshire | 3 | 1 | 3 | 88 |
| New Jersey | 11 | 9 | 18 | 51 |
| New Mexico | 3 | 2 | 40 | 42 |
| New York | 14 | 15 | 21 | 69 |
| North Carolina | 4 | 20 | 10 | 68 |
| Ohio | 3 | 10 | 4 | 78 |
| Oklahoma | 4 | 7 | 12 | 77 |
| Oregon | 6 | 3 | 10 | 74 |
| Pennsylvania* | 2 | 6 | 7 | 79 |
| Rhode Island | 4 | 5 | 16 | 75 |
| South Carolina | 22 | 6 | 59 | |
| South Dakota | <0.01 | 1 | <0.01 | 92 |
| Tennessee | 2 | 12 | 6 | 63 |
| Texas | 6 | 8 | 36 | 36 |
| Utah | 3 | 1 | 12 | 78 |
| Vermont | 2 | 1 | 2 | 95 |
| Virginia | 9 | 17 | 10 | 58 |
| Washington | 11 | 4 | 11 | 62 |
| West Virginia | 3 | 90 | ||
| Wisconsin | 4 | 5 | 6 | 90 |
Kaiser Family Foundation data
这是一个Python基础教程，展示如何定义函数。
+def greet(name):
+ ""问候函数""
+ return f"Hello, {name}!"
+
+# 使用示例
+result = greet("World")
+print(result)这个函数可以用来问候任何人。
+ + + """ + result = extractor.extract(html_content) + self.assertEqual(isinstance(result, ExtractionResult), True) + self.assertEqual(result.success in [True, False], True) + +# def test_magic_html_extractor(self): +# # 测试 Magic HTML 抽取器 +# try: +# extractor = ExtractorFactory.create("magic-html") +# html_content = """ +# +# +#这是一个Python基础教程，展示如何定义函数。
+#def greet(name):
+# ""问候函数""
+# return f"Hello, {name}!"
+#
+# # 使用示例
+# result = greet("World")
+# print(result)这个函数可以用来问候任何人。
+# +# +# """ +# result = extractor.extract(html_content) +# self.assertEqual(isinstance(result, ExtractionResult), True) +# self.assertEqual(result.success in [True, False], True) +# except ValueError as e: +# # 如果抽取器未注册，跳过测试 +# self.skipTest(f"Magic HTML 抽取器未注册: {e}") + + def test_resiliparse_extractor(self): + # 测试 Resiliparse 抽取器 + try: + extractor = ExtractorFactory.create("resiliparse") + html_content = """ + + +这是一个Python基础教程，展示如何定义函数。
+def greet(name):
+ ""问候函数""
+ return f"Hello, {name}!"
+
+# 使用示例
+result = greet("World")
+print(result)这个函数可以用来问候任何人。
+ + + """ + result = extractor.extract(html_content) + self.assertEqual(isinstance(result, ExtractionResult), True) + self.assertEqual(result.success in [True, False], True) + except ValueError as e: + # 如果抽取器未注册，跳过测试 + self.skipTest(f"Resiliparse 抽取器未注册: {e}") + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 0e308b5..5c38b24 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -7,14 +7,14 @@ class TestContentMetrics(unittest.TestCase): """测试内容类型指标""" - + def setUp(self): """测试前准备""" self.calculator = MetricCalculator() - + # 测试数据 self.predicted_content = """# 标题 - + 这是一段文字内容。 ```python @@ -57,12 +57,10 @@ def hello(): 最后是正确的文字内容。 """ - - def test_available_metrics(self): """测试可用指标列表""" metrics = self.calculator.list_available_metrics() - + # 验证必要的指标都存在 expected_metrics = ['code_edit', 'formula_edit', 'table_edit', 'table_TEDS', 'text_edit'] for metric in expected_metrics: @@ -74,12 +72,13 @@ def test_metric_calculation_success(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + # 验证所有指标都计算成功 expected_metrics = ['code_edit', 'formula_edit', 'table_edit', 'table_TEDS', 'text_edit', 'overall'] for metric_name in expected_metrics: self.assertIn(metric_name, results, f"缺少指标结果: {metric_name}") - self.assertTrue(results[metric_name].success, f"指标 {metric_name} 计算失败: {results[metric_name].error_message}") + self.assertTrue(results[metric_name].success, + f"指标 {metric_name} 计算失败: {results[metric_name].error_message}") def test_code_edit_metric(self): """测试代码编辑距离指标""" @@ -87,14 +86,14 @@ def test_code_edit_metric(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + code_result = results['code_edit'] self.assertTrue(code_result.success) self.assertIsInstance(code_result.score, float) # 验证固定内容的确定分数 self.assertAlmostEqual(code_result.score, 0.918367, places=5, - msg=f"code_edit分数应该是0.918367，实际: {code_result.score}") - + msg=f"code_edit分数应该是0.918367，实际: {code_result.score}") + # 验证详细信息 self.assertEqual(code_result.details['content_type'], 'code') self.assertIn('distance', code_result.details) @@ -107,14 +106,14 @@ def test_formula_edit_metric(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + formula_result = results['formula_edit'] self.assertTrue(formula_result.success) self.assertIsInstance(formula_result.score, float) # 验证固定内容的确定分数 self.assertAlmostEqual(formula_result.score, 1.000000, places=5, - msg=f"formula_edit分数应该是1.000000，实际: {formula_result.score}") - + msg=f"formula_edit分数应该是1.000000，实际: {formula_result.score}") + # 验证详细信息 self.assertEqual(formula_result.details['content_type'], 'formula') self.assertIn('distance', formula_result.details) @@ -125,14 +124,14 @@ def test_table_edit_metric(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + table_result = results['table_edit'] self.assertTrue(table_result.success) self.assertIsInstance(table_result.score, float) # 验证固定内容的确定分数 self.assertAlmostEqual(table_result.score, 0.868852, places=5, - msg=f"table_edit分数应该是0.868852，实际: {table_result.score}") - + msg=f"table_edit分数应该是0.868852，实际: {table_result.score}") + # 验证详细信息 self.assertEqual(table_result.details['content_type'], 'table') self.assertIn('distance', table_result.details) @@ -143,14 +142,14 @@ def test_table_teds_metric(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + teds_result = results['table_TEDS'] self.assertTrue(teds_result.success) self.assertIsInstance(teds_result.score, float) # 验证固定内容的确定分数 self.assertAlmostEqual(teds_result.score, 0.300000, places=5, - msg=f"table_TEDS分数应该是0.300000，实际: {teds_result.score}") - + msg=f"table_TEDS分数应该是0.300000，实际: {teds_result.score}") + # 验证详细信息 self.assertEqual(teds_result.details['content_type'], 'table') @@ -160,14 +159,14 @@ def test_text_edit_metric(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + text_result = results['text_edit'] self.assertTrue(text_result.success) self.assertIsInstance(text_result.score, float) # 验证固定内容的确定分数 self.assertAlmostEqual(text_result.score, 0.769231, places=5, - msg=f"text_edit分数应该是0.769231，实际: {text_result.score}") - + msg=f"text_edit分数应该是0.769231，实际: {text_result.score}") + # 验证详细信息 self.assertEqual(text_result.details['content_type'], 'text') self.assertIn('distance', text_result.details) @@ -178,25 +177,25 @@ def test_overall_metric_calculation(self): predicted_content=self.predicted_content, groundtruth_content=self.groundtruth_content ) - + # 获取individual指标分数 individual_metrics = ['code_edit', 'formula_edit', 'table_edit', 'table_TEDS', 'text_edit'] individual_scores = [] - + for metric_name in individual_metrics: self.assertIn(metric_name, results) self.assertTrue(results[metric_name].success) individual_scores.append(results[metric_name].score) - + # 计算期望的overall分数 expected_overall = sum(individual_scores) / len(individual_scores) - + # 验证overall分数 overall_result = results['overall'] self.assertTrue(overall_result.success) - self.assertAlmostEqual(overall_result.score, expected_overall, places=5, - msg="overall分数应该是其他指标的平均值") - + self.assertAlmostEqual(overall_result.score, expected_overall, places=5, + msg="overall分数应该是其他指标的平均值") + # 验证overall详细信息 self.assertEqual(overall_result.details['source'], 'average_of_all_metrics') self.assertEqual(overall_result.details['successful_metrics'], len(individual_metrics)) @@ -208,12 +207,12 @@ def test_identical_content(self): predicted_content=self.groundtruth_content, groundtruth_content=self.groundtruth_content ) - + # 完全相同的内容应该得到满分 for metric_name in ['code_edit', 'formula_edit', 'table_edit', 'text_edit']: if metric_name in results and results[metric_name].success: self.assertAlmostEqual(results[metric_name].score, 1.0, places=5, - msg=f"相同内容的{metric_name}应该得到满分，实际: {results[metric_name].score}") + msg=f"相同内容的{metric_name}应该得到满分，实际: {results[metric_name].score}") def test_empty_content(self): """测试空内容的情况""" @@ -221,17 +220,17 @@ def test_empty_content(self): predicted_content="", groundtruth_content="" ) - + # 空内容应该能正确处理，不应该出错 for metric_name, result in results.items(): if metric_name != 'overall': # overall可能会有特殊处理 - self.assertTrue(result.success or result.score == 0.0, - f"空内容的{metric_name}应该正确处理") + self.assertTrue(result.success or result.score == 0.0, + f"空内容的{metric_name}应该正确处理") class TestErrorHandling(unittest.TestCase): """测试错误处理""" - + def setUp(self): self.calculator = MetricCalculator() @@ -242,7 +241,7 @@ def test_malformed_content(self): predicted_content="test", groundtruth_content="test" ) - + # 不应该有未捕获的异常 self.assertIsInstance(results, dict) @@ -252,18 +251,18 @@ def test_none_inputs(self): predicted_content=None, groundtruth_content=None ) - + # 应该能处理None输入 self.assertIsInstance(results, dict) class TestRealSampleMetrics(unittest.TestCase): """测试基于LLM-WebKit实际提取结果的指标计算""" - + def setUp(self): """测试前准备""" self.calculator = MetricCalculator() - + def test_text_code_sample_edit_distance(self): """测试文本+代码样本的编辑距离""" # 基于实际调试结果的数据 @@ -278,7 +277,7 @@ def hello_world(): ``` 以上代码展示了一个简单的Python函数。""" - + predicted = """# Python编程示例 这是一段关于Python编程的介绍文本。 @@ -290,25 +289,25 @@ def hello_world(): ``` 以上代码展示了一个简单的Python函数。""" - + # 计算编辑距离（基于实际调试结果） results = self.calculator.calculate_all( predicted_content=predicted, groundtruth_content=groundtruth ) - + # 验证文本编辑距离（固定内容应该有确定分数） self.assertIn("text_edit", results) self.assertTrue(results["text_edit"].success) self.assertAlmostEqual(results["text_edit"].score, 1.000000, places=5, - msg=f"text_edit分数应该是1.000000，实际: {results['text_edit'].score}") - + msg=f"text_edit分数应该是1.000000，实际: {results['text_edit'].score}") + # 验证代码编辑距离（缺少python标识符导致轻微差异） self.assertIn("code_edit", results) self.assertTrue(results["code_edit"].success) self.assertAlmostEqual(results["code_edit"].score, 0.905797, places=5, - msg=f"code_edit分数应该是0.905797，实际: {results['code_edit'].score}") - + msg=f"code_edit分数应该是0.905797，实际: {results['code_edit'].score}") + def test_table_sample_edit_distance(self): """测试表格样本的编辑距离""" groundtruth = """## 销售数据统计 @@ -317,31 +316,31 @@ def test_table_sample_edit_distance(self): |------|------|------| | 产品A | 100 | 1000 | | 产品B | 200 | 3000 |""" - + predicted = """## 销售数据统计 | 产品 | 销量 | 收入 | |---|---|---| | 产品A | 100 | 1000 | | 产品B | 200 | 3000 |""" - + results = self.calculator.calculate_all( predicted_content=predicted, groundtruth_content=groundtruth ) - + # 验证表格编辑距离（分隔符长度差异导致的固定分数） self.assertIn("table_edit", results) self.assertTrue(results["table_edit"].success) self.assertAlmostEqual(results["table_edit"].score, 0.888889, places=5, - msg=f"table_edit分数应该是0.888889，实际: {results['table_edit'].score}") - + msg=f"table_edit分数应该是0.888889，实际: {results['table_edit'].score}") + # 验证TEDS指标（表格结构完全相同，满分） self.assertIn("table_TEDS", results) self.assertTrue(results["table_TEDS"].success) self.assertAlmostEqual(results["table_TEDS"].score, 1.000000, places=5, - msg=f"table_TEDS分数应该是1.000000，实际: {results['table_TEDS'].score}") - + msg=f"table_TEDS分数应该是1.000000，实际: {results['table_TEDS'].score}") + def test_formula_sample_edit_distance(self): """测试公式样本的编辑距离""" groundtruth = """## 数学公式示例 @@ -351,7 +350,7 @@ def test_formula_sample_edit_distance(self): 这是一个行间公式: $$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}$$""" - + predicted = """## 数学公式示例 这是一个行内公式: \\$E = mc^2\\$ @@ -359,24 +358,24 @@ def test_formula_sample_edit_distance(self): 这是一个行间公式: \\$\\$\\int_{-\\infty}^{\\infty} e^{-x^2} dx = \\sqrt{\\pi}\\$""" - + results = self.calculator.calculate_all( predicted_content=predicted, groundtruth_content=groundtruth ) - + # 验证公式编辑距离（符号转义导致的固定低分） self.assertIn("formula_edit", results) self.assertTrue(results["formula_edit"].success) self.assertAlmostEqual(results["formula_edit"].score, 0.000000, places=5, - msg=f"formula_edit分数应该是0.000000，实际: {results['formula_edit'].score}") - + msg=f"formula_edit分数应该是0.000000，实际: {results['formula_edit'].score}") + # 验证文本编辑距离（去除公式后的纯文本，也受符号转义影响） self.assertIn("text_edit", results) self.assertTrue(results["text_edit"].success) self.assertAlmostEqual(results["text_edit"].score, 0.320000, places=5, - msg=f"text_edit分数应该是0.320000，实际: {results['text_edit'].score}") - + msg=f"text_edit分数应该是0.320000，实际: {results['text_edit'].score}") + def test_overall_score_calculation(self): """测试综合分数计算""" # 使用第一个样本测试综合分数 @@ -391,7 +390,7 @@ def hello_world(): ``` 以上代码展示了一个简单的Python函数。""" - + predicted = """# Python编程示例 这是一段关于Python编程的介绍文本。 @@ -403,29 +402,29 @@ def hello_world(): ``` 以上代码展示了一个简单的Python函数。""" - + results = self.calculator.calculate_all( predicted_content=predicted, groundtruth_content=groundtruth ) - + # 验证overall分数存在且合理 self.assertIn("overall", results) self.assertTrue(results["overall"].success) - + # overall应该是所有成功指标的平均值 successful_scores = [] for metric_name, result in results.items(): if metric_name != "overall" and result.success: successful_scores.append(result.score) - + if successful_scores: expected_overall = sum(successful_scores) / len(successful_scores) actual_overall = results["overall"].score - + # 允许小幅计算误差 self.assertAlmostEqual(actual_overall, expected_overall, places=3) - + def test_all_metrics_coverage(self): """测试所有6项指标都被计算""" groundtruth = """# 综合示例 @@ -444,50 +443,50 @@ def test(): | 1 | 2 | 更多文本。""" - + predicted = groundtruth # 使用相同内容测试 - + results = self.calculator.calculate_all( predicted_content=predicted, groundtruth_content=groundtruth ) - + # 验证所有6项指标都存在 expected_metrics = ["overall", "text_edit", "code_edit", "table_edit", "table_TEDS", "formula_edit"] - + print(f"\n=== 完全相同内容的指标测试结果 ===") - + for metric in expected_metrics: self.assertIn(metric, results, f"指标 {metric} 缺失") self.assertTrue(results[metric].success, f"指标 {metric} 计算失败") - + score = results[metric].score print(f"{metric}: {score:.6f}") - + # 完全相同的内容应该得到满分 1.0 - self.assertAlmostEqual(score, 1.0, - places=4, - msg=f"完全相同内容的 {metric} 应该得到满分，实际得分: {score}") - + self.assertAlmostEqual(score, 1.0, + places=4, + msg=f"完全相同内容的 {metric} 应该得到满分，实际得分: {score}") + print("✅ 所有指标都正确得到满分!") def run_visual_test(): """运行可视化测试（保留原有的打印功能）""" print("=== 新指标功能测试 ===\n") - + calculator = MetricCalculator() - + # 显示可用指标 print("可用的指标:") metrics = calculator.list_available_metrics() for metric in metrics: print(f" - {metric}") print() - + # 测试数据 predicted_content = """# 标题 - + 这是一段文字内容。 ```python @@ -536,11 +535,11 @@ def hello(): predicted_content=predicted_content, groundtruth_content=groundtruth_content ) - + # 显示结果 print("\n=== 评测结果 ===") print("-" * 60) - + for metric_name, result in results.items(): if result.success: print(f"{metric_name:15}: {result.score:.4f}") @@ -551,7 +550,7 @@ def hello(): else: print(f"{metric_name:15}: ERROR - {result.error_message}") print() - + # 显示详细信息 print("\n=== 详细信息 ===") for metric_name in ["code_edit", "formula_edit", "table_edit", "text_edit"]: @@ -565,7 +564,7 @@ def hello(): if __name__ == "__main__": import sys - + if len(sys.argv) > 1 and sys.argv[1] == "--visual": # 运行可视化测试 try: @@ -574,7 +573,8 @@ def hello(): except Exception as e: print(f"\n❌ 测试失败: {e}") import traceback + traceback.print_exc() else: # 运行单元测试 - unittest.main(verbosity=2) \ No newline at end of file + unittest.main(verbosity=2) \ No newline at end of file diff --git a/webmainbench/extractors/__init__.py b/webmainbench/extractors/__init__.py index e94c225..d71ccc9 100644 --- a/webmainbench/extractors/__init__.py +++ b/webmainbench/extractors/__init__.py @@ -8,6 +8,9 @@ from .factory import ExtractorFactory from .llm_webkit_extractor import LlmWebkitExtractor from .jina_extractor import JinaExtractor +from .trafilatura_extractor import TrafilaturaExtractor +from .resiliparse_extractor import ResiliparseExtractor +from .magic_html_extractor import MagicHtmlExtractor @@ -17,4 +20,7 @@ "ExtractorFactory", "LlmWebkitExtractor", "JinaExtractor", + "TrafilaturaExtractor", + "ResiliparseExtractor", + "MagicHtmlExtractor", ] \ No newline at end of file diff --git a/webmainbench/extractors/magic_html_extractor.py b/webmainbench/extractors/magic_html_extractor.py new file mode 100644 index 0000000..7578de8 --- /dev/null +++ b/webmainbench/extractors/magic_html_extractor.py @@ -0,0 +1,80 @@ +# webmainbench/extractors/magic_html_extractor.py +""" +Magic HTML extractor implementation. +""" + +from typing import Dict, Any, Optional, List +from .base import BaseExtractor, ExtractionResult +from .factory import extractor +from magic_html import GeneralExtractor +import re +import html2text + + +@extractor("magic-html") +class MagicHtmlExtractor(BaseExtractor): + """Extractor using Magic HTML.""" + + version = "0.1.5" + description = "Magic HTML based content extractor" + + def _setup(self) -> None: + """Set up the Magic HTML extractor.""" + try: + self.extractor = GeneralExtractor() + except Exception as e: + raise RuntimeError(f"Failed to initialize Magic HTML extractor: {e}") + + def _extract_content(self, html: str, url: str = None) -> ExtractionResult: + try: + # Use Magic HTML for extraction + data = self.extractor.extract(html) + + # 从输出中提取所需信息 + extracted_html = data.get('html', '') + markdown = html2text.html2text(extracted_html) + title = data.get('title', '') + # 简单地将提取的 HTML 作为内容 + content = markdown + # 创建 content_list（简单分割段落） + content_list = [] + if content: + paragraphs = content.split('\n\n') + for i, para in enumerate(paragraphs): + if para.strip(): + content_list.append({ + "type": "paragraph", + "content": para.strip(), + "index": i + }) + + return ExtractionResult( + content=content, + # content_list=content_list, + title=title, + language=self._detect_language(content), + success=True + ) + + except Exception as e: + return ExtractionResult.create_error_result( + f"Magic HTML extraction failed: {str(e)}" + ) + + + def _detect_language(self, content: str) -> Optional[str]: + """检测内容语言.""" + if not content: + return None + + # 简单的语言检测逻辑 + chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', content)) + english_chars = len(re.findall(r'[a-zA-Z]', content)) + + if chinese_chars > english_chars: + return "zh" + elif english_chars > 0: + return "en" + else: + return None + diff --git a/webmainbench/extractors/resiliparse_extractor.py b/webmainbench/extractors/resiliparse_extractor.py new file mode 100644 index 0000000..e47c87d --- /dev/null +++ b/webmainbench/extractors/resiliparse_extractor.py @@ -0,0 +1,123 @@ +""" +resiliparse extractor implementation. +""" +from typing import Dict, Any, Optional, List +from dataclasses import dataclass +from .base import BaseExtractor, ExtractionResult +from .factory import extractor +from resiliparse.extract.html2text import extract_plain_text +import re + +@dataclass +class ResiliparseInferenceConfig: + """Configuration for Resiliparse extractor.""" + main_content: bool = True + alt_texts: bool = True + links: bool = False + form_fields: bool = False + noscript: bool = False + list_bullets: bool = True + preserve_formatting: bool = True + comments: bool = True + # 可根据需要添加更多resiliparse支持的参数 + + +@extractor("resiliparse") +class ResiliparseExtractor(BaseExtractor): + """Extractor using Resiliparse.""" + + version = "0.14.5" + description = "Resiliparse based content extractor" + + def __init__(self, name: str, config: Optional[Dict[str, Any]] = None): + super().__init__(name, config) + self.inference_config = ResiliparseInferenceConfig() + + # 应用用户配置 + if config: + for key, value in config.items(): + if hasattr(self.inference_config, key): + setattr(self.inference_config, key, value) + + def _setup(self) -> None: + """Set up the Resiliparse extractor.""" + # 初始化操作 + pass + + def _extract_content(self, html: str, url: str = None) -> ExtractionResult: + """ + Extract content using Resiliparse. + + Args: + html: HTML content to extract from + url: Optional URL of the page + + Returns: + ExtractionResult instance + """ + try: + # 使用配置参数进行内容抽取 + content = extract_plain_text( + html, + main_content=self.inference_config.main_content, + alt_texts=self.inference_config.alt_texts, + links=self.inference_config.links, + form_fields=self.inference_config.form_fields, + noscript=self.inference_config.noscript, + list_bullets=self.inference_config.list_bullets, + preserve_formatting=self.inference_config.preserve_formatting, + comments=self.inference_config.comments + ) + + # 创建 content_list（简单分割段落） + content_list = [] + if content: + paragraphs = content.split('\n\n') + for i, para in enumerate(paragraphs): + if para.strip(): + content_list.append({ + "type": "paragraph", + "content": para.strip(), + "index": i + }) + + return ExtractionResult( + content=content, + # content_list=content_list, + title=self._extract_title(html), + language=self._detect_language(content), + success=True + ) + + except Exception as e: + return ExtractionResult.create_error_result( + f"Resiliparse extraction failed: {str(e)}" + ) + + def _extract_title(self, html: str) -> Optional[str]: + """提取页面标题.""" + try: + import re + title_match = re.search(r'这是一个Python基础教程，展示如何定义函数。
-#def greet(name):
-# ""问候函数""
-# return f"Hello, {name}!"
-#
-# # 使用示例
-# result = greet("World")
-# print(result)这个函数可以用来问候任何人。
-# -# -# """ -# result = extractor.extract(html_content) -# self.assertEqual(isinstance(result, ExtractionResult), True) -# self.assertEqual(result.success in [True, False], True) -# except ValueError as e: -# # 如果抽取器未注册，跳过测试 -# self.skipTest(f"Magic HTML 抽取器未注册: {e}") + def test_magic_html_extractor(self): + # 测试 Magic HTML 抽取器 + try: + extractor = ExtractorFactory.create("magic-html") + html_content = """ + + +这是一个Python基础教程，展示如何定义函数。
+def greet(name):
+ ""问候函数""
+ return f"Hello, {name}!"
+
+# 使用示例
+result = greet("World")
+print(result)这个函数可以用来问候任何人。
+ + + """ + result = extractor.extract(html_content) + self.assertEqual(isinstance(result, ExtractionResult), True) + self.assertEqual(result.success in [True, False], True) + except ValueError as e: + # 如果抽取器未注册，跳过测试 + self.skipTest(f"Magic HTML 抽取器未注册: {e}") def test_resiliparse_extractor(self): # 测试 Resiliparse 抽取器