Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions tests/test_frequency.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import tempfile
import unittest
from pathlib import Path

from trendradar.core.frequency import load_frequency_words, matches_word_groups


class GlobalFilterTests(unittest.TestCase):
def test_global_filter_supports_regular_expressions(self):
config = """[GLOBAL_FILTER]
/spam|clickbait/

[WORD_GROUPS]
news
"""

with tempfile.TemporaryDirectory() as temp_dir:
config_path = Path(temp_dir) / "frequency_words.txt"
config_path.write_text(config, encoding="utf-8")
word_groups, filter_words, global_filters = load_frequency_words(
str(config_path)
)

self.assertFalse(
matches_word_groups(
"Breaking news: spam", word_groups, filter_words, global_filters
)
)
self.assertFalse(
matches_word_groups(
"Clickbait news", word_groups, filter_words, global_filters
)
)
self.assertTrue(
matches_word_groups(
"Breaking news", word_groups, filter_words, global_filters
)
)


if __name__ == "__main__":
unittest.main()
12 changes: 7 additions & 5 deletions trendradar/core/frequency.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def _word_matches(word_config: Union[str, Dict], title_lower: str) -> bool:

def load_frequency_words(
frequency_file: Optional[str] = None,
) -> Tuple[List[Dict], List[str], List[str]]:
) -> Tuple[List[Dict], List[Dict], List[Dict]]:
"""
加载频率词配置

Expand Down Expand Up @@ -161,13 +161,13 @@ def load_frequency_words(

# 处理全局过滤区域
if current_section == "GLOBAL_FILTER":
# 直接添加所有非空行到全局过滤列表
# 解析所有非空行,支持普通词和正则语法
for line in lines:
# 忽略特殊语法前缀,只提取纯文本
if line.startswith(("!", "+", "@")):
continue # 全局过滤区不支持特殊语法
if line:
global_filters.append(line)
global_filters.append(_parse_word(line))
continue

# 处理词组区域
Expand Down Expand Up @@ -247,7 +247,7 @@ def matches_word_groups(
title: str,
word_groups: List[Dict],
filter_words: List,
global_filters: Optional[List[str]] = None
global_filters: Optional[List[Union[str, Dict]]] = None,
) -> bool:
"""
检查标题是否匹配词组规则
Expand All @@ -271,7 +271,9 @@ def matches_word_groups(

# 全局过滤检查(优先级最高)
if global_filters:
if any(global_word.lower() in title_lower for global_word in global_filters):
if any(
_word_matches(global_item, title_lower) for global_item in global_filters
):
return False

# 如果没有配置词组,则匹配所有标题(支持显示全部新闻)
Expand Down