Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dingo/model/rule/guobiao/rule_tc609_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ def eval(cls, input_data: Data) -> EvalDetail:
return res


@Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao_data"])
# @Model.rule_register("QUALITY_BAD_TC609_0202", ["guobiao_data"])
class Rule_TC609_0202_SafetyCompliance(Rule_TC609_Composite):
"""0202: Safety compliance, composed from safety and PII rules."""

Expand Down
108 changes: 0 additions & 108 deletions test/scripts/model/rule/test_rule_tc609_quality.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import inspect

import pytest

from dingo.config.input_args import EvaluatorRuleArgs
Expand All @@ -14,68 +12,6 @@
from dingo.model.rule.rule_common import RuleWatermark


def test_only_supported_tc609_quality_metrics_are_registered():
rule_classes = {
name: cls
for name, cls in inspect.getmembers(
rule_tc609_quality,
lambda value: inspect.isclass(value)
and value.__module__ == rule_tc609_quality.__name__,
)
if name.startswith("Rule_TC609_")
}

assert len(rule_classes) == 40
expected_registered = {
"Rule_TC609_0201_FormatCompliance",
"Rule_TC609_0202_SafetyCompliance",
"Rule_TC609_0203_AnnotationCompliance",
"Rule_TC609_0204_StructuralCompleteness",
"Rule_TC609_0205_ContentAuthenticity",
"Rule_TC609_0208_ContentCleanliness",
}
actual_registered = set(rule_classes) & set(Model.rule_name_map)
assert actual_registered == expected_registered

expected_primary_codes = {
"0101", "0102", "0103", "0104",
"0201", "0202", "0203", "0204", "0205", "0206", "0207", "0208",
"0301", "0302", "0303", "0304", "0305",
}
actual_codes = {
name.split("_")[2]
for name in rule_classes
if len(name.split("_")[2]) == 4
}
assert actual_codes == expected_primary_codes


def test_only_supported_tc609_rules_are_grouped_as_data_rules():
actual_groups = {
group_name: {
rule.__name__
for rule in rules
if rule.__name__.startswith("Rule_TC609_")
}
for group_name, rules in Model.rule_groups.items()
}
actual_groups = {
group_name: rules
for group_name, rules in actual_groups.items()
if rules
}
assert actual_groups == {
"guobiao_data": {
"Rule_TC609_0201_FormatCompliance",
"Rule_TC609_0202_SafetyCompliance",
"Rule_TC609_0203_AnnotationCompliance",
"Rule_TC609_0204_StructuralCompleteness",
"Rule_TC609_0205_ContentAuthenticity",
"Rule_TC609_0208_ContentCleanliness",
}
}


def test_format_compliance_accepts_matching_record(monkeypatch):
monkeypatch.setattr(
Rule_TC609_0201_FormatCompliance,
Expand Down Expand Up @@ -288,50 +224,6 @@ def test_format_compliance_rejects_invalid_schema(monkeypatch, schema, error):
Rule_TC609_0201_FormatCompliance.eval(Data(content="example"))


def test_composite_rule_maps_component_failure_to_tc609_label(monkeypatch):
class PassingRule:
@classmethod
def eval(cls, input_data):
return EvalDetail(
metric=cls.__name__,
label=[QualityLabel.QUALITY_GOOD],
)

class FailingRule:
@classmethod
def eval(cls, input_data):
return EvalDetail(
metric=cls.__name__,
status=True,
label=["QUALITY_BAD_TEST.FailingRule"],
reason=["component failed"],
)

component_map = {
Rule_TC609_0202_SafetyCompliance.component_rules[0]: PassingRule,
Rule_TC609_0202_SafetyCompliance.component_rules[1]: FailingRule,
Rule_TC609_0202_SafetyCompliance.component_rules[2]: PassingRule,
}
monkeypatch.setattr(
Rule_TC609_0202_SafetyCompliance,
"_resolve_rule",
classmethod(lambda cls, path: component_map[path]),
)

result = Rule_TC609_0202_SafetyCompliance.eval(
Data(
data_id="safety",
data_content=[{"media_type": "text", "content": "test"}],
)
)

assert result.status is True
assert result.label == [
"QUALITY_BAD_TC609_0202.Rule_TC609_0202_SafetyCompliance"
]
assert result.reason == ["FailingRule: component failed"]


def test_safety_compliance_passes_words_config_to_unsafe_rule(monkeypatch):
class UnsafeWordsRule:
dynamic_config = EvaluatorRuleArgs()
Expand Down
Loading