diff --git a/spiderfoot/correlation.py b/spiderfoot/correlation.py index 8b812fb150..e7e5950570 100644 --- a/spiderfoot/correlation.py +++ b/spiderfoot/correlation.py @@ -555,7 +555,11 @@ def event_strip(event: dict, field: str, value: str) -> None: """ topfield, subfield = field.split(".") if field.startswith(topfield + "."): - for s in event[topfield]: + # Iterate over a snapshot copy: removing from the list being + # iterated advances the iterator past the next element, so a + # non-matching sub-event right after a removed one is skipped + # and wrongly survives in the bucket. + for s in event[topfield][:]: if s[subfield] != value: event[topfield].remove(s) diff --git a/test/unit/spiderfoot/test_spiderfootcorrelator.py b/test/unit/spiderfoot/test_spiderfootcorrelator.py index 3521c4d833..039a887f8a 100644 --- a/test/unit/spiderfoot/test_spiderfootcorrelator.py +++ b/test/unit/spiderfoot/test_spiderfootcorrelator.py @@ -37,6 +37,32 @@ def test_run_correlations_invalid_scan_instance_should_raise_ValueError(self): with self.assertRaises(ValueError): correlator.run_correlations() + def test_aggregate_events_does_not_leak_non_matching_subevents(self): + sfdb = SpiderFootDb(self.default_options, False) + correlator = SpiderFootCorrelator(sfdb, {}) + + rule = {"id": "r", "field": "child.data"} + events = [ + { + "id": "E1", + "child": [ + {"data": "a"}, + {"data": "b"}, + {"data": "c"}, + {"data": "d"}, + ], + } + ] + buckets = correlator.aggregate_events(rule, events) + + # Each bucket must keep only the sub-events whose subfield matches the + # bucket value. Before the fix, mutating the list while iterating + # skipped the element after each removed one, leaking non-matching + # children (e.g. bucket "a" wrongly kept ["a", "c"]). + for value in ("a", "b", "c", "d"): + kept = [child["data"] for child in buckets[value][0]["child"]] + self.assertEqual(kept, [value]) + def test_build_db_criteria_argument_matchrule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {})