From 71dbc4bf0ecc10be6a15311c336f167f28e8278c Mon Sep 17 00:00:00 2001 From: cka-y Date: Wed, 26 Aug 2026 09:35:58 -0400 Subject: [PATCH 1/5] feat: evaluate fresh + stable criteria --- api/src/shared/common/seal_criteria.py | 87 ++++ .../db_models/reliability_criterion_impl.py | 25 +- .../test_feed_reliability_report_impl.py | 15 +- .../models/test_reliability_criterion_impl.py | 38 +- .../models/test_seal_enum_contract.py | 36 +- .../src/tasks/seal_of_reliability/context.py | 95 +++- .../src/tasks/seal_of_reliability/criteria.py | 103 ---- .../evaluators/__init__.py | 15 +- .../seal_of_reliability/evaluators/base.py | 38 +- .../evaluators/fresh_coverage.py | 84 +++ .../evaluators/official.py | 10 +- .../seal_of_reliability/evaluators/stable.py | 81 +++ .../tasks/seal_of_reliability/seal_updater.py | 10 +- .../seal_of_reliability/state_machine.py | 2 +- .../test_seal_end_to_end_db.py | 79 ++- .../test_seal_enum_contract.py | 27 +- .../test_seal_evaluators.py | 201 +++++++- .../test_seal_state_machine.py | 2 +- .../test_seal_updater_db.py | 480 +++++++++++++++++- 19 files changed, 1170 insertions(+), 258 deletions(-) delete mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index 2efbbfd17..c52695891 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -9,6 +9,13 @@ share one definition. The job owns the writing of `seal_criterion`; the API only reads it back and derives the two countdowns from these windows. +Every policy value the seal depends on belongs here and nowhere else: the criterion names, +the two status/phase vocabularies, each criterion's grace and probation windows, and the +windows that are part of a criterion's own check (Stable's tracking period, Fresh's coverage +horizon). Nothing downstream may restate one - a job evaluator asks +`grace_period_for(name)` rather than declaring its own, so changing a window here changes it +for the job, the API and the materialized view roll-up at once. + See #1761 for the algorithm and #1760 for the tables. """ @@ -30,6 +37,54 @@ class SealCriterionName(str, Enum): FRESH_CONTINUOUS = "fresh_continuous" +class CriterionStatus(str, Enum): + """A criterion's status. Values match the `seal_criterion_status` DB enum. + + Only PASS and FAIL are verdicts. The other three say why there is no verdict, and they + are not interchangeable - the job's roll-up sends them in three different directions: + + * UNKNOWN - we could not look; the inputs the check needs were not there. A property of + the run, not of the feed. The criterion keeps its last confirmed verdict and stays in + the roll-up, so an upstream outage freezes a criterion rather than waiving it. + * NOT_APPLICABLE - there is no question to ask; the criterion is deliberately excluded + for this feed (Fresh / future coverage on a seasonal feed). A property of the feed. + The criterion leaves the roll-up entirely. + * NEVER_EVALUATED - never had a verdict, since the feed first appeared. The initial + value, and the only one the job never writes back once a criterion has left it. + + UNKNOWN is never written to `confirmed_status`: a run that could not look does not + change the answer, it leaves the previous one standing. + """ + + PASS = "pass" + FAIL = "fail" + UNKNOWN = "unknown" + NEVER_EVALUATED = "never_evaluated" + NOT_APPLICABLE = "not_applicable" + + @property + def is_verdict(self) -> bool: + """True for PASS and FAIL - the two values that mean the check actually answered.""" + return self in (CriterionStatus.PASS, CriterionStatus.FAIL) + + +class CriterionPhase(str, Enum): + """Which of the two debouncing mechanisms is currently acting on a criterion. + + Derived from the stored row rather than stored itself (see the job's + `state_machine.phase`): it is a pure function of `probation_start`, `confirmed_status` + and `first_observed_failure_at`, all of which are already on the row, so a stored copy + would be a second thing to keep in step for no gain. + + The three values are mutually exclusive: probation suspends the grace period, so a + criterion can never be serving a penalty and holding a failure under grace at once. + """ + + STEADY = "steady" + IN_GRACE_PERIOD = "in_grace_period" + ON_PROBATION = "on_probation" + + # How long a criterion may keep failing its own check before the failure is confirmed and # the seal is withdrawn. None means the status flips on the first failing day. GRACE_PERIODS: Final[Dict[SealCriterionName, Optional[timedelta]]] = { @@ -63,6 +118,38 @@ class SealCriterionName(str, Enum): ) +# Windows that are part of a criterion's own check rather than of the debouncing machinery, +# so they are applied by the evaluator instead of by the state machine. + +# Stable: how long we must have been tracking a feed - measured from its +# `feed_reliability_seal.created_at` - before it can be called stable. +TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) + +# Fresh / future coverage: how far ahead the latest dataset's service coverage must reach. A +# feed whose coverage ends inside this window is about to go stale, so the criterion fails +# before riders are affected rather than on the day the data runs out. +FUTURE_COVERAGE_HORIZON: Final[timedelta] = timedelta(days=7) + + +def grace_period_for(criterion: str | SealCriterionName) -> Optional[timedelta]: + """How long an observed failure of `criterion` may run before it is confirmed. + + None means the criterion has no grace period and its status flips on the first failing + day. Callers ask for the window rather than declaring their own, so `GRACE_PERIODS` + stays the only place a value can change. + """ + return GRACE_PERIODS[resolve_criterion(criterion)] + + +def probation_period_for(criterion: str | SealCriterionName) -> Optional[timedelta]: + """How long `criterion` must go with no observed failure after a confirmed failure. + + None means the criterion never serves probation (`official` and `stable`, which are + point-in-time state checks). + """ + return PROBATION_PERIODS[resolve_criterion(criterion)] + + def resolve_criterion(criterion: str | SealCriterionName) -> SealCriterionName: """Coerce a stored criterion value to a `SealCriterionName`. diff --git a/api/src/shared/db_models/reliability_criterion_impl.py b/api/src/shared/db_models/reliability_criterion_impl.py index 66e15e34b..a83623c83 100644 --- a/api/src/shared/db_models/reliability_criterion_impl.py +++ b/api/src/shared/db_models/reliability_criterion_impl.py @@ -2,21 +2,18 @@ from feeds_gen.models.reliability_criterion import ReliabilityCriterion from shared.common.seal_criteria import ( - GRACE_PERIODS, - PROBATION_PERIODS, + CriterionStatus, SealCriterionName, + grace_period_for, + probation_period_for, resolve_criterion, window_end, ) from shared.database_gen.sqlacodegen_models import SealCriterion as SealCriterionOrm # The API `status` values are the `seal_criterion_status` DB enum verbatim, so a stored status is -# served as-is with no translation. -STATUS_PASS = "pass" -STATUS_FAIL = "fail" -STATUS_UNKNOWN = "unknown" -STATUS_NEVER_EVALUATED = "never_evaluated" -STATUS_NOT_APPLICABLE = "not_applicable" +# served as-is with no translation - `CriterionStatus` is a `str` enum over exactly those values, +# shared with the nightly job so the two cannot drift apart. class ReliabilityCriterionImpl(ReliabilityCriterion): @@ -42,7 +39,7 @@ def never_evaluated(cls, criterion: SealCriterionName) -> ReliabilityCriterion: """ return cls( criterion=criterion.value, - status=STATUS_NEVER_EVALUATED, + status=CriterionStatus.NEVER_EVALUATED.value, in_grace_period=False, on_probation=False, ) @@ -65,7 +62,7 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # `not_applicable` (withdrawn for this feed) and `never_evaluated` (no verdict ever) do not # participate in the seal, so they carry no grace period, no probation and no windows - just # the flat status, mirroring the row-less `never_evaluated` entry. - if status in (STATUS_NEVER_EVALUATED, STATUS_NOT_APPLICABLE): + if status in (CriterionStatus.NEVER_EVALUATED, CriterionStatus.NOT_APPLICABLE): return cls( criterion=criterion.value, status=status, @@ -76,8 +73,8 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # Criteria exempt from a window (`official` and `stable` from both, `fresh_continuous` from # grace) have their stored values ignored rather than trusted - the policy maps are the # authority on which criteria serve them. - grace_period = GRACE_PERIODS.get(criterion) - probation_period = PROBATION_PERIODS.get(criterion) + grace_period = grace_period_for(criterion) + probation_period = probation_period_for(criterion) probation_start = criterion_row.probation_start if probation_period else None on_probation = probation_start is not None @@ -87,8 +84,8 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # nothing left for grace to protect. in_grace_period = ( grace_period is not None - and status == STATUS_FAIL - and criterion_row.confirmed_status == STATUS_PASS + and status == CriterionStatus.FAIL + and criterion_row.confirmed_status == CriterionStatus.PASS and not on_probation ) diff --git a/api/tests/unittest/models/test_feed_reliability_report_impl.py b/api/tests/unittest/models/test_feed_reliability_report_impl.py index b6470740f..95d7eaeb5 100644 --- a/api/tests/unittest/models/test_feed_reliability_report_impl.py +++ b/api/tests/unittest/models/test_feed_reliability_report_impl.py @@ -3,10 +3,9 @@ from types import SimpleNamespace from shared.common.error_handling import InternalHTTPException -from shared.common.seal_criteria import PROBATION_PERIOD, SealCriterionName +from shared.common.seal_criteria import PROBATION_PERIOD, CriterionStatus, SealCriterionName from shared.database_gen.sqlacodegen_models import FeedReliabilitySeal, SealCriterion from shared.db_models.feed_reliability_report_impl import FeedReliabilityReportImpl -from shared.db_models.reliability_criterion_impl import STATUS_FAIL, STATUS_NEVER_EVALUATED, STATUS_PASS # Anchored to the real clock because the countdowns are derived against `datetime.now`. Every window # below is at least a day clear of its boundary, so the assertions do not race the wall clock. @@ -68,7 +67,7 @@ def test_never_evaluated_feed(self): assert report.evaluated_at is None assert report.on_probation is False assert len(report.criteria) == 6 - assert all(criterion.status == STATUS_NEVER_EVALUATED for criterion in report.criteria) + assert all(criterion.status == CriterionStatus.NEVER_EVALUATED.value for criterion in report.criteria) def test_all_six_criteria_always_returned_in_order(self): """Criteria with no row are filled in, so a client can render six cards unconditionally.""" @@ -77,8 +76,8 @@ def test_all_six_criteria_always_returned_in_order(self): assert [criterion.criterion for criterion in report.criteria] == [name.value for name in SealCriterionName] criteria = by_criterion(report) - assert criteria["official"].status == STATUS_PASS - assert criteria["compliant"].status == STATUS_NEVER_EVALUATED + assert criteria["official"].status == CriterionStatus.PASS.value + assert criteria["compliant"].status == CriterionStatus.NEVER_EVALUATED.value def test_mixed_criteria(self): """The report carries each criterion's own verdict alongside the stored seal outcome.""" @@ -106,10 +105,10 @@ def test_mixed_criteria(self): criteria = by_criterion(report) assert report.has_seal is False - assert criteria["official"].status == STATUS_PASS - assert criteria["available"].status == STATUS_FAIL + assert criteria["official"].status == CriterionStatus.PASS.value + assert criteria["available"].status == CriterionStatus.FAIL.value assert criteria["available"].in_grace_period is False - assert criteria["compliant"].status == STATUS_FAIL + assert criteria["compliant"].status == CriterionStatus.FAIL.value assert criteria["compliant"].in_grace_period is True def test_evaluated_at_is_latest_across_criteria(self): diff --git a/api/tests/unittest/models/test_reliability_criterion_impl.py b/api/tests/unittest/models/test_reliability_criterion_impl.py index 714ad4590..b495b813d 100644 --- a/api/tests/unittest/models/test_reliability_criterion_impl.py +++ b/api/tests/unittest/models/test_reliability_criterion_impl.py @@ -2,16 +2,14 @@ from datetime import datetime, timedelta, timezone from shared.common.error_handling import InternalHTTPException -from shared.common.seal_criteria import GRACE_PERIODS, PROBATION_PERIOD, SealCriterionName -from shared.database_gen.sqlacodegen_models import SealCriterion -from shared.db_models.reliability_criterion_impl import ( - STATUS_FAIL, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, - STATUS_PASS, - STATUS_UNKNOWN, - ReliabilityCriterionImpl, +from shared.common.seal_criteria import ( + GRACE_PERIODS, + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, ) +from shared.database_gen.sqlacodegen_models import SealCriterion +from shared.db_models.reliability_criterion_impl import ReliabilityCriterionImpl # Anchored to the real clock because the countdowns are derived against `datetime.now`. Every window # below is at least a day clear of its boundary, so the assertions do not race the wall clock. @@ -43,7 +41,7 @@ def test_passing_criterion(self): result = ReliabilityCriterionImpl.from_orm(make_row()) assert result.criterion == "compliant" - assert result.status == STATUS_PASS + assert result.status == CriterionStatus.PASS.value assert result.in_grace_period is False assert result.grace_period_ends_at is None assert result.on_probation is False @@ -73,7 +71,7 @@ def test_never_evaluated_factory(self): result = ReliabilityCriterionImpl.never_evaluated(SealCriterionName.AVAILABLE) assert result.criterion == "available" - assert result.status == STATUS_NEVER_EVALUATED + assert result.status == CriterionStatus.NEVER_EVALUATED.value assert result.in_grace_period is False assert result.on_probation is False @@ -82,7 +80,7 @@ def test_never_evaluated_status_passes_through(self): row = make_row(observed_status="never_evaluated", confirmed_status="never_evaluated") result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_NEVER_EVALUATED + assert result.status == CriterionStatus.NEVER_EVALUATED.value assert result.in_grace_period is False assert result.on_probation is False @@ -91,7 +89,7 @@ def test_unknown_status_passes_through(self): row = make_row(criterion=SealCriterionName.AVAILABLE, observed_status="unknown", confirmed_status="pass") result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_UNKNOWN + assert result.status == CriterionStatus.UNKNOWN.value assert result.in_grace_period is False def test_not_applicable_status_is_withdrawn(self): @@ -107,7 +105,7 @@ def test_not_applicable_status_is_withdrawn(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_NOT_APPLICABLE + assert result.status == CriterionStatus.NOT_APPLICABLE.value assert result.in_grace_period is False assert result.on_probation is False assert result.probation_ends_at is None @@ -126,7 +124,7 @@ def test_failing_inside_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is True assert result.grace_period_ends_at == first_failure + GRACE_PERIODS[SealCriterionName.COMPLIANT] assert result.first_failure_at == first_failure @@ -147,7 +145,7 @@ def test_grace_exempt_criterion_is_never_in_grace(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -162,7 +160,7 @@ def test_failing_beyond_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -175,7 +173,7 @@ def test_passing_while_on_probation(self): row = make_row(criterion=SealCriterionName.AVAILABLE, probation_start=probation_start) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_PASS + assert result.status == CriterionStatus.PASS.value assert result.on_probation is True assert result.probation_ends_at == probation_start + PROBATION_PERIOD @@ -191,7 +189,7 @@ def test_grace_does_not_apply_during_probation(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.on_probation is True assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -218,7 +216,7 @@ def test_official_and_stable_have_no_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None diff --git a/api/tests/unittest/models/test_seal_enum_contract.py b/api/tests/unittest/models/test_seal_enum_contract.py index 0db1effb9..39cf3f433 100644 --- a/api/tests/unittest/models/test_seal_enum_contract.py +++ b/api/tests/unittest/models/test_seal_enum_contract.py @@ -11,8 +11,9 @@ mirror of these tests over its own enums, in `functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py`. -Adding a criterion or status means updating, in lockstep: the Liquibase enum, `SealCriterionName` -and the status constants here, `docs/DatabaseCatalogAPI.yaml` (plus a stub regen), and the job enums. +Adding a criterion or status means updating, in lockstep: the Liquibase enum, the enums in +`shared.common.seal_criteria` (which the API and the nightly job both read), and +`docs/DatabaseCatalogAPI.yaml` plus a stub regen. """ import unittest @@ -21,24 +22,12 @@ from shared.common.seal_criteria import ( GRACE_PERIODS, PROBATION_PERIODS, + CriterionStatus, SealCriterionName, ) from shared.database_gen.sqlacodegen_models import SealCriterion -from shared.db_models.reliability_criterion_impl import ( - STATUS_FAIL, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, - STATUS_PASS, - STATUS_UNKNOWN, -) -API_STATUSES = { - STATUS_PASS, - STATUS_FAIL, - STATUS_UNKNOWN, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, -} +API_STATUSES = {status.value for status in CriterionStatus} def db_enum_values(column_name: str) -> set: @@ -53,13 +42,19 @@ def test_criterion_names_match_db_enum(self): """`SealCriterionName` is the full `seal_criterion_name` type, no more and no less.""" assert {criterion.value for criterion in SealCriterionName} == db_enum_values("criterion") - def test_status_constants_match_db_enum(self): - """The API `status` values are the `seal_criterion_status` type verbatim. + def test_status_values_match_db_enum(self): + """`CriterionStatus` is the `seal_criterion_status` type verbatim. There is no DB-to-API translation left, so any divergence would be served raw to clients. + The nightly job writes these same values from the same enum, so one assertion pins both + sides to the schema. """ assert API_STATUSES == db_enum_values("observed_status") + def test_confirmed_status_shares_the_same_type(self): + """Both status columns are the one enum; a check that only covered one could drift.""" + assert db_enum_values("confirmed_status") == db_enum_values("observed_status") + def test_every_db_status_passes_response_validation(self): """Every status the job can store must be accepted by the generated response model. @@ -84,8 +79,9 @@ def test_every_criterion_passes_response_validation(self): def test_policy_maps_cover_every_criterion(self): """Every criterion needs a grace and probation entry, even if the window is None. - `GRACE_PERIODS.get()` would silently return None for a criterion missing from the map, - turning it into a no-grace criterion by accident rather than by decision. + The maps are the single source for both the API's countdowns and the job's debouncing + (`grace_period_for` / `probation_period_for`), so a criterion missing from one would raise + on lookup rather than quietly becoming a no-grace criterion. """ assert set(GRACE_PERIODS) == set(SealCriterionName) assert set(PROBATION_PERIODS) == set(SealCriterionName) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index f33767660..71380816d 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -18,10 +18,12 @@ The evaluators are pure functions over a `FeedSealContext`, so every DB read for a batch of feeds happens here, in a fixed number of queries regardless of batch size. -Only Official is implemented, so the context currently carries just the feed row. Each new -criterion adds the fields it needs here plus one bulk query to populate them: the latest -dataset for Compliant and Fresh, the day's availability rows for Available, the full -dataset coverage history for Fresh continuous coverage. +Official, Stable and Fresh (future coverage) are implemented. Official and Stable read the +feed row alone; Fresh needs one bulk-loaded extra, the feed's latest dataset. Each new +criterion adds the fields it needs here plus, where they are not already on the feed row, one +bulk query to populate them: the latest dataset's validation report for Compliant, the day's +availability rows for Available, the full dataset coverage history for Fresh continuous +coverage. """ import itertools @@ -29,9 +31,21 @@ from datetime import datetime from typing import Dict, Iterator, List, Optional, Sequence +from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed +from shared.database_gen.sqlacodegen_models import Feed, Gtfsdataset, Gtfsfeed + + +@dataclass(frozen=True) +class LatestDataset: + """ + The feed's latest dataset as of the run's `now`, and the fields criteria read off it. + """ + + dataset_id: str + downloaded_at: datetime + service_date_range_end: Optional[datetime] = None @dataclass @@ -49,6 +63,14 @@ class FeedSealContext: # Feed-level flags official: Optional[bool] = None + is_producer_url_unstable: Optional[bool] = None + seasonal: Optional[bool] = None + + # Stable: when the feed was first added to the database. + feed_created_at: Optional[datetime] = None + + # The feed's latest dataset as of `now` - resolved by `downloaded_at` vs `now` + latest_dataset: Optional[LatestDataset] = None # Feeds in these statuses, or not published, are never eligible for the seal. @@ -130,15 +152,56 @@ def iter_eligible_stable_ids( yield chunk +def _load_latest_datasets( + db_session: Session, feed_ids: Sequence[str], now: datetime +) -> Dict[str, LatestDataset]: + """feed_id -> the feed's latest dataset as of `now`, for feeds that had one. + + "Latest as of `now`" is the most recently downloaded dataset with + `downloaded_at <= now`. + + A feed missing from the result had no dataset at all as of `now`. That is deliberately + distinct from a `LatestDataset` whose `service_date_range_end` is None, which had one + whose coverage was never extracted - the criteria read both as UNKNOWN but report which. + """ + if not feed_ids: + return {} + rows = db_session.execute( + select( + Gtfsdataset.feed_id, + Gtfsdataset.id, + Gtfsdataset.downloaded_at, + Gtfsdataset.service_date_range_end, + ) + .where( + Gtfsdataset.feed_id.in_(list(feed_ids)), + Gtfsdataset.downloaded_at.is_not(None), + Gtfsdataset.downloaded_at <= now, + ) + .distinct(Gtfsdataset.feed_id) + .order_by( + Gtfsdataset.feed_id, + Gtfsdataset.downloaded_at.desc(), + Gtfsdataset.id.desc(), + ) + ).all() + return { + row.feed_id: LatestDataset( + dataset_id=row.id, + downloaded_at=row.downloaded_at, + service_date_range_end=row.service_date_range_end, + ) + for row in rows + } + + def build_contexts( db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime ) -> Dict[str, FeedSealContext]: """Load everything the evaluators need for `feeds`, in a fixed number of queries. Args: - db_session: SQLAlchemy session. Unused while Official is the only criterion, since - everything it needs is already on the feed row, but kept in the signature - because every further criterion needs it. + db_session: SQLAlchemy session. feeds: The batch of feeds to load, already loaded (and eligibility-checked via `is_seal_eligible`) by the caller — `update_seals`. now: The evaluation timestamp. @@ -153,10 +216,10 @@ def build_contexts( below. No query, no cost. 2. Needs its own query. Add the field, then a module-level `_load_*` helper that takes - the whole batch and returns a dict keyed by feed_id, and call it once here. Keeping - the query per batch rather than per feed is what holds the query count proportional - to the number of criteria instead of the number of feeds. For example, Available - (issue #1784) would add: + the whole batch and returns a dict keyed by feed_id, and call it once here, as + `_load_latest_datasets` does. Keeping the query per batch rather than per feed + is what holds the query count proportional to the number of criteria instead of the + number of feeds. For example, Available (issue #1784) would add: def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool]: '''feed_id -> whether any availability check succeeded since day_start. @@ -166,12 +229,20 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] called once as `availability = _load_availability_today(...)` and consumed per feed as `availability_success_today=availability.get(feed.id, False)`. """ + latest_datasets = _load_latest_datasets( + db_session, [feed.id for feed in feeds], now + ) + return { feed.id: FeedSealContext( feed_id=feed.id, now=now, stable_id=feed.stable_id, official=feed.official, + is_producer_url_unstable=feed.is_producer_url_unstable, + seasonal=feed.seasonal, + feed_created_at=feed.created_at, + latest_dataset=latest_datasets.get(feed.id), ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py deleted file mode 100644 index ed4574b56..000000000 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# MobilityData 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Seal of Reliability policy values and domain enums. - -The policy values here are the published definition of the seal, so they live in code -rather than in DB config: changing any of them changes which feeds qualify, and that should -go through code review, tests and a deploy. - -Each criterion's grace period belongs with the criterion, as a class attribute on its -evaluator. Only Official is implemented so far and it has none. -""" - -from datetime import timedelta -from enum import Enum -from typing import Final - - -class SealCriterionName(str, Enum): - """The six seal criteria. Values match the seal_criterion_name DB enum. - - All six are listed even though only Official has an evaluator (see #1784 and #1782), - so that the enum stays a faithful mirror of the database type. - """ - - OFFICIAL = "official" - STABLE = "stable" - AVAILABLE = "available" - COMPLIANT = "compliant" - FRESH_COVERAGE = "fresh_coverage" - FRESH_CONTINUOUS = "fresh_continuous" - - -class CriterionStatus(str, Enum): - """A criterion's status. Values match the seal_criterion_status DB enum. - - Only PASS and FAIL are verdicts. The other three say why there is no verdict, and they - are not interchangeable — the roll-up in `seal_updater` sends them in three different - directions: - - * UNKNOWN — we could not look; the inputs the check needs were not there. A property of - the run, not of the feed. The criterion keeps its last confirmed verdict and stays in - the roll-up, so an upstream outage freezes a criterion rather than waiving it. - * NOT_APPLICABLE — there is no question to ask; the criterion is deliberately excluded - for this feed (Fresh / future coverage on a seasonal feed). A property of the feed. - The criterion leaves the roll-up entirely. - * NEVER_EVALUATED — never had a verdict, since the feed first appeared. The initial value, - and the only one the job never writes back once a criterion has left it. - - UNKNOWN is never written to `confirmed_status`: a run that could not look does not - change the answer, it leaves the previous one standing. - """ - - PASS = "pass" - FAIL = "fail" - UNKNOWN = "unknown" - NEVER_EVALUATED = "never_evaluated" - NOT_APPLICABLE = "not_applicable" - - @property - def is_verdict(self) -> bool: - """True for PASS and FAIL — the two values that mean the check actually answered.""" - return self in (CriterionStatus.PASS, CriterionStatus.FAIL) - - -class CriterionPhase(str, Enum): - """Which of the two debouncing mechanisms is currently acting on a criterion. - - Derived from the stored row rather than stored itself (see `state_machine.phase`): it is - a pure function of `probation_start`, `confirmed_status` and `first_observed_failure_at`, - all of which are already on the row, so a stored copy would be a second thing to keep in - step for no gain. - - The three values are mutually exclusive: probation suspends the grace period, so a - criterion can never be serving a penalty and holding a failure under grace at once. - """ - - STEADY = "steady" - IN_GRACE_PERIOD = "in_grace_period" - ON_PROBATION = "on_probation" - - -# A criterion that recovers from a confirmed failure is put on probation: it must then go -# this long with no observed failure before it can contribute to the seal again. It is the -# default for new evaluators; Official is exempt because it is a point-in-time state check -# (see OfficialEvaluator). -# -# Probation is opened only by a recovery. A first evaluation that passes is not a recovery, -# so a feed that has never had a confirmed failure never serves probation at all and can -# hold the seal from its very first evaluation. -PROBATION_PERIOD: Final[timedelta] = timedelta(days=180) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 3ddabbe94..349f33311 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -15,9 +15,12 @@ # """The seal criterion evaluators. -`EVALUATORS` is the registry the job iterates. Only Official is implemented so far -(issue #1783); the remaining criteria are tracked by #1784 and #1782. Adding one means a -new subclass, an entry here, and whatever fields it needs on `FeedSealContext`. +`EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable and Fresh / +future coverage (issue #1784) are implemented; Available and Compliant are the rest of +#1784, and Fresh / continuous coverage is tracked by #1782. Adding one means a new subclass, +an entry here, and whatever fields it needs on `FeedSealContext`. Its windows are not declared +on the subclass: they come from the policy maps in `shared.common.seal_criteria`, which the +read API reads too. `seal_criterion_name` in the database already declares all six values, so a criterion can be added without a schema change. @@ -29,15 +32,21 @@ CriterionEvaluator, CriterionObservation, ) +from tasks.seal_of_reliability.evaluators.fresh_coverage import FreshCoverageEvaluator from tasks.seal_of_reliability.evaluators.official import OfficialEvaluator +from tasks.seal_of_reliability.evaluators.stable import StableEvaluator EVALUATORS: Final[List[CriterionEvaluator]] = [ OfficialEvaluator(), + StableEvaluator(), + FreshCoverageEvaluator(), ] __all__ = [ "EVALUATORS", "CriterionEvaluator", "CriterionObservation", + "FreshCoverageEvaluator", "OfficialEvaluator", + "StableEvaluator", ] diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py index 7291ee2f4..5436557d6 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py @@ -19,12 +19,13 @@ from datetime import timedelta from typing import Optional, Tuple -from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import ( - PROBATION_PERIOD, +from shared.common.seal_criteria import ( CriterionStatus, SealCriterionName, + grace_period_for, + probation_period_for, ) +from tasks.seal_of_reliability.context import FeedSealContext @dataclass(frozen=True) @@ -49,18 +50,33 @@ class CriterionObservation: class CriterionEvaluator: """Evaluates one criterion against a pre-loaded feed context. - Subclasses set `name` and, where they differ from the defaults, `grace_period` and - `probation_period`, then implement `_evaluate`. They never touch the database: all the - data they need is on the context, loaded in bulk by `context.build_contexts`. + A subclass sets `name` and implements `_evaluate`. It does not declare its own windows: + both are resolved from `name` against the policy maps in `shared.common.seal_criteria`, + which the read API reads too, so a criterion cannot debounce one way for the job and + another way for the API. To change a window, change the map. - `grace_period` holds a passing status while an observed failure is still young. - `probation_period` is how long the criterion must go with no observed failure after - recovering from a confirmed failure. None on either means the criterion does not use it. + Evaluators never touch the database: all the data they need is on the context, loaded in + bulk by `context.build_contexts`. """ name: SealCriterionName = None - grace_period: Optional[timedelta] = None - probation_period: Optional[timedelta] = PROBATION_PERIOD + + @property + def grace_period(self) -> Optional[timedelta]: + """How long an observed failure may run before it is confirmed, per the policy map. + + None means the criterion has no grace period and its status flips on the first + failing day. + """ + return grace_period_for(self.name) + + @property + def probation_period(self) -> Optional[timedelta]: + """How long this criterion serves after recovering, per the policy map. + + None means it never serves probation. + """ + return probation_period_for(self.name) def evaluate(self, ctx: FeedSealContext) -> CriterionObservation: """Evaluate the criterion and label the result with this evaluator's name. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py new file mode 100644 index 000000000..c0dcf0f85 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py @@ -0,0 +1,84 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Fresh (future coverage) criterion: the latest dataset still covers the near future.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + FUTURE_COVERAGE_HORIZON, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class FreshCoverageEvaluator(CriterionEvaluator): + """`latest dataset.service_date_range_end >= now + 7 days`. + + This is the only implemented criterion that can return NOT_APPLICABLE. A seasonal feed + is expected to have coverage that runs out between seasons, so the question "does this + feed cover the next week" has no meaningful answer for it — and withdrawing the + criterion is not the same as failing it: the feed keeps its seal on the strength of the + criteria that do apply, and its accumulated state here is frozen in case it stops being + seasonal later. + + The two UNKNOWN cases are both missing inputs rather than verdicts. A feed we have never + fetched a dataset for, or one whose service date range was never extracted from its + validation report, tells us nothing about its freshness; reading either as a failure + would deny the seal on evidence we do not have, and would put the criterion on probation + for it. + + Its grace period — long enough for a producer to notice and publish a fresh dataset — + comes from the policy map. Coverage lapses are the routine kind of failure a grace period + exists to absorb: a publishing pipeline that skips a day should not cost a feed its seal. + """ + + name = SealCriterionName.FRESH_COVERAGE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + # Applicability is a property of the feed, so it is settled before the inputs are + # looked at: a seasonal feed's missing dataset is not an UNKNOWN worth reporting. + if ctx.seasonal is True: + return ( + CriterionStatus.NOT_APPLICABLE, + "the feed is seasonal, so future coverage is not required", + ) + + # Two different missing inputs, kept apart so the report says which: no dataset at + # all as of this run, or one whose coverage was never extracted. + if ctx.latest_dataset is None: + return CriterionStatus.UNKNOWN, "the feed has no latest dataset" + + coverage_end = ctx.latest_dataset.service_date_range_end + if coverage_end is None: + return ( + CriterionStatus.UNKNOWN, + "the latest dataset has no service_date_range_end", + ) + + horizon = ctx.now + FUTURE_COVERAGE_HORIZON + if coverage_end < horizon: + return ( + CriterionStatus.FAIL, + f"coverage ends {coverage_end.isoformat()}, before the " + f"{FUTURE_COVERAGE_HORIZON.days}-day horizon {horizon.isoformat()}", + ) + return ( + CriterionStatus.PASS, + f"coverage ends {coverage_end.isoformat()}, at or beyond the " + f"{FUTURE_COVERAGE_HORIZON.days}-day horizon {horizon.isoformat()}", + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py index 340a2fc31..d848bb4d1 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py @@ -17,8 +17,8 @@ from typing import Tuple +from shared.common.seal_criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator @@ -26,14 +26,12 @@ class OfficialEvaluator(CriterionEvaluator): """`feed.official IS TRUE`. A point-in-time state check: official at the time of reviewing the dataset, with no - 6-month check. Both the grace period and the probation period are None, so the criterion - fails the same day the flag is lost and clears the same day it comes back, taking the - seal with it in both directions. + 6-month check. The policy maps give it neither a grace period nor a probation period, so + the criterion fails the same day the flag is lost and clears the same day it comes back, + taking the seal with it in both directions. """ name = SealCriterionName.OFFICIAL - grace_period = None - probation_period = None def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: # `is True` rather than a truthiness test: NULL is not an endorsement. It is a FAIL diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py new file mode 100644 index 000000000..f13bb4ce8 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py @@ -0,0 +1,81 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Stable criterion: the feed is old enough in the database, from a stable producer URL.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + TRACKING_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class StableEvaluator(CriterionEvaluator): + """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable. + + The clock is the feed's own `created_at`, when it was added to the Mobility Database, so + the criterion asks whether we hold at least six months of history for this feed. Reading + the feed row rather than anything the seal job writes has two consequences worth naming: + a feed that has been in the catalog for years qualifies on the very first seal run, and a + replay at a historical `now` evaluates Stable as correctly as any other criterion, + because the input does not move when the job runs. + + A producer URL change creates a new feed in our data model, with its own `created_at`, so + the six months restart with it. That is the point: the new URL has no history yet. + + TRACKING_PERIOD is inside the check rather than in the state machine, so the criterion is + a point-in-time state check like Official: the policy maps give it no grace period and no + probation. There is nothing for a grace period to absorb, since neither input flickers - + one changes when a reviewer changes it, the other only ever crosses its threshold once - + and probation on a criterion that moves from fail to pass a single time would never be + served. + + Note the check conflates two things: an observed failure means either "the producer URL + is flagged unstable" or "the feed is younger than 180 days", and the stored state cannot + say which. That is a deliberate, accepted loss of detail (see #1761); the `reason` on the + observation says which, for the run that produced it. + """ + + name = SealCriterionName.STABLE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + # Never UNKNOWN: both inputs are columns on the feed row and always readable. + # + # `is True` rather than a truthiness test, to match the SQL predicate + # `is_producer_url_unstable IS NOT TRUE`: NULL is not a claim of instability. + if ctx.is_producer_url_unstable is True: + return CriterionStatus.FAIL, "feed.is_producer_url_unstable is True" + + if ctx.feed_created_at is None: + # feed.created_at is NOT NULL, so this is unreachable from the database and means + # a context was built without it. Still a verdict rather than an UNKNOWN: an + # UNKNOWN would freeze the criterion at whatever it last said, and reporting a + # feed as not yet stable beats holding a seal on a value nobody supplied. + return CriterionStatus.FAIL, "feed.created_at is missing" + + age = ctx.now - ctx.feed_created_at + if age < TRACKING_PERIOD: + return ( + CriterionStatus.FAIL, + f"in the database for {age.days} day(s), needs {TRACKING_PERIOD.days}", + ) + return ( + CriterionStatus.PASS, + f"in the database for {age.days} day(s) with a stable producer URL", + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 3edf7e21c..37807c1d2 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -37,6 +37,11 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from shared.common.seal_criteria import ( + CriterionPhase, + CriterionStatus, + SealCriterionName, +) from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( FeedReliabilitySeal, @@ -50,11 +55,6 @@ build_contexts, is_seal_eligible, ) -from tasks.seal_of_reliability.criteria import ( - CriterionPhase, - CriterionStatus, - SealCriterionName, -) from tasks.seal_of_reliability.evaluators import EVALUATORS from tasks.seal_of_reliability.state_machine import ( SealCriterionState, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py index 4dc5d37a0..8a3a1405f 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py @@ -38,7 +38,7 @@ from datetime import datetime, timedelta, timezone from typing import Optional -from tasks.seal_of_reliability.criteria import ( +from shared.common.seal_criteria import ( CriterionPhase, CriterionStatus, SealCriterionName, diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py index 2989187eb..1b49964c3 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py @@ -43,13 +43,15 @@ from main import tasks_executor from sqlalchemy import delete, select -from tasks.seal_of_reliability.criteria import CriterionStatus +from shared.common.seal_criteria import CriterionStatus, SealCriterionName +from tasks.seal_of_reliability.evaluators import EVALUATORS from tasks.seal_of_reliability.update_seal_of_reliability import get_parameters from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + Gtfsdataset, Gtfsfeed, SealCriterion, ) @@ -67,6 +69,11 @@ # The task always runs against an explicit feed list. These are the eligible seeded feeds. REQUESTED = [OFFICIAL, NOT_OFFICIAL, UNKNOWN_OFFICIAL] +# Every seeded feed gets a dataset covering the next 400 days, so Fresh passes and `official` +# stays the only criterion that separates the feeds. Stable needs nothing seeded: it reads +# `feed.created_at`, which `_seed` already backdates by 400 days. +COVERAGE_END = NOW + timedelta(days=400) + def _seed(db_session, feed_id, official=True, status="active", operational="published"): db_session.add( @@ -79,9 +86,31 @@ def _seed(db_session, feed_id, official=True, status="active", operational="publ official=official, created_at=NOW - timedelta(days=400), producer_url=f"https://example.com/{feed_id}.zip", + seasonal=False, + ) + ) + db_session.flush() + + # Fresh's input. Seeded rather than produced by the run so the criterion passes and + # `official` stays the only variable. + dataset_id = f"{feed_id}_dataset" + db_session.add( + Gtfsdataset( + id=dataset_id, + feed_id=feed_id, + stable_id=dataset_id, + downloaded_at=NOW - timedelta(days=1), + service_date_range_start=NOW - timedelta(days=30), + service_date_range_end=COVERAGE_END, ) ) db_session.flush() + db_session.execute( + Gtfsfeed.__table__.update() + .where(Gtfsfeed.__table__.c.id == feed_id) + .values(latest_dataset_id=dataset_id) + ) + db_session.flush() def _cleanup(db_session): @@ -162,14 +191,36 @@ def seal_state(db_session): @staticmethod @with_db_session(db_url=default_db_url) def criterion_state(db_session): - """stable_id -> the sealcriterion row, for the seeded feeds.""" + """(stable_id, criterion) -> the seal_criterion row, for the seeded feeds. + + Keyed by the criterion too, not just the feed: the registry holds several + evaluators, so a feed has one row per criterion. + """ criterion = SealCriterion.__table__ rows = db_session.execute( select(Feed.stable_id, criterion) .join(criterion, criterion.c.feed_id == Feed.id) .where(Feed.stable_id.like(f"{PREFIX}%")) ).all() - return {row.stable_id: row for row in rows} + return {(row.stable_id, row.criterion): row for row in rows} + + @classmethod + def official_state(cls): + """stable_id -> the `official` seal_criterion row, the one these tests drive.""" + return { + stable_id: row + for (stable_id, criterion), row in cls.criterion_state().items() + if criterion == SealCriterionName.OFFICIAL.value + } + + @staticmethod + def official_criterion(feed_report: dict) -> dict: + """The `official` entry of a report's per-feed criteria list.""" + return next( + row + for row in feed_report["criteria"] + if row["criterion"] == SealCriterionName.OFFICIAL.value + ) @staticmethod @with_db_session(db_url=default_db_url) @@ -199,7 +250,7 @@ def test_two_runs_with_a_change_in_between(self): # --- first run: writes the initial state first = self.run_task({"dry_run": False, "now": NOW.isoformat()}) self.assertFalse(first["dry_run"]) - self.assertGreaterEqual(first["criterion_rows_written"], 3) + self.assertGreaterEqual(first["criterion_rows_written"], 3 * len(EVALUATORS)) self.assertEqual(first["seals_revoked"], 0, "nothing was held beforehand") self.assertEqual( {row["stable_id"] for row in self.ours(first)}, @@ -217,8 +268,8 @@ def test_two_runs_with_a_change_in_between(self): }, "only the official feed earned it; the others were never granted or lost", ) - criteria = self.criterion_state() - self.assertNotIn(DEPRECATED, criteria) + criteria = self.official_state() + self.assertNotIn(DEPRECATED, criteria, "ineligible feeds are never evaluated") self.assertNotIn(UNPUBLISHED, criteria) self.assertEqual( criteria[OFFICIAL].confirmed_status, CriterionStatus.PASS.value @@ -259,22 +310,22 @@ def test_two_runs_with_a_change_in_between(self): self.assertTrue(moved[OFFICIAL]["had_seal"]) self.assertFalse(moved[OFFICIAL]["has_seal"]) self.assertEqual( - moved[OFFICIAL]["criteria"][0]["confirmed_status"], + self.official_criterion(moved[OFFICIAL])["confirmed_status"], CriterionStatus.FAIL.value, ) self.assertEqual( - moved[OFFICIAL]["criteria"][0]["previously_confirmed_status"], + self.official_criterion(moved[OFFICIAL])["previously_confirmed_status"], CriterionStatus.PASS.value, ) self.assertFalse(moved[NOT_OFFICIAL]["had_seal"]) self.assertTrue(moved[NOT_OFFICIAL]["has_seal"]) self.assertEqual( - moved[NOT_OFFICIAL]["criteria"][0]["confirmed_status"], + self.official_criterion(moved[NOT_OFFICIAL])["confirmed_status"], CriterionStatus.PASS.value, ) self.assertEqual( - moved[NOT_OFFICIAL]["criteria"][0]["previously_confirmed_status"], + self.official_criterion(moved[NOT_OFFICIAL])["previously_confirmed_status"], CriterionStatus.FAIL.value, ) @@ -288,7 +339,7 @@ def test_two_runs_with_a_change_in_between(self): }, "the revoked feed keeps its earned_at; the recovered one gains earned_at", ) - criteria = self.criterion_state() + criteria = self.official_state() self.assertEqual(criteria[OFFICIAL].first_observed_failure_at, later) self.assertIsNone( criteria[NOT_OFFICIAL].first_observed_failure_at, "the streak ended" @@ -313,11 +364,11 @@ def test_third_run_with_no_change_is_a_no_op(self): self.assertEqual(report["seals_before_run"], report["seals_after_run"]) after = self.criterion_state() - for stable_id, row in after.items(): - with self.subTest(stable_id=stable_id): + for (stable_id, criterion), row in after.items(): + with self.subTest(stable_id=stable_id, criterion=criterion): self.assertEqual( row.first_observed_failure_at, - before[stable_id].first_observed_failure_at, + before[(stable_id, criterion)].first_observed_failure_at, "a re-evaluation must not restart a failure streak", ) self.assertEqual(row.evaluated_at, later, "but it is re-evaluated") diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py index 87738fb6f..f94f9938b 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py @@ -13,24 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Contract tests pinning the job's seal enums to the database types. No database connection. +"""Contract tests pinning the seal enums to the database types, from the job's tree. -The job writes `seal_criterion.criterion`, `observed_status` and `confirmed_status` from its own -`SealCriterionName` / `CriterionStatus` enums, and the read API serves those stored values straight -through to clients (the API `status` enum is the DB enum verbatim). So a value added here that the -database type does not have, or vice versa, breaks either the write or the read - and the read API -raises on a criterion it does not recognise rather than hiding it. +The job writes `seal_criterion.criterion`, `observed_status` and `confirmed_status` from +`SealCriterionName` / `CriterionStatus`, and the read API serves those stored values straight +through to clients. So a value the enum has that the database type does not, or vice versa, +breaks either the write or the read. -These tests compare against the SQLAlchemy Enum types in the generated models, which are generated -from the Liquibase changelog, so the assertion is against the real schema and needs no live DB. -Adding a criterion or status means updating, in lockstep: the Liquibase enum, the API's -`SealCriterionName` and status constants, `docs/DatabaseCatalogAPI.yaml`, and the enums here. +Both sides now import those enums from one module, `shared.common.seal_criteria`, so this file +asserts the same contract as `api/tests/unittest/models/test_seal_enum_contract.py` rather than a +mirror of it. It is kept because it runs from the function's own tree and venv: it is what catches +the enums failing to resolve at all through the `include_api_folders` symlink, which the API-side +test cannot see. + +The assertions compare against the SQLAlchemy Enum types in the generated models, which come from +the Liquibase changelog, so this checks the real schema with no live DB. Adding a criterion or +status means updating, in lockstep: the Liquibase enum, `shared.common.seal_criteria`, and +`docs/DatabaseCatalogAPI.yaml` plus a stub regen. """ import unittest +from shared.common.seal_criteria import CriterionStatus, SealCriterionName from shared.database_gen.sqlacodegen_models import SealCriterion -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName def db_enum_values(column_name: str) -> set: diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index 190102e29..a37940102 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -16,14 +16,22 @@ """Unit tests for the seal criterion evaluators. No database.""" import unittest -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone -from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName +from shared.common.seal_criteria import ( + FUTURE_COVERAGE_HORIZON, + PROBATION_PERIOD, + TRACKING_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext, LatestDataset from tasks.seal_of_reliability.evaluators import ( EVALUATORS, CriterionEvaluator, + FreshCoverageEvaluator, OfficialEvaluator, + StableEvaluator, ) NOW = datetime(2026, 6, 1, tzinfo=timezone.utc) @@ -133,13 +141,196 @@ def test_never_returns_a_no_verdict_status(self): def test_has_no_grace_or_probation(self): """A point-in-time check: it clears as soon as the feed is official again.""" - self.assertIsNone(OfficialEvaluator.grace_period) - self.assertIsNone(OfficialEvaluator.probation_period) + self.assertIsNone(OfficialEvaluator().grace_period) + self.assertIsNone(OfficialEvaluator().probation_period) def test_reason_names_the_offending_value(self): result = OfficialEvaluator().evaluate(_ctx(official=None)) self.assertIn("None", result.reason) +class TestStable(unittest.TestCase): + """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable.""" + + def _stable_ctx(self, **overrides): + defaults = {"feed_created_at": NOW - TRACKING_PERIOD - timedelta(days=1)} + defaults.update(overrides) + return _ctx(**defaults) + + def test_an_old_feed_with_a_stable_url_passes(self): + self.assertIs( + StableEvaluator().evaluate(self._stable_ctx()).observed_status, + CriterionStatus.PASS, + ) + + def test_an_unstable_producer_url_fails(self): + result = StableEvaluator().evaluate( + self._stable_ctx(is_producer_url_unstable=True) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("is_producer_url_unstable", result.reason) + + def test_a_null_unstable_flag_is_not_a_claim_of_instability(self): + """The SQL predicate is `IS NOT TRUE`, so NULL and False both leave the check open.""" + for flag in (None, False): + with self.subTest(is_producer_url_unstable=flag): + self.assertIs( + StableEvaluator() + .evaluate(self._stable_ctx(is_producer_url_unstable=flag)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_young_feed_fails(self): + result = StableEvaluator().evaluate( + self._stable_ctx(feed_created_at=NOW - timedelta(days=179)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("179", result.reason) + + def test_a_feed_created_today_fails(self): + result = StableEvaluator().evaluate(self._stable_ctx(feed_created_at=NOW)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("0 day(s)", result.reason) + + def test_the_boundary_day_passes(self): + """Exactly 180 days in the database is enough; the check is `<= now - 180 days`.""" + self.assertIs( + StableEvaluator() + .evaluate(self._stable_ctx(feed_created_at=NOW - TRACKING_PERIOD)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_missing_creation_date_fails_rather_than_withholding(self): + """feed.created_at is NOT NULL, so this is unreachable from the database. + + It is still a verdict: an UNKNOWN would freeze the criterion at whatever it last + said, and reporting a feed as not yet stable beats holding a seal on a value nobody + supplied. + """ + result = StableEvaluator().evaluate(self._stable_ctx(feed_created_at=None)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("created_at", result.reason) + + def test_the_unstable_flag_is_checked_before_the_feed_age(self): + """Both fail, but the reason has to name the one an operator can act on.""" + result = StableEvaluator().evaluate( + _ctx(feed_created_at=NOW, is_producer_url_unstable=True) + ) + self.assertIn("is_producer_url_unstable", result.reason) + + def test_never_returns_a_no_verdict_status(self): + """Both inputs are on the feed row, so Stable can never withhold a verdict.""" + for created_at in (None, NOW, NOW - timedelta(days=400)): + for flag in (None, False, True): + with self.subTest(feed_created_at=created_at, unstable=flag): + status = ( + StableEvaluator() + .evaluate( + _ctx( + feed_created_at=created_at, + is_producer_url_unstable=flag, + ) + ) + .observed_status + ) + self.assertTrue(status.is_verdict) + + def test_has_no_grace_or_probation(self): + """A point-in-time check, like Official: neither input flickers.""" + self.assertIsNone(StableEvaluator().grace_period) + self.assertIsNone(StableEvaluator().probation_period) + + +class TestFreshCoverage(unittest.TestCase): + """`latest dataset.service_date_range_end >= now + 7 days`.""" + + @staticmethod + def _dataset(coverage_end): + return LatestDataset( + dataset_id="mdb-1-202606010000", + downloaded_at=NOW - timedelta(days=1), + service_date_range_end=coverage_end, + ) + + def _fresh_ctx(self, coverage_end=NOW + timedelta(days=90), **overrides): + defaults = {"latest_dataset": self._dataset(coverage_end)} + defaults.update(overrides) + return _ctx(**defaults) + + def test_coverage_beyond_the_horizon_passes(self): + self.assertIs( + FreshCoverageEvaluator().evaluate(self._fresh_ctx()).observed_status, + CriterionStatus.PASS, + ) + + def test_coverage_inside_the_horizon_fails(self): + """It fails before the data runs out, not on the day it does.""" + result = FreshCoverageEvaluator().evaluate( + self._fresh_ctx(NOW + timedelta(days=3)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("before the", result.reason) + + def test_the_horizon_itself_passes(self): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(NOW + FUTURE_COVERAGE_HORIZON)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_expired_coverage_fails(self): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(NOW - timedelta(days=1))) + .observed_status, + CriterionStatus.FAIL, + ) + + def test_a_seasonal_feed_is_not_applicable(self): + """Withdrawn from the roll-up rather than failed: the question is meaningless.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(seasonal=True)) + self.assertIs(result.observed_status, CriterionStatus.NOT_APPLICABLE) + self.assertIn("seasonal", result.reason) + + def test_a_seasonal_feed_is_not_applicable_even_with_no_dataset(self): + """Applicability is a property of the feed, so it is settled before the inputs.""" + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(seasonal=True, latest_dataset=None)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_a_non_seasonal_feed_is_evaluated(self): + for seasonal in (None, False): + with self.subTest(seasonal=seasonal): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(seasonal=seasonal)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_no_latest_dataset_is_unknown(self): + """Not a failure: a feed we have never fetched says nothing about its freshness.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(latest_dataset=None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no latest dataset", result.reason) + + def test_a_dataset_with_no_coverage_end_is_unknown(self): + """The other missing input, and the reason has to tell the two apart.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("service_date_range_end", result.reason) + + def test_has_a_grace_period_and_serves_probation(self): + """Coverage lapses are the routine failure the grace period exists to absorb.""" + self.assertEqual(FreshCoverageEvaluator().grace_period, timedelta(days=14)) + self.assertEqual(FreshCoverageEvaluator().probation_period, PROBATION_PERIOD) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py index 17bb85518..e70838347 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py @@ -18,7 +18,7 @@ import unittest from datetime import datetime, timedelta, timezone -from tasks.seal_of_reliability.criteria import ( +from shared.common.seal_criteria import ( PROBATION_PERIOD, CriterionPhase, CriterionStatus, diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index aa0e073ca..70af32f04 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -20,18 +20,22 @@ from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch +from shared.common.seal_criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) from tasks.seal_of_reliability.context import ( build_contexts, count_eligible_feeds, is_seal_eligible, iter_eligible_stable_ids, ) -from tasks.seal_of_reliability.criteria import ( - PROBATION_PERIOD, - CriterionStatus, - SealCriterionName, +from tasks.seal_of_reliability.evaluators import ( + EVALUATORS, + CriterionEvaluator, + OfficialEvaluator, ) -from tasks.seal_of_reliability.evaluators import CriterionEvaluator, OfficialEvaluator from tasks.seal_of_reliability.seal_updater import update_seals from tasks.seal_of_reliability.state_machine import SealCriterionState from sqlalchemy import delete, select @@ -40,6 +44,7 @@ from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + Gtfsdataset, Gtfsfeed, SealCriterion, SealCriterionSnapshot, @@ -55,6 +60,9 @@ DEPRECATED = f"{PREFIX}deprecated" UNPUBLISHED = f"{PREFIX}unpublished" INACTIVE = f"{PREFIX}inactive" +# An official feed old enough in the database for Stable to pass (every seeded feed is +# backdated 400 days), used by the tests that run the real registry rather than a patched one. +TRACKED = f"{PREFIX}tracked" # The eligible feeds this module seeds. Runs that assert exact counts must be scoped to # these: an unnamed run also covers the fixtures seeded by conftest.pytest_sessionstart and @@ -85,6 +93,13 @@ def _evaluate(self, ctx): # Patched over the registry so the roll-up sees a criterion that can be on probation. WITH_PROBATION = [OfficialEvaluator(), _StandInEvaluator()] +# Official on its own. The classes below that assert seal outcomes are about the roll-up, +# the report and the snapshot table rather than about any particular criterion, and Official +# is the one that reaches a verdict from nothing but the feed row. Stable and Fresh need a +# seal row and a dataset to say anything, and both are exercised against real data in +# TestFullRegistry. +ONLY_OFFICIAL = [OfficialEvaluator()] + DARK_FROM = NOW + timedelta(days=2) @@ -120,13 +135,17 @@ def _evaluate(self, ctx): class _StopsApplyingEvaluator(CriterionEvaluator): """A criterion that stops applying to the feed partway through, standing in for #1782. - Fresh (future coverage) does not apply to a seasonal feed, and a feed can be marked + Fresh / continuous coverage does not apply to a seasonal feed, and a feed can be marked seasonal at any time. Keyed on the clock rather than on `official` so a test can drive it and Official in opposite directions at the same moment. No grace period, so its verdicts land immediately and the tests are about what happens once it withdraws. + + It borrows `fresh_continuous`, which has no evaluator of its own yet, rather than + `fresh_coverage`, whose real evaluator now covers the same seasonal case against real + data in `TestFullRegistry`. """ - name = SealCriterionName.FRESH_COVERAGE + name = SealCriterionName.FRESH_CONTINUOUS grace_period = None def _evaluate(self, ctx): @@ -145,6 +164,8 @@ def _seed_feed( official=True, status="active", operational_status="published", + seasonal=False, + is_producer_url_unstable=None, ): """Insert one GTFS feed.""" db_session.add( @@ -155,6 +176,8 @@ def _seed_feed( status=status, operational_status=operational_status, official=official, + seasonal=seasonal, + is_producer_url_unstable=is_producer_url_unstable, created_at=NOW - timedelta(days=400), producer_url=f"https://example.com/{feed_id}.zip", ) @@ -162,6 +185,45 @@ def _seed_feed( db_session.flush() +def _seed_dataset( + db_session, feed_id: str, coverage_end, downloaded_at=None, suffix="" +): + """Give the feed a dataset covering up to `coverage_end` (None = never extracted). + + `latest_dataset_id` is pointed at it too, so a test that only seeds one dataset matches + what the catalog looks like. The criterion resolves the latest dataset from + `downloaded_at` rather than from that pointer, which is what the two-dataset tests below + exercise. + """ + dataset_id = f"{feed_id}_dataset{suffix}" + db_session.add( + Gtfsdataset( + id=dataset_id, + feed_id=feed_id, + stable_id=dataset_id, + downloaded_at=downloaded_at or NOW - timedelta(days=1), + service_date_range_start=NOW - timedelta(days=30), + service_date_range_end=coverage_end, + ) + ) + db_session.flush() + db_session.execute( + Gtfsfeed.__table__.update() + .where(Gtfsfeed.__table__.c.id == feed_id) + .values(latest_dataset_id=dataset_id) + ) + db_session.commit() + + +def _set_seasonal(db_session, feed_id: str, seasonal): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(seasonal=seasonal) + ) + db_session.commit() + + def _set_official(db_session, feed_id: str, official): db_session.execute( Feed.__table__.update() @@ -195,6 +257,7 @@ def setUp(self, db_session): _seed_feed(db_session, DEPRECATED, status="deprecated") _seed_feed(db_session, UNPUBLISHED, operational_status="unpublished") _seed_feed(db_session, INACTIVE, status="inactive") + _seed_feed(db_session, TRACKED) db_session.commit() @with_db_session(db_url=default_db_url) @@ -236,6 +299,38 @@ def seal_row(feed_id, db_session): def set_official(feed_id, official, db_session): _set_official(db_session, feed_id, official) + @staticmethod + @with_db_session(db_url=default_db_url) + def set_seasonal(feed_id, seasonal, db_session): + _set_seasonal(db_session, feed_id, seasonal) + + @staticmethod + @with_db_session(db_url=default_db_url) + def set_producer_url_unstable(feed_id, unstable, db_session): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(is_producer_url_unstable=unstable) + ) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_dataset( + feed_id, coverage_end, downloaded_at=None, suffix="", db_session=None + ): + _seed_dataset(db_session, feed_id, coverage_end, downloaded_at, suffix) + + @staticmethod + @with_db_session(db_url=default_db_url) + def set_feed_created_at(feed_id, created_at, db_session): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(created_at=created_at) + ) + db_session.commit() + def _feeds_by_stable_id(db_session, *stable_ids): """Plain by-id load — mirrors what `update_seals` does before checking eligibility.""" @@ -310,6 +405,95 @@ def test_loads_the_fields_the_evaluators_need(self, db_session): self.assertEqual(ctx.stable_id, OFFICIAL) self.assertTrue(ctx.official) self.assertEqual(ctx.now, NOW) + self.assertFalse(ctx.seasonal) + self.assertIsNone(ctx.is_producer_url_unstable) + + @with_db_session(db_url=default_db_url) + def test_stables_clock_is_the_feed_row_and_needs_no_query(self, db_session): + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.feed_created_at, NOW - timedelta(days=400)) + + @with_db_session(db_url=default_db_url) + def test_a_feed_with_no_dataset_says_so(self, db_session): + """The bulk load misses, and the context says so rather than guessing a value.""" + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_dataset) + + @with_db_session(db_url=default_db_url) + def test_the_latest_dataset_coverage_is_loaded(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.latest_dataset.dataset_id, f"{TRACKED}_dataset") + self.assertEqual( + ctx.latest_dataset.service_date_range_end, NOW + timedelta(days=90) + ) + + @with_db_session(db_url=default_db_url) + def test_a_dataset_with_no_coverage_end_is_not_a_missing_dataset(self, db_session): + """The two UNKNOWN cases must stay distinguishable at the context layer.""" + _seed_dataset(db_session, TRACKED, coverage_end=None) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.latest_dataset, "the dataset is there ...") + self.assertIsNone( + ctx.latest_dataset.service_date_range_end, "... its coverage end is not" + ) + + @with_db_session(db_url=default_db_url) + def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): + """A replay must not see a dataset published after the day it is evaluating. + + `gtfsfeed.latest_dataset_id` points at the newest dataset that exists today, so + reading it would report the feed as fresh on a day when the data covering that day + had not been published yet. + """ + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=5), + downloaded_at=NOW - timedelta(days=10), + suffix="_old", + ) + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=400), + downloaded_at=NOW + timedelta(days=10), + suffix="_new", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + + as_of_now = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual( + as_of_now.latest_dataset.service_date_range_end, + NOW + timedelta(days=5), + "the newer dataset had not been downloaded yet", + ) + + later = NOW + timedelta(days=20) + as_of_later = build_contexts(db_session, feeds, later)[feeds[0].id] + self.assertEqual( + as_of_later.latest_dataset.service_date_range_end, + NOW + timedelta(days=400), + "by then it had", + ) + + @with_db_session(db_url=default_db_url) + def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_session): + """It is excluded rather than guessed at: we cannot say whether it existed yet.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + db_session.execute( + Gtfsdataset.__table__.update() + .where(Gtfsdataset.__table__.c.feed_id == TRACKED) + .values(downloaded_at=None) + ) + db_session.commit() + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_dataset) @with_db_session(db_url=default_db_url) def test_builds_one_context_per_feed(self, db_session): @@ -319,7 +503,10 @@ def test_builds_one_context_per_feed(self, db_session): self.assertEqual({ctx.official for ctx in contexts.values()}, {True, False}) +@patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestUpdateSeals(SealDbTestCase): + """The report, the roll-up and the two seal transitions, driven by Official alone.""" + def test_dry_run_writes_nothing(self): report = update_seals(dry_run=True, stable_feed_ids=[OFFICIAL], now=NOW) self.assertTrue(report["dry_run"]) @@ -508,7 +695,11 @@ def test_regaining_official_status_clears_the_criterion(self): self.assertEqual(seal.seal_earned_at, later) def test_partial_criteria_run_skips_the_roll_up(self): - """Named explicitly, `official` is still the whole registry, so not partial.""" + """Naming every criterion in the registry is not a partial run. + + Under this class's patch the registry is Official alone; `TestCriteriaSelection` + covers the same distinction against the real, larger registry. + """ report = update_seals( dry_run=False, stable_feed_ids=[OFFICIAL], @@ -526,21 +717,6 @@ def test_a_feed_list_is_required(self): update_seals(dry_run=True, stable_feed_ids=feeds, now=NOW) self.assertIn("stable_feed_ids", str(caught.exception)) - def test_unknown_criterion_raises(self): - with self.assertRaises(ValueError): - update_seals( - stable_feed_ids=[OFFICIAL], criteria=["not_a_criterion"], now=NOW - ) - - def test_criterion_without_an_evaluator_raises(self): - """`stable` is a valid DB enum value but has no evaluator yet (#1784).""" - with self.assertRaises(ValueError): - update_seals( - stable_feed_ids=[OFFICIAL], - criteria=[SealCriterionName.STABLE.value], - now=NOW, - ) - def test_a_run_with_no_usable_feed_raises(self): """Nothing was evaluated, so a report saying so would be too quiet.""" with self.assertRaises(ValueError) as caught: @@ -591,6 +767,7 @@ def test_feeds_omitted_is_zero_when_nothing_was_dropped(self): self.assertEqual(report["feeds_omitted"], 0) +@patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestCriterionSnapshot(SealDbTestCase): """seal_criterion_snapshot, the per-day record of each criterion (issue #1809). @@ -695,6 +872,259 @@ def test_the_log_covers_every_field_the_state_carries(self): self.assertEqual(carried - recorded, set()) +class TestFullRegistry(SealDbTestCase): + """Official, Stable and Fresh together, against real rows and the real registry. + + The classes above patch the registry down to Official because they are about the + report, the roll-up and the snapshot table rather than about any one criterion. These + tests are the other half: the three implemented criteria, driven by the columns they + actually read. + """ + + FAR_FUTURE = NOW + timedelta(days=90) + + def criteria_of(self, feed_id=TRACKED): + return { + criterion: row.confirmed_status + for criterion, row in self.criterion_rows(feed_id).items() + } + + def test_every_criterion_is_written_for_every_feed(self): + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + set(self.criterion_rows(TRACKED)), + {evaluator.name.value for evaluator in EVALUATORS}, + ) + + def test_a_feed_meeting_all_three_earns_the_seal(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of(), + { + SealCriterionName.OFFICIAL.value: CriterionStatus.PASS.value, + SealCriterionName.STABLE.value: CriterionStatus.PASS.value, + SealCriterionName.FRESH_COVERAGE.value: CriterionStatus.PASS.value, + }, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_feed_new_to_the_database_cannot_hold_the_seal_yet(self): + """Stable reads the feed's own age, so a freshly added feed fails it.""" + self.set_feed_created_at(TRACKED, NOW - timedelta(days=30)) + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.FAIL.value, + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_the_seal_arrives_once_the_feed_is_old_enough(self): + """The same feed, evaluated the day it was added and 181 days later. + + Stable's clock is `feed.created_at`, which does not move when the job runs, so the + second run is a plain replay at a later `now` rather than something the first run + had to set up. + """ + self.set_feed_created_at(TRACKED, NOW) + self.seed_dataset(TRACKED, NOW + timedelta(days=400)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertFalse( + self.seal_row(TRACKED).has_seal, "in the database for zero days" + ) + + later = NOW + timedelta(days=181) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.STABLE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + self.assertIsNone(row.probation_start, "Stable serves no probation") + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_an_old_feed_qualifies_on_its_very_first_run(self): + """The point of reading `feed.created_at`: no six-month wait after deployment for a + feed that has already been in the catalog for years.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.PASS.value, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_an_unstable_producer_url_denies_the_seal_immediately(self): + """Stable has no grace period, so the flag costs the seal the day it is set.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + self.set_producer_url_unstable(TRACKED, True) + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.FAIL.value, + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): + """UNKNOWN is not a failure: the other two criteria still decide the seal.""" + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + "no verdict was ever produced, so the criterion is out of service", + ) + self.assertIsNone(row.last_verdict_at) + self.assertTrue( + self.seal_row(TRACKED).has_seal, + "Official and Stable carry it while Fresh has nothing to say", + ) + + def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): + """Fresh has a 14-day grace period, but a criterion that has never passed has not + earned it, so its first verdict lands as a confirmed failure the same day.""" + self.seed_dataset(TRACKED, NOW + timedelta(days=2)) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_the_grace_period_absorbs_a_lapse_on_a_feed_that_was_passing(self): + self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Five days on, the same dataset now covers only five more days: inside the horizon. + within_grace = NOW + timedelta(days=5) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=within_grace) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "the grace period holds the verdict while the producer catches up", + ) + self.assertEqual(row.first_observed_failure_at, within_grace) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_lapse_outlasting_the_grace_period_costs_the_seal(self): + self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + first_failure = NOW + timedelta(days=5) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=first_failure) + + outlasted = first_failure + timedelta(days=15) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=outlasted) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.first_observed_failure_at, + first_failure, + "the streak is measured from its start, not from this run", + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_seasonal_feed_is_not_denied_by_fresh(self): + """NOT_APPLICABLE withdraws the criterion instead of failing it, which is the whole + point of the value: a seasonal feed keeps the seal on the criteria that do apply. + """ + self.set_seasonal(TRACKED, True) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertEqual(row.confirmed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self): + """A feed marked seasonal after a confirmed Fresh failure stops being judged on it.""" + self.seed_dataset(TRACKED, NOW - timedelta(days=1)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + self.set_seasonal(TRACKED, True) + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertEqual( + row.last_confirmed_failure_at, NOW, "the failure stays on record" + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_run_reports_all_three_criteria_with_their_reasons(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + report = update_seals(dry_run=True, stable_feed_ids=[TRACKED], now=NOW) + + criteria = report["feeds"][0]["criteria"] + self.assertEqual( + [row["criterion"] for row in criteria], + [evaluator.name.value for evaluator in EVALUATORS], + ) + for row in criteria: + with self.subTest(criterion=row["criterion"]): + self.assertTrue(row["reason"]) + + +class TestCriteriaSelection(SealDbTestCase): + """The `criteria` filter, against the real registry rather than a patched one.""" + + def test_unknown_criterion_raises(self): + with self.assertRaises(ValueError): + update_seals( + stable_feed_ids=[OFFICIAL], criteria=["not_a_criterion"], now=NOW + ) + + def test_criterion_without_an_evaluator_raises(self): + """`fresh_continuous` is a valid DB enum value but has no evaluator yet (#1782).""" + with self.assertRaises(ValueError): + update_seals( + stable_feed_ids=[OFFICIAL], + criteria=[SealCriterionName.FRESH_CONTINUOUS.value], + now=NOW, + ) + + def test_naming_every_implemented_criterion_is_not_a_partial_run(self): + report = update_seals( + dry_run=True, + stable_feed_ids=[OFFICIAL], + criteria=[evaluator.name.value for evaluator in EVALUATORS], + now=NOW, + ) + self.assertFalse(report["partial_run"]) + + def test_naming_a_subset_is_a_partial_run(self): + """More than one criterion is implemented now, so a subset is genuinely partial.""" + report = update_seals( + dry_run=True, + stable_feed_ids=[OFFICIAL], + criteria=[SealCriterionName.OFFICIAL.value], + now=NOW, + ) + self.assertTrue(report["partial_run"]) + self.assertIn("note", report) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", WITH_PROBATION) class TestProbation(SealDbTestCase): """Probation persisted and rolled up, driven by `_StandInEvaluator`. @@ -1003,7 +1433,9 @@ class TestCriterionThatStopsApplying(SealDbTestCase): FAILED_AT = NOW + timedelta(days=1) def stand_in(self): - return self.criterion_rows(OFFICIAL).get(SealCriterionName.FRESH_COVERAGE.value) + return self.criterion_rows(OFFICIAL).get( + SealCriterionName.FRESH_CONTINUOUS.value + ) def run_at(self, moment): update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=moment) From 0e37ae79ba9cd5b05ca07eb796f4a26f4e72d5b0 Mon Sep 17 00:00:00 2001 From: cka-y Date: Wed, 26 Aug 2026 11:07:09 -0400 Subject: [PATCH 2/5] cleared up comments overload --- api/src/shared/common/seal_criteria.py | 14 +-------- .../evaluators/fresh_coverage.py | 15 +--------- .../seal_of_reliability/evaluators/stable.py | 30 ++----------------- 3 files changed, 5 insertions(+), 54 deletions(-) diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index c52695891..8c609c854 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -9,13 +9,6 @@ share one definition. The job owns the writing of `seal_criterion`; the API only reads it back and derives the two countdowns from these windows. -Every policy value the seal depends on belongs here and nowhere else: the criterion names, -the two status/phase vocabularies, each criterion's grace and probation windows, and the -windows that are part of a criterion's own check (Stable's tracking period, Fresh's coverage -horizon). Nothing downstream may restate one - a job evaluator asks -`grace_period_for(name)` rather than declaring its own, so changing a window here changes it -for the job, the API and the materialized view roll-up at once. - See #1761 for the algorithm and #1760 for the tables. """ @@ -118,16 +111,11 @@ class CriterionPhase(str, Enum): ) -# Windows that are part of a criterion's own check rather than of the debouncing machinery, -# so they are applied by the evaluator instead of by the state machine. - # Stable: how long we must have been tracking a feed - measured from its # `feed_reliability_seal.created_at` - before it can be called stable. TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) -# Fresh / future coverage: how far ahead the latest dataset's service coverage must reach. A -# feed whose coverage ends inside this window is about to go stale, so the criterion fails -# before riders are affected rather than on the day the data runs out. +# Fresh / future coverage: how far ahead the latest dataset's service coverage must reach FUTURE_COVERAGE_HORIZON: Final[timedelta] = timedelta(days=7) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py index c0dcf0f85..c932cb4e1 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py @@ -31,20 +31,7 @@ class FreshCoverageEvaluator(CriterionEvaluator): This is the only implemented criterion that can return NOT_APPLICABLE. A seasonal feed is expected to have coverage that runs out between seasons, so the question "does this - feed cover the next week" has no meaningful answer for it — and withdrawing the - criterion is not the same as failing it: the feed keeps its seal on the strength of the - criteria that do apply, and its accumulated state here is frozen in case it stops being - seasonal later. - - The two UNKNOWN cases are both missing inputs rather than verdicts. A feed we have never - fetched a dataset for, or one whose service date range was never extracted from its - validation report, tells us nothing about its freshness; reading either as a failure - would deny the seal on evidence we do not have, and would put the criterion on probation - for it. - - Its grace period — long enough for a producer to notice and publish a fresh dataset — - comes from the policy map. Coverage lapses are the routine kind of failure a grace period - exists to absorb: a publishing pipeline that skips a day should not cost a feed its seal. + feed cover the next week" has no meaningful answer for it. """ name = SealCriterionName.FRESH_COVERAGE diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py index f13bb4ce8..0372a700e 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py @@ -29,44 +29,20 @@ class StableEvaluator(CriterionEvaluator): """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable. - The clock is the feed's own `created_at`, when it was added to the Mobility Database, so - the criterion asks whether we hold at least six months of history for this feed. Reading - the feed row rather than anything the seal job writes has two consequences worth naming: - a feed that has been in the catalog for years qualifies on the very first seal run, and a - replay at a historical `now` evaluates Stable as correctly as any other criterion, - because the input does not move when the job runs. - - A producer URL change creates a new feed in our data model, with its own `created_at`, so - the six months restart with it. That is the point: the new URL has no history yet. - - TRACKING_PERIOD is inside the check rather than in the state machine, so the criterion is - a point-in-time state check like Official: the policy maps give it no grace period and no - probation. There is nothing for a grace period to absorb, since neither input flickers - - one changes when a reviewer changes it, the other only ever crosses its threshold once - - and probation on a criterion that moves from fail to pass a single time would never be - served. - Note the check conflates two things: an observed failure means either "the producer URL is flagged unstable" or "the feed is younger than 180 days", and the stored state cannot - say which. That is a deliberate, accepted loss of detail (see #1761); the `reason` on the - observation says which, for the run that produced it. + say which. That is a deliberate. """ name = SealCriterionName.STABLE def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: - # Never UNKNOWN: both inputs are columns on the feed row and always readable. - # - # `is True` rather than a truthiness test, to match the SQL predicate - # `is_producer_url_unstable IS NOT TRUE`: NULL is not a claim of instability. - if ctx.is_producer_url_unstable is True: + if ctx.is_producer_url_unstable: return CriterionStatus.FAIL, "feed.is_producer_url_unstable is True" if ctx.feed_created_at is None: # feed.created_at is NOT NULL, so this is unreachable from the database and means - # a context was built without it. Still a verdict rather than an UNKNOWN: an - # UNKNOWN would freeze the criterion at whatever it last said, and reporting a - # feed as not yet stable beats holding a seal on a value nobody supplied. + # a context was built without it. Still a verdict rather than an UNKNOWN. return CriterionStatus.FAIL, "feed.created_at is missing" age = ctx.now - ctx.feed_created_at From 31f754592dd68a999494d7d01a05b7934405e325 Mon Sep 17 00:00:00 2001 From: cka-y Date: Thu, 27 Aug 2026 13:12:22 -0400 Subject: [PATCH 3/5] fix: comment --- api/src/shared/common/seal_criteria.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index 8c609c854..ffcb972a8 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -112,7 +112,7 @@ class CriterionPhase(str, Enum): # Stable: how long we must have been tracking a feed - measured from its -# `feed_reliability_seal.created_at` - before it can be called stable. +# `feed..created_at` - before it can be called stable. TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) # Fresh / future coverage: how far ahead the latest dataset's service coverage must reach From 3ad517e4978432a7f146a3f8b797cf25a47ef5c8 Mon Sep 17 00:00:00 2001 From: cka-y Date: Thu, 27 Aug 2026 15:45:41 -0400 Subject: [PATCH 4/5] feat: available + compliant --- api/src/shared/common/seal_criteria.py | 49 +- .../db_models/feed_reliability_report_impl.py | 26 +- api/tests/unittest/test_feeds.py | 36 + docs/DatabaseCatalogAPI.yaml | 22 + docs/OperationsAPI.yaml | 24 + .../src/tasks/seal_of_reliability/context.py | 131 +++- .../evaluators/__init__.py | 11 +- .../evaluators/available.py | 47 ++ .../evaluators/compliant.py | 57 ++ .../tasks/seal_of_reliability/seal_updater.py | 76 +- .../test_seal_end_to_end_db.py | 56 +- .../test_seal_evaluators.py | 145 +++- .../test_seal_updater_db.py | 661 +++++++++++++++++- 13 files changed, 1279 insertions(+), 62 deletions(-) create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index ffcb972a8..c0fbac829 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta from enum import Enum -from typing import Dict, Final, Optional +from typing import Dict, Final, Iterable, Optional, Tuple from shared.common.error_handling import raise_internal_http_error, unknown_seal_criterion @@ -61,6 +61,20 @@ def is_verdict(self) -> bool: return self in (CriterionStatus.PASS, CriterionStatus.FAIL) +class SealStatus(str, Enum): + """The feed-level seal outcome.""" + + GRANTED = "granted" + NOT_GRANTED = "not_granted" + UNKNOWN = "unknown" + NEVER_EVALUATED = "never_evaluated" + + @property + def is_answer(self) -> bool: + """True for GRANTED and NOT_GRANTED - the two values that actually decide the seal.""" + return self in (SealStatus.GRANTED, SealStatus.NOT_GRANTED) + + class CriterionPhase(str, Enum): """Which of the two debouncing mechanisms is currently acting on a criterion. @@ -111,10 +125,41 @@ class CriterionPhase(str, Enum): ) +def roll_up_seal_status( + criteria: Iterable[Tuple[CriterionStatus, bool]], +) -> SealStatus: + """The feed-level seal outcome from its criteria. + * every criterion NEVER_EVALUATED -> seal NEVER_EVALUATED. + * any criterion NEVER_EVALUATED -> seal UNKNOWN. + * otherwise every criterion is a confirmed verdict: GRANTED when they all pass and none is on + probation, NOT_GRANTED otherwise. + """ + in_scope = [ + (status, on_probation) for status, on_probation in criteria if status is not CriterionStatus.NOT_APPLICABLE + ] + if not in_scope: + # Every criterion is NOT_APPLICABLE, so there is nothing left to judge the feed by. + return SealStatus.NEVER_EVALUATED + + unjudged = sum(1 for status, _ in in_scope if status is CriterionStatus.NEVER_EVALUATED) + if unjudged == len(in_scope): + # Every criterion that isn't NOT_APPLICABLE is NEVER_EVALUATED + return SealStatus.NEVER_EVALUATED + if unjudged: + # At least one criterion is NEVER_EVALUATED + return SealStatus.UNKNOWN + + granted = all(status is CriterionStatus.PASS and not on_probation for status, on_probation in in_scope) + return SealStatus.GRANTED if granted else SealStatus.NOT_GRANTED + + # Stable: how long we must have been tracking a feed - measured from its -# `feed..created_at` - before it can be called stable. +# `feed.created_at` - before it can be called stable. TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) +# Available: how far back to look for an availability check +AVAILABILITY_LOOKBACK: Final[timedelta] = timedelta(hours=24) + # Fresh / future coverage: how far ahead the latest dataset's service coverage must reach FUTURE_COVERAGE_HORIZON: Final[timedelta] = timedelta(days=7) diff --git a/api/src/shared/db_models/feed_reliability_report_impl.py b/api/src/shared/db_models/feed_reliability_report_impl.py index 9b568e307..b46c1e0e0 100644 --- a/api/src/shared/db_models/feed_reliability_report_impl.py +++ b/api/src/shared/db_models/feed_reliability_report_impl.py @@ -1,9 +1,32 @@ from feeds_gen.models.feed_reliability_report import FeedReliabilityReport -from shared.common.seal_criteria import SealCriterionName, resolve_criterion +from shared.common.seal_criteria import ( + PROBATION_EXEMPT_CRITERIA, + CriterionStatus, + SealCriterionName, + resolve_criterion, + roll_up_seal_status, +) from shared.database_gen.sqlacodegen_models import Gtfsfeed as GtfsfeedOrm +from shared.database_gen.sqlacodegen_models import SealCriterion as SealCriterionOrm from shared.db_models.reliability_criterion_impl import ReliabilityCriterionImpl +def _seal_status_of(criterion_rows: list[SealCriterionOrm]) -> str: + """The feed-level seal status derived from a feed's `seal_criterion` rows. + + Not stored: the rule lives in `shared.common.seal_criteria` and is shared with the nightly job, + so the status served and the one `has_seal` was decided by cannot drift apart. Probation is read + only for the criteria that serve it, matching `on_probation` below. + """ + return roll_up_seal_status( + ( + CriterionStatus(row.confirmed_status), + row.probation_start is not None and row.criterion not in PROBATION_EXEMPT_CRITERIA, + ) + for row in criterion_rows + ).value + + class FeedReliabilityReportImpl(FeedReliabilityReport): """Implementation of the `FeedReliabilityReport` model. @@ -58,6 +81,7 @@ def from_orm(cls, feed: GtfsfeedOrm | None) -> FeedReliabilityReport | None: return cls( feed_id=feed.stable_id, has_seal=bool(seal.has_seal) if seal is not None else False, + seal_status=_seal_status_of(criterion_rows), earned_at=seal.seal_earned_at if seal is not None else None, lost_at=seal.seal_lost_at if seal is not None else None, evaluated_at=max(evaluated_ats) if evaluated_ats else None, diff --git a/api/tests/unittest/test_feeds.py b/api/tests/unittest/test_feeds.py index 1ae240bbc..8ff15e50c 100644 --- a/api/tests/unittest/test_feeds.py +++ b/api/tests/unittest/test_feeds.py @@ -466,6 +466,7 @@ def test_gtfs_feed_reliability_never_evaluated(client: TestClient): body = response.json() assert body["feed_id"] == TEST_GTFS_FEED_STABLE_IDS[0] assert body["has_seal"] is False + assert body["seal_status"] == "never_evaluated", "no criterion row at all, so nothing was ever decided" assert body["on_probation"] is False assert len(body["criteria"]) == 6 assert {criterion["status"] for criterion in body["criteria"]} == {"never_evaluated"} @@ -566,6 +567,41 @@ def test_gtfs_feed_get_embeds_reliability_seal(client: TestClient): assert seal["evaluated_at"] is not None +def test_gtfs_feed_reliability_reports_an_undecided_seal_as_unknown(client: TestClient): + """`has_seal: false` covers three different things; `seal_status` is what separates them. + + A feed whose criteria have not all been judged is not a feed that was judged and failed, and a + client has to be able to tell those apart. The status is derived from the criterion rows rather + than stored, so one passing criterion beside one never-evaluated one is all it takes. + """ + feed_stable_id = TEST_GTFS_FEED_STABLE_IDS[2] + criteria = { + "official": {"observed_status": "pass", "confirmed_status": "pass", "evaluated_at": SEAL_NOW}, + "available": {"observed_status": "unknown", "confirmed_status": "never_evaluated", "evaluated_at": SEAL_NOW}, + } + with _seal_rows(feed_stable_id, has_seal=False, criteria=criteria): + response = client.request("GET", f"/v1/gtfs_feeds/{feed_stable_id}/reliability", headers=authHeaders) + + assert response.status_code == 200, f"Response status code was {response.status_code} instead of 200" + body = response.json() + assert body["has_seal"] is False + assert body["seal_status"] == "unknown" + + +def test_gtfs_feed_reliability_reports_a_judged_seal_as_granted(client: TestClient): + """The other side of the same coin: every criterion in scope judged, and all passing.""" + feed_stable_id = TEST_GTFS_FEED_STABLE_IDS[3] + criteria = { + criterion: {"observed_status": "pass", "confirmed_status": "pass", "evaluated_at": SEAL_NOW} + for criterion in ("official", "stable", "available", "compliant", "fresh_coverage", "fresh_continuous") + } + with _seal_rows(feed_stable_id, has_seal=True, criteria=criteria): + response = client.request("GET", f"/v1/gtfs_feeds/{feed_stable_id}/reliability", headers=authHeaders) + + assert response.status_code == 200, f"Response status code was {response.status_code} instead of 200" + assert response.json()["seal_status"] == "granted" + + def test_gtfs_feed_get_without_seal_reports_null(client: TestClient): """A feed that has never been evaluated reports a null summary rather than an empty object.""" response = client.request( diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 98868bb6d..1eff8d904 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -1119,6 +1119,28 @@ components: description: Whether the feed currently holds the Seal of Reliability. type: boolean example: false + seal_status: + description: > + Why the feed does or does not hold the seal. `has_seal` is true exactly when this is + `granted`; the other three values are all `has_seal: false` and are not the same thing: + + * `granted` - every criterion in scope is a confirmed pass and none is on probation. + * `not_granted` - every criterion was judged and the feed did not qualify. + * `unknown` - at least one criterion has never produced a verdict, so whether the + feed qualifies cannot be decided yet. The others may all pass. + * `never_evaluated` - no criterion has ever produced a verdict for this feed. + + Derived from the `criteria` below, so it never disagrees with them. It is not on the + embedded `FeedReliabilitySummary`, which is served without the per-criterion rows in + search results - a field that could only ever be filled in on some responses would be + worse than not having one. + type: string + enum: + - granted + - not_granted + - unknown + - never_evaluated + example: granted earned_at: description: When the feed most recently earned the seal, in ISO 8601 date-time format. type: string diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 5febae2de..ed8ec2b42 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -1211,6 +1211,26 @@ components: description: Whether the feed currently holds the Seal of Reliability. type: boolean example: false + seal_status: + description: > + Why the feed does or does not hold the seal. `has_seal` is true exactly when this is `granted`; the other three values are all `has_seal: false` and are not the same thing: + + + * `granted` - every criterion in scope is a confirmed pass and none is on probation. + * `not_granted` - every criterion was judged and the feed did not qualify. + * `unknown` - at least one criterion has never produced a verdict, so whether the + feed qualifies cannot be decided yet. The others may all pass. + * `never_evaluated` - no criterion has ever produced a verdict for this feed. + + Derived from the `criteria` below, so it never disagrees with them. It is not on the embedded `FeedReliabilitySummary`, which is served without the per-criterion rows in search results - a field that could only ever be filled in on some responses would be worse than not having one. + + type: string + enum: + - granted + - not_granted + - unknown + - never_evaluated + example: granted earned_at: description: When the feed most recently earned the seal, in ISO 8601 date-time format. type: string @@ -2270,6 +2290,7 @@ components: The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2396,6 +2417,7 @@ components: The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2497,6 +2519,7 @@ components: Describes status of the Feed. Should be one of + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2516,6 +2539,7 @@ components: Describes data type of a feed. Should be one of + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index 71380816d..0e4a074c9 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -34,7 +34,33 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsdataset, Gtfsfeed +from shared.common.seal_criteria import AVAILABILITY_LOOKBACK +from shared.database_gen.sqlacodegen_models import ( + Feed, + GtfsFeedAvailabilityCheck, + Gtfsdataset, + Gtfsfeed, + Validationreport, + t_validationreportgtfsdataset, +) + + +@dataclass(frozen=True) +class AvailabilityCheck: + """The latest availability check for a feed within the run's window.""" + + checked_at: datetime + success: bool + + +@dataclass(frozen=True) +class ValidationReport: + """The latest validation report of the feed's latest dataset, as of the run's `now`.""" + + report_id: str + dataset_id: str + validated_at: datetime + total_error: Optional[int] = None @dataclass(frozen=True) @@ -72,6 +98,12 @@ class FeedSealContext: # The feed's latest dataset as of `now` - resolved by `downloaded_at` vs `now` latest_dataset: Optional[LatestDataset] = None + # Available: the latest availability check in the window this run covers. + availability_check: Optional[AvailabilityCheck] = None + + # Compliant: the latest validation report of the dataset in `latest_dataset`. + latest_validation_report: Optional[ValidationReport] = None + # Feeds in these statuses, or not published, are never eligible for the seal. # `inactive` and `future` feeds are deliberately kept eligible. @@ -195,6 +227,94 @@ def _load_latest_datasets( } +def _load_validation_reports( + db_session: Session, + latest_datasets: Dict[str, LatestDataset], + now: datetime, +) -> Dict[str, ValidationReport]: + """feed_id -> the latest validation report of that feed's latest dataset, as of `now`. + + Scoped to the latest dataset, not the feed: a verdict on a superseded dataset does not describe + what is being served. A feed whose latest dataset is not validated yet is left out, which the + evaluator reads as UNKNOWN. One dataset can have several reports (a re-validation); the most + recently validated wins, and ones with no `validated_at` are excluded. + """ + if not latest_datasets: + return {} + feed_id_by_dataset = { + dataset.dataset_id: feed_id for feed_id, dataset in latest_datasets.items() + } + join_table = t_validationreportgtfsdataset + rows = db_session.execute( + select( + join_table.c.dataset_id, + Validationreport.id, + Validationreport.validated_at, + Validationreport.total_error, + ) + .select_from(join_table) + .join( + Validationreport, + Validationreport.id == join_table.c.validation_report_id, + ) + .where( + join_table.c.dataset_id.in_(list(feed_id_by_dataset)), + Validationreport.validated_at.is_not(None), + Validationreport.validated_at <= now, + ) + .distinct(join_table.c.dataset_id) + .order_by( + join_table.c.dataset_id, + Validationreport.validated_at.desc(), + Validationreport.id.desc(), + ) + ).all() + return { + feed_id_by_dataset[row.dataset_id]: ValidationReport( + report_id=row.id, + dataset_id=row.dataset_id, + validated_at=row.validated_at, + total_error=row.total_error, + ) + for row in rows + } + + +def _load_availability( + db_session: Session, feed_ids: Sequence[str], now: datetime +) -> Dict[str, AvailabilityCheck]: + """feed_id -> its latest availability check in the 24 hours up to `now`. + + A rolling window rather than the UTC day of `now`, so a check still counts when the + availability job (02:00 UTC) and the seal run (04:00 UTC) drift apart or one of them runs + late. + """ + if not feed_ids: + return {} + rows = db_session.execute( + select( + GtfsFeedAvailabilityCheck.feed_id, + GtfsFeedAvailabilityCheck.checked_at, + GtfsFeedAvailabilityCheck.success, + ) + .where( + GtfsFeedAvailabilityCheck.feed_id.in_(list(feed_ids)), + GtfsFeedAvailabilityCheck.checked_at > now - AVAILABILITY_LOOKBACK, + GtfsFeedAvailabilityCheck.checked_at <= now, + ) + .distinct(GtfsFeedAvailabilityCheck.feed_id) + .order_by( + GtfsFeedAvailabilityCheck.feed_id, + GtfsFeedAvailabilityCheck.checked_at.desc(), + GtfsFeedAvailabilityCheck.id.desc(), + ) + ).all() + return { + row.feed_id: AvailabilityCheck(checked_at=row.checked_at, success=row.success) + for row in rows + } + + def build_contexts( db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime ) -> Dict[str, FeedSealContext]: @@ -229,9 +349,10 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] called once as `availability = _load_availability_today(...)` and consumed per feed as `availability_success_today=availability.get(feed.id, False)`. """ - latest_datasets = _load_latest_datasets( - db_session, [feed.id for feed in feeds], now - ) + feed_ids = [feed.id for feed in feeds] + latest_datasets = _load_latest_datasets(db_session, feed_ids, now) + availability = _load_availability(db_session, feed_ids, now) + validation_reports = _load_validation_reports(db_session, latest_datasets, now) return { feed.id: FeedSealContext( @@ -243,6 +364,8 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] seasonal=feed.seasonal, feed_created_at=feed.created_at, latest_dataset=latest_datasets.get(feed.id), + availability_check=availability.get(feed.id), + latest_validation_report=validation_reports.get(feed.id), ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 349f33311..c3004c72a 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -15,9 +15,8 @@ # """The seal criterion evaluators. -`EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable and Fresh / -future coverage (issue #1784) are implemented; Available and Compliant are the rest of -#1784, and Fresh / continuous coverage is tracked by #1782. Adding one means a new subclass, +`EVALUATORS` is the registry the job iterates. All criteria are implemented except Fresh / continuous +coverage, which is tracked by #1782. Adding one means a new subclass, an entry here, and whatever fields it needs on `FeedSealContext`. Its windows are not declared on the subclass: they come from the policy maps in `shared.common.seal_criteria`, which the read API reads too. @@ -32,6 +31,8 @@ CriterionEvaluator, CriterionObservation, ) +from tasks.seal_of_reliability.evaluators.available import AvailableEvaluator +from tasks.seal_of_reliability.evaluators.compliant import CompliantEvaluator from tasks.seal_of_reliability.evaluators.fresh_coverage import FreshCoverageEvaluator from tasks.seal_of_reliability.evaluators.official import OfficialEvaluator from tasks.seal_of_reliability.evaluators.stable import StableEvaluator @@ -39,11 +40,15 @@ EVALUATORS: Final[List[CriterionEvaluator]] = [ OfficialEvaluator(), StableEvaluator(), + AvailableEvaluator(), + CompliantEvaluator(), FreshCoverageEvaluator(), ] __all__ = [ "EVALUATORS", + "AvailableEvaluator", + "CompliantEvaluator", "CriterionEvaluator", "CriterionObservation", "FreshCoverageEvaluator", diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py new file mode 100644 index 000000000..3edb8e121 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/available.py @@ -0,0 +1,47 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Available criterion: the feed's producer URL answered today.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + AVAILABILITY_LOOKBACK, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class AvailableEvaluator(CriterionEvaluator): + """The latest `gtfs_feed_availability_check` since the previous run has `success = TRUE`. + + The window runs from the last time this criterion was evaluated for the feed up to + `now`. A window with no check at all is UNKNOWN, not a failure: it means we + did not look, not that the feed was down. + """ + + name = SealCriterionName.AVAILABLE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + check = ctx.availability_check + if check is None: + since = (ctx.now - AVAILABILITY_LOOKBACK).isoformat() + return CriterionStatus.UNKNOWN, f"no availability check since {since}" + + status = CriterionStatus.PASS if check.success else CriterionStatus.FAIL + outcome = "succeeded" if check.success else "failed" + return status, f"the check at {check.checked_at.isoformat()} {outcome}" diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py new file mode 100644 index 000000000..74307599a --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/compliant.py @@ -0,0 +1,57 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Compliant criterion: the latest dataset validates with no errors.""" + +from typing import Tuple + +from shared.common.seal_criteria import CriterionStatus, SealCriterionName +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class CompliantEvaluator(CriterionEvaluator): + """`total_error = 0` on the latest validation report of the feed's latest dataset. + + A dataset with no report yet - unvalidated, or validation lagging publication - is UNKNOWN, + which freezes the criterion at its last confirmed verdict rather than failing it. So is a feed + with no dataset: a missing report is not a clean bill of health, nor evidence of one. + """ + + name = SealCriterionName.COMPLIANT + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + if ctx.latest_dataset is None: + return CriterionStatus.UNKNOWN, "the feed has no dataset" + + report = ctx.latest_validation_report + if report is None: + return ( + CriterionStatus.UNKNOWN, + f"dataset {ctx.latest_dataset.dataset_id} has no validation report", + ) + + if report.total_error is None: + return ( + CriterionStatus.UNKNOWN, + f"validation report {report.report_id} has no total_error", + ) + + validated = ( + f"dataset {report.dataset_id}, validated {report.validated_at.isoformat()}" + ) + if report.total_error == 0: + return CriterionStatus.PASS, f"no errors ({validated})" + return CriterionStatus.FAIL, f"{report.total_error} error(s) ({validated})" diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 37807c1d2..e942239db 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -41,6 +41,8 @@ CriterionPhase, CriterionStatus, SealCriterionName, + SealStatus, + roll_up_seal_status, ) from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( @@ -169,36 +171,15 @@ def _load_previous_seals( return {row.feed_id: bool(row.has_seal) for row in rows} -def _roll_up_has_seal(states: Dict[str, SealCriterionState]) -> bool: - """True when every criterion in service is a confirmed pass and not on probation. +def _roll_up_seal_status(states: Dict[str, SealCriterionState]) -> SealStatus: + """The feed-level seal outcome from its criteria. - A criterion is *in service* when its `confirmed_status` is a verdict. `confirmed_status` - is only ever PASS, FAIL, NEVER_EVALUATED or NOT_APPLICABLE — never UNKNOWN, since an - unevaluable run leaves the stored value alone rather than writing UNKNOWN into it (see - `transition`). So the roll-up only has to skip the two non-verdict values it can see: - - * NEVER_EVALUATED — never produced a verdict, so it is skipped rather than counted as a - failure. This is what lets the seal be computed before every criterion has a data - source: one whose source starts collecting later simply is not part of the roll-up. - * NOT_APPLICABLE — deliberately excluded for this feed, so it is skipped too. This is why - a seasonal feed is not denied the seal by a criterion that is meaningless for it. - - An unevaluable run (UNKNOWN) does not appear here at all: `transition` has already frozen the - criterion at its last verdict, so it stays in the roll-up with that verdict if it had - one, or stays NEVER_EVALUATED and skipped if it never did. - - A criterion IN_GRACE_PERIOD is a confirmed pass and holds the seal. - One ON_PROBATION denies it. + A thin adapter over `roll_up_seal_status`, which owns the rule and is shared with the read API. + All this adds is reading `on_probation` off the state the way the job derives it, via `phase`. """ - in_service = [ - state for state in states.values() if state.confirmed_status.is_verdict - ] - if not in_service: - return False - return all( - state.confirmed_status is CriterionStatus.PASS - and phase(state) is not CriterionPhase.ON_PROBATION - for state in in_service + return roll_up_seal_status( + (state.confirmed_status, phase(state) is CriterionPhase.ON_PROBATION) + for state in states.values() ) @@ -452,6 +433,18 @@ def update_seals( ): first_evaluations += 1 + logging.info( + "Seal criterion evaluated: feed=%s criterion=%s observed=%s " + "confirmed=%s (was %s) phase=%s reason=%s", + ctx.stable_id, + evaluator.name.value, + observation.observed_status.value, + state.confirmed_status.value, + previous.confirmed_status.value if previous is not None else None, + phase(state).value, + observation.reason, + ) + criteria_report.append( { "criterion": evaluator.name.value, @@ -486,12 +479,20 @@ def update_seals( # A feed with no seal row yet is treated as not holding one, so a first run can # grant the seal but can never withdraw one: nothing was held to lose. had_seal = previous_seals.get(feed.id) - has_seal = _roll_up_has_seal(merged) + # Not stored: `seal_status` is derived from the same seal_criterion rows the read + # API derives it from (see `roll_up_seal_status`), so a column would be a second copy + # to keep in step. The job computes it to report and log the outcome, and to answer + # the one question feed_reliability_seal does store. + seal_status = _roll_up_seal_status(merged) + # The boolean stays the narrow question it always was: only GRANTED holds the seal, + # so unknown and never-evaluated read as `false` to everything already consuming it. + has_seal = seal_status is SealStatus.GRANTED outcome = { "feed_id": feed.id, "stable_id": ctx.stable_id, "had_seal": bool(had_seal), "has_seal": has_seal, + "seal_status": seal_status, # A first evaluation is a grant if it passes, but it is not a loss if # it fails: nothing was held, so nothing was lost. Only these two # flags stamp seal_earned_at / seal_lost_at. @@ -501,11 +502,20 @@ def update_seals( outcomes.append(outcome) # Every requested feed is reported, capped at max_reported_feeds below. + logging.info( + "Seal rolled up: feed=%s status=%s has_seal=%s (had_seal=%s)", + ctx.stable_id, + seal_status.value, + has_seal, + outcome["had_seal"], + ) + feed_reports.append( { "stable_id": ctx.stable_id, "had_seal": outcome["had_seal"], "has_seal": has_seal, + "seal_status": seal_status.value, "criteria": criteria_report, } ) @@ -548,6 +558,14 @@ def update_seals( "seals_after_run": sum(1 for outcome in outcomes if outcome["has_seal"]), "seals_granted": len(granted), "seals_revoked": len(revoked), + # The four-way outcome behind `seals_after_run`: which feeds were judged and did not + # qualify, and which could not be judged because a criterion has never had a verdict. + "seal_status_counts": { + status.value: sum( + 1 for outcome in outcomes if outcome["seal_status"] is status + ) + for status in SealStatus + }, # The two transitions in feed_reliability_seal, by feed. Counts alone cannot say # which feed moved, and that is the first thing anyone asks of a run. "granted_stable_ids": [outcome["stable_id"] for outcome in granted], diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py index 1b49964c3..335fa2e6a 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py @@ -52,8 +52,11 @@ Feed, FeedReliabilitySeal, Gtfsdataset, + GtfsFeedAvailabilityCheck, Gtfsfeed, SealCriterion, + Validationreport, + t_validationreportgtfsdataset, ) from test_shared.test_utils.database_utils import default_db_url @@ -69,8 +72,11 @@ # The task always runs against an explicit feed list. These are the eligible seeded feeds. REQUESTED = [OFFICIAL, NOT_OFFICIAL, UNKNOWN_OFFICIAL] -# Every seeded feed gets a dataset covering the next 400 days, so Fresh passes and `official` -# stays the only criterion that separates the feeds. Stable needs nothing seeded: it reads +# Every seeded feed gets a dataset covering the next 400 days, a successful availability check +# and a clean validation report of that dataset, so Fresh, Available and Compliant all pass and +# `official` stays the only criterion that separates the feeds. Feeding all four matters beyond +# tidiness: a criterion with no verdict at all makes the whole seal `unknown`, so a feed missing +# one of these inputs would never be granted or revoked. Stable needs nothing seeded: it reads # `feed.created_at`, which `_seed` already backdates by 400 days. COVERAGE_END = NOW + timedelta(days=400) @@ -112,9 +118,53 @@ def _seed(db_session, feed_id, official=True, status="active", operational="publ ) db_session.flush() + # Available's input. + db_session.add( + GtfsFeedAvailabilityCheck( + feed_id=feed_id, + checked_at=NOW - timedelta(hours=2), + request_url=f"https://example.com/{feed_id}.zip", + request_type="http_head", + status_code=200, + success=True, + ) + ) + db_session.flush() + + # Compliant's input: a clean report of the dataset seeded just above. + report_id = f"{dataset_id}_report" + db_session.add( + Validationreport( + id=report_id, + validator_version="1.0.0", + validated_at=NOW - timedelta(hours=1), + total_error=0, + ) + ) + db_session.flush() + db_session.execute( + t_validationreportgtfsdataset.insert().values( + dataset_id=dataset_id, validation_report_id=report_id + ) + ) + db_session.flush() + def _cleanup(db_session): - """Deleting the parent Feed cascades to gtfsfeed and both seal tables.""" + """Deleting the parent Feed cascades to gtfsfeed, its datasets and both seal tables. + + Validation reports are not owned by the feed - they hang off the dataset through + validationreportgtfsdataset - so they and their link rows are removed by hand, dependency + first, before the ids are reused by the next test. + """ + db_session.execute( + t_validationreportgtfsdataset.delete().where( + t_validationreportgtfsdataset.c.validation_report_id.like(f"{PREFIX}%") + ) + ) + db_session.execute( + delete(Validationreport).where(Validationreport.id.like(f"{PREFIX}%")) + ) db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) db_session.commit() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index a37940102..2e8bbc5fc 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -25,9 +25,16 @@ CriterionStatus, SealCriterionName, ) -from tasks.seal_of_reliability.context import FeedSealContext, LatestDataset +from tasks.seal_of_reliability.context import ( + AvailabilityCheck, + FeedSealContext, + LatestDataset, + ValidationReport, +) from tasks.seal_of_reliability.evaluators import ( EVALUATORS, + AvailableEvaluator, + CompliantEvaluator, CriterionEvaluator, FreshCoverageEvaluator, OfficialEvaluator, @@ -332,5 +339,141 @@ def test_has_a_grace_period_and_serves_probation(self): self.assertEqual(FreshCoverageEvaluator().probation_period, PROBATION_PERIOD) +class TestAvailable(unittest.TestCase): + """The latest availability check in the window since the previous evaluation.""" + + @staticmethod + def _check(success, checked_at=None): + return AvailabilityCheck(checked_at=checked_at or NOW, success=success) + + def test_a_successful_check_passes(self): + self.assertIs( + AvailableEvaluator() + .evaluate(_ctx(availability_check=self._check(True))) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_failed_check_fails(self): + result = AvailableEvaluator().evaluate( + _ctx(availability_check=self._check(False)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("failed", result.reason) + + def test_no_check_in_the_window_is_unknown_not_a_failure(self): + """A window the availability job did not cover says nothing about the producer.""" + result = AvailableEvaluator().evaluate(_ctx(availability_check=None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no availability check since", result.reason) + + def test_the_reason_names_the_check_it_read(self): + """The window makes "which check decided this" a real question, so answer it.""" + checked_at = NOW - timedelta(hours=3) + result = AvailableEvaluator().evaluate( + _ctx(availability_check=self._check(False, checked_at)) + ) + self.assertIn(checked_at.isoformat(), result.reason) + + def test_is_never_not_applicable(self): + """Availability applies to every feed, seasonal ones included.""" + for check in (self._check(True), self._check(False), None): + with self.subTest(check=check): + self.assertIsNot( + AvailableEvaluator() + .evaluate(_ctx(availability_check=check, seasonal=True)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_has_a_grace_period_and_serves_probation(self): + self.assertEqual(AvailableEvaluator().grace_period, timedelta(days=14)) + self.assertEqual(AvailableEvaluator().probation_period, PROBATION_PERIOD) + + +class TestCompliant(unittest.TestCase): + """`total_error = 0` on the latest validation report of the feed's latest dataset.""" + + DATASET_ID = "mdb-1-202605280000" + + def _compliant_ctx( + self, total_error=0, with_report=True, with_dataset=True, **overrides + ): + report = ( + ValidationReport( + report_id="report-1", + dataset_id=self.DATASET_ID, + validated_at=NOW - timedelta(hours=1), + total_error=total_error, + ) + if with_report + else None + ) + dataset = ( + LatestDataset( + dataset_id=self.DATASET_ID, + downloaded_at=NOW - timedelta(hours=2), + ) + if with_dataset + else None + ) + defaults = {"latest_validation_report": report, "latest_dataset": dataset} + defaults.update(overrides) + return _ctx(**defaults) + + def test_a_clean_report_passes(self): + self.assertIs( + CompliantEvaluator().evaluate(self._compliant_ctx(0)).observed_status, + CriterionStatus.PASS, + ) + + def test_any_error_fails(self): + result = CompliantEvaluator().evaluate(self._compliant_ctx(1)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("1 error(s)", result.reason) + + def test_a_feed_with_no_dataset_is_unknown(self): + """Nothing published means nothing to validate, not a failure.""" + result = CompliantEvaluator().evaluate( + self._compliant_ctx(with_report=False, with_dataset=False) + ) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no dataset", result.reason) + + def test_an_unvalidated_latest_dataset_is_unknown(self): + """The case that keeps a never-validated feed off a confirmed failure. + + Validation lags publication, so a feed publishing faster than the validator sits at + UNKNOWN and keeps whatever verdict it last earned. + """ + result = CompliantEvaluator().evaluate(self._compliant_ctx(with_report=False)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no validation report", result.reason) + self.assertIn(self.DATASET_ID, result.reason) + + def test_the_reason_names_the_dataset_that_was_validated(self): + result = CompliantEvaluator().evaluate(self._compliant_ctx(3)) + self.assertIn(self.DATASET_ID, result.reason) + + def test_a_report_with_no_error_count_is_unknown_not_a_pass(self): + """total_error is nullable, and a missing count must not read as zero errors.""" + result = CompliantEvaluator().evaluate(self._compliant_ctx(None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no total_error", result.reason) + + def test_is_never_not_applicable(self): + """Compliance applies to every feed, seasonal ones included.""" + self.assertIsNot( + CompliantEvaluator() + .evaluate(self._compliant_ctx(0, seasonal=True)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_has_a_grace_period_and_serves_probation(self): + self.assertEqual(CompliantEvaluator().grace_period, timedelta(days=30)) + self.assertEqual(CompliantEvaluator().probation_period, PROBATION_PERIOD) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index 70af32f04..db414c708 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -21,9 +21,13 @@ from unittest.mock import MagicMock, patch from shared.common.seal_criteria import ( + AVAILABILITY_LOOKBACK, PROBATION_PERIOD, + PROBATION_EXEMPT_CRITERIA, CriterionStatus, SealCriterionName, + SealStatus, + roll_up_seal_status, ) from tasks.seal_of_reliability.context import ( build_contexts, @@ -44,10 +48,13 @@ from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + GtfsFeedAvailabilityCheck, Gtfsdataset, Gtfsfeed, SealCriterion, SealCriterionSnapshot, + Validationreport, + t_validationreportgtfsdataset, ) from test_shared.test_utils.database_utils import default_db_url @@ -79,7 +86,8 @@ class _StandInEvaluator(CriterionEvaluator): It reads `official` so a test can drive it with the existing `set_official` helper, but unlike Official it debounces failures and serves probation afterwards. It borrows the - `available` enum value, which has no evaluator of its own yet (#1784). + `available` enum value and is patched over the whole registry, so the real + AvailableEvaluator never runs alongside it. """ name = SealCriterionName.AVAILABLE @@ -105,7 +113,7 @@ def _evaluate(self, ctx): class _GoesDarkEvaluator(CriterionEvaluator): - """A criterion that loses its upstream input partway through, standing in for #1784. + """A criterion that loses its upstream input partway through. It returns no verdict from `DARK_FROM` onwards, keyed on the clock rather than on `official` so that a test can drive it and Official in opposite directions at the same @@ -158,6 +166,25 @@ def _evaluate(self, ctx): STOPS_APPLYING = [OfficialEvaluator(), _StopsApplyingEvaluator()] +class _NeverAnswersEvaluator(CriterionEvaluator): + """A criterion whose input never arrives, so it never produces a verdict. + + Stands in for a criterion whose data source has not started collecting yet. Its + `confirmed_status` therefore stays NEVER_EVALUATED for good, which is the input the + feed-level UNKNOWN roll-up is about. + """ + + name = SealCriterionName.AVAILABLE + grace_period = None + + def _evaluate(self, ctx): + return CriterionStatus.UNKNOWN, "stand-in never has an input" + + +NEVER_ANSWERS = [_NeverAnswersEvaluator()] +OFFICIAL_AND_NEVER_ANSWERS = [OfficialEvaluator(), _NeverAnswersEvaluator()] + + def _seed_feed( db_session, feed_id: str, @@ -215,6 +242,43 @@ def _seed_dataset( db_session.commit() +def _seed_availability_check(db_session, feed_id: str, success, checked_at=None): + """One `gtfs_feed_availability_check` row for the feed.""" + db_session.add( + GtfsFeedAvailabilityCheck( + feed_id=feed_id, + checked_at=checked_at or NOW, + request_url=f"https://example.com/{feed_id}.zip", + request_type="http_head", + status_code=200 if success else 503, + success=success, + ) + ) + db_session.commit() + + +def _seed_validation_report( + db_session, dataset_id: str, total_error, validated_at=None, suffix="" +): + """A validation report for the dataset, linked through validationreportgtfsdataset.""" + report_id = f"{dataset_id}_report{suffix}" + db_session.add( + Validationreport( + id=report_id, + validator_version=f"1.0.0{suffix}", + validated_at=validated_at or NOW - timedelta(hours=1), + total_error=total_error, + ) + ) + db_session.flush() + db_session.execute( + t_validationreportgtfsdataset.insert().values( + dataset_id=dataset_id, validation_report_id=report_id + ) + ) + db_session.commit() + + def _set_seasonal(db_session, feed_id: str, seasonal): db_session.execute( Feed.__table__.update() @@ -242,6 +306,12 @@ def _cleanup(db_session): tables, all of which are ON DELETE CASCADE. """ db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) + # validationreport is not reachable by cascade from feed: deleting the feed cascades to + # gtfsdataset and to the validationreportgtfsdataset join row, but leaves the report + # itself orphaned, and the next test collides on its primary key. + db_session.execute( + delete(Validationreport).where(Validationreport.id.like(f"{PREFIX}%")) + ) db_session.commit() @@ -294,6 +364,21 @@ def seal_row(feed_id, db_session): select(table).where(table.c.feed_id == feed_id) ).first() + def derived_seal_status(self, feed_id): + """The feed's seal status, derived from the persisted rows the way the read API does. + + Nothing stores it, so an assertion has to re-derive it - and doing so from the rows the run + wrote checks what the API will actually see. + """ + return roll_up_seal_status( + ( + CriterionStatus(row.confirmed_status), + row.probation_start is not None + and row.criterion not in PROBATION_EXEMPT_CRITERIA, + ) + for row in self.criterion_rows(feed_id).values() + ).value + @staticmethod @with_db_session(db_url=default_db_url) def set_official(feed_id, official, db_session): @@ -321,6 +406,20 @@ def seed_dataset( ): _seed_dataset(db_session, feed_id, coverage_end, downloaded_at, suffix) + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_availability_check(feed_id, success, checked_at=None, db_session=None): + _seed_availability_check(db_session, feed_id, success, checked_at) + + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_validation_report( + dataset_id, total_error, validated_at=None, suffix="", db_session=None + ): + _seed_validation_report( + db_session, dataset_id, total_error, validated_at, suffix + ) + @staticmethod @with_db_session(db_url=default_db_url) def set_feed_created_at(feed_id, created_at, db_session): @@ -481,6 +580,162 @@ def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): "by then it had", ) + @with_db_session(db_url=default_db_url) + def test_a_check_inside_the_rolling_window_counts(self, db_session): + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(hours=6)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertTrue(ctx.availability_check.success) + + @with_db_session(db_url=default_db_url) + def test_the_window_is_exactly_the_lookback(self, db_session): + """A check just inside the window counts; the same check an hour older does not.""" + _seed_availability_check( + db_session, + TRACKED, + True, + NOW - AVAILABILITY_LOOKBACK + timedelta(minutes=1), + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + self.assertIsNotNone( + build_contexts(db_session, feeds, NOW)[feeds[0].id].availability_check + ) + self.assertIsNone( + build_contexts(db_session, feeds, NOW + timedelta(hours=1))[ + feeds[0].id + ].availability_check + ) + + @with_db_session(db_url=default_db_url) + def test_a_check_older_than_the_fallback_window_is_ignored(self, db_session): + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(days=3)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.availability_check) + + @with_db_session(db_url=default_db_url) + def test_a_check_after_now_is_not_visible_yet(self, db_session): + """Same replay rule as everywhere else: a run never reads its own future.""" + _seed_availability_check(db_session, TRACKED, True, NOW + timedelta(hours=1)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + self.assertIsNone( + build_contexts(db_session, feeds, NOW)[feeds[0].id].availability_check + ) + + @with_db_session(db_url=default_db_url) + def test_the_latest_check_in_the_window_decides(self, db_session): + """Not "any success": the most recent answer describes the feed now.""" + _seed_availability_check(db_session, TRACKED, True, NOW - timedelta(hours=5)) + _seed_availability_check(db_session, TRACKED, False, NOW - timedelta(hours=1)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertFalse(ctx.availability_check.success) + self.assertEqual(ctx.availability_check.checked_at, NOW - timedelta(hours=1)) + + @with_db_session(db_url=default_db_url) + def test_a_failed_check_is_a_verdict_not_a_missing_one(self, db_session): + """A check that ran and failed and no check at all are different answers.""" + _seed_availability_check(db_session, TRACKED, False, NOW) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.availability_check) + self.assertFalse(ctx.availability_check.success) + + @with_db_session(db_url=default_db_url) + def test_the_latest_validation_report_is_loaded(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + _seed_validation_report(db_session, f"{TRACKED}_dataset", total_error=3) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.latest_validation_report.total_error, 3) + + @with_db_session(db_url=default_db_url) + def test_a_feed_with_no_report_carries_none(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.latest_dataset, "the dataset is there ...") + self.assertIsNone(ctx.latest_validation_report, "... a report is not") + + @with_db_session(db_url=default_db_url) + def test_a_report_on_a_superseded_dataset_is_not_loaded(self, db_session): + """The report is the latest dataset's, so an older dataset's report is not it. + + Validation lags publication, so the newest dataset may have no report yet. That is + left as "no report" rather than backfilled from the dataset before it: the criterion + is about the data being served now. + """ + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=90), + downloaded_at=NOW - timedelta(days=5), + suffix="_older", + ) + _seed_validation_report( + db_session, f"{TRACKED}_dataset_older", total_error=0, suffix="_older" + ) + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=90), + downloaded_at=NOW - timedelta(hours=2), + suffix="_newest", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + + self.assertEqual(ctx.latest_dataset.dataset_id, f"{TRACKED}_dataset_newest") + self.assertIsNone( + ctx.latest_validation_report, + "the older dataset's report does not describe what is being served", + ) + + @with_db_session(db_url=default_db_url) + def test_the_validation_report_is_resolved_as_of_now(self, db_session): + """Same replay rule as the dataset: a later re-validation must not leak backwards.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + dataset_id = f"{TRACKED}_dataset" + _seed_validation_report( + db_session, + dataset_id, + 5, + validated_at=NOW - timedelta(days=2), + suffix="_old", + ) + _seed_validation_report( + db_session, + dataset_id, + 0, + validated_at=NOW + timedelta(days=2), + suffix="_new", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + + as_of_now = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual( + as_of_now.latest_validation_report.total_error, + 5, + "the re-validation had not happened yet", + ) + + later = NOW + timedelta(days=3) + as_of_later = build_contexts(db_session, feeds, later)[feeds[0].id] + self.assertEqual(as_of_later.latest_validation_report.total_error, 0) + + @with_db_session(db_url=default_db_url) + def test_a_report_with_no_validated_at_is_excluded(self, db_session): + """It cannot be placed in time, so it is dropped rather than guessed at.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + _seed_validation_report(db_session, f"{TRACKED}_dataset", total_error=0) + db_session.execute( + Validationreport.__table__.update().values(validated_at=None) + ) + db_session.commit() + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_validation_report) + @with_db_session(db_url=default_db_url) def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_session): """It is excluded rather than guessed at: we cannot say whether it existed yet.""" @@ -767,6 +1022,140 @@ def test_feeds_omitted_is_zero_when_nothing_was_dropped(self): self.assertEqual(report["feeds_omitted"], 0) +class TestSealStatusRollUp(SealDbTestCase): + """The four-way feed-level outcome, and the boolean that hangs off it. + + `has_seal` answers only "does the feed hold the seal", so all three non-granting values + read as false through it. `seal_status` is what tells them apart, and the distinction that + matters is between a feed that was judged and did not qualify and one that could not be + judged at all. + """ + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_every_criterion_passing_grants_the_seal(self): + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_a_judged_feed_that_does_not_qualify_is_not_granted(self): + """Not the same as unknown: every criterion answered, and the answer was no.""" + update_seals(dry_run=False, stable_feed_ids=[NOT_OFFICIAL], now=NOW) + + seal = self.seal_row(NOT_OFFICIAL) + self.assertEqual( + self.derived_seal_status(NOT_OFFICIAL), SealStatus.NOT_GRANTED.value + ) + self.assertFalse(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", NEVER_ANSWERS) + def test_no_criterion_ever_judged_is_never_evaluated(self): + """A row is written - the attempt happened - but nothing has been decided.""" + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + seal = self.seal_row(OFFICIAL) + self.assertEqual( + self.derived_seal_status(OFFICIAL), SealStatus.NEVER_EVALUATED.value + ) + self.assertFalse(seal.has_seal) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_one_criterion_without_a_verdict_makes_the_whole_seal_unknown(self): + """Official passes, but the other criterion has never been judged. + + The feed may well qualify - which is exactly why this is not NOT_GRANTED - but it + cannot be granted the seal on evidence covering only half its criteria. + """ + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + + rows = self.criterion_rows(OFFICIAL) + self.assertEqual( + rows[SealCriterionName.OFFICIAL.value].confirmed_status, + CriterionStatus.PASS.value, + ) + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.UNKNOWN.value) + self.assertFalse(seal.has_seal) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_a_transient_outage_does_not_make_a_judged_seal_unknown(self): + """The distinction the roll-up rests on: no verdict *ever*, not no verdict today. + + The first run is driven by the real registry, so both criteria reach a verdict. The + second patches one of them dark; it keeps its stored pass, and the seal stays granted. + """ + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [OfficialEvaluator(), _StandInEvaluator()], + ): + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=NOW) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=later) + + row = self.criterion_rows(OFFICIAL)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) + def test_the_report_counts_and_names_the_outcomes(self): + """The two decided values, from `official` alone across the seeded feeds.""" + report = update_seals(dry_run=True, stable_feed_ids=OURS, now=NOW) + + counts = report["seal_status_counts"] + self.assertEqual(set(counts), {status.value for status in SealStatus}) + self.assertEqual( + sum(counts.values()), + report["total_feeds"], + "every feed lands in exactly one", + ) + self.assertEqual( + {row["stable_id"]: row["seal_status"] for row in report["feeds"]}, + { + OFFICIAL: SealStatus.GRANTED.value, + INACTIVE: SealStatus.GRANTED.value, + NOT_OFFICIAL: SealStatus.NOT_GRANTED.value, + # `official IS NULL` is a verdict for Official, not an absent one. + UNKNOWN_OFFICIAL: SealStatus.NOT_GRANTED.value, + }, + ) + self.assertEqual(counts[SealStatus.GRANTED.value], 2) + + @patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS + ) + def test_an_unjudged_criterion_outranks_a_failing_one(self): + """A feed failing Official is still UNKNOWN, not NOT_GRANTED, while a criterion has + never been judged. + + Deliberate: the roll-up reports whether the feed could be judged before it reports + the verdict, so `unknown` always means the same thing - the evaluation is incomplete - + rather than meaning it only when nothing else had an opinion. + """ + update_seals(dry_run=False, stable_feed_ids=[NOT_OFFICIAL], now=NOW) + + rows = self.criterion_rows(NOT_OFFICIAL) + self.assertEqual( + rows[SealCriterionName.OFFICIAL.value].confirmed_status, + CriterionStatus.FAIL.value, + ) + seal = self.seal_row(NOT_OFFICIAL) + self.assertEqual( + self.derived_seal_status(NOT_OFFICIAL), SealStatus.UNKNOWN.value + ) + self.assertFalse(seal.has_seal) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestCriterionSnapshot(SealDbTestCase): """seal_criterion_snapshot, the per-day record of each criterion (issue #1809). @@ -896,21 +1285,243 @@ def test_every_criterion_is_written_for_every_feed(self): {evaluator.name.value for evaluator in EVALUATORS}, ) - def test_a_feed_meeting_all_three_earns_the_seal(self): + def satisfy_everything(self): + """Seed the inputs every implemented criterion needs, all passing.""" self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) + + def test_a_feed_meeting_every_criterion_earns_the_seal(self): + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertEqual( self.criteria_of(), { - SealCriterionName.OFFICIAL.value: CriterionStatus.PASS.value, - SealCriterionName.STABLE.value: CriterionStatus.PASS.value, - SealCriterionName.FRESH_COVERAGE.value: CriterionStatus.PASS.value, + evaluator.name.value: CriterionStatus.PASS.value + for evaluator in EVALUATORS }, ) self.assertTrue(self.seal_row(TRACKED).has_seal) + def test_criteria_with_no_data_leave_the_seal_unknown(self): + """Available and Compliant have never had a verdict, so the seal cannot be decided. + + They do not *deny* the seal - the feed may well qualify - but with two of five + criteria unjudged, saying it does not qualify would be as wrong as saying it does. + """ + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + rows = self.criterion_rows(TRACKED) + for criterion in ( + SealCriterionName.AVAILABLE.value, + SealCriterionName.COMPLIANT.value, + ): + with self.subTest(criterion=criterion): + self.assertEqual( + rows[criterion].observed_status, CriterionStatus.UNKNOWN.value + ) + self.assertEqual( + rows[criterion].confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + ) + seal = self.seal_row(TRACKED) + self.assertEqual(self.derived_seal_status(TRACKED), SealStatus.UNKNOWN.value) + self.assertFalse(seal.has_seal, "unknown is not a grant") + + def test_a_failed_availability_check_denies_the_seal(self): + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Next day: the only check that ran failed. + later = NOW + timedelta(days=1) + self.seed_availability_check(TRACKED, success=False, checked_at=later) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "Available has a 14-day grace period and had already passed", + ) + self.assertTrue(self.seal_row(TRACKED).has_seal, "still held, under grace") + + def test_a_recovery_later_in_the_window_wins(self): + """Several checks in one window: the most recent one is the verdict.""" + self.satisfy_everything() + later = NOW + timedelta(days=1) + self.seed_availability_check(TRACKED, success=False, checked_at=later) + self.seed_availability_check( + TRACKED, success=True, checked_at=later + timedelta(hours=2) + ) + + update_seals( + dry_run=False, + stable_feed_ids=[TRACKED], + now=later + timedelta(hours=3), + ) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + + def test_a_check_still_answers_a_second_run_inside_the_window(self): + """The window is a rolling 24h, not "checks this run has not seen yet".""" + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + # Six hours later, no new check: the earlier one is still inside the window. + later = NOW + timedelta(hours=6) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_the_criterion_goes_quiet_once_the_check_ages_out(self): + """A day with no check at all is UNKNOWN, which freezes the last verdict.""" + self.satisfy_everything() + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + stale = NOW + AVAILABILITY_LOOKBACK + timedelta(hours=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=stale) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "UNKNOWN freezes the criterion at its last verdict rather than failing it", + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_late_availability_job_is_picked_up_by_the_next_run(self): + """The reason for the window: a check the seal run missed is not lost. + + The seal runs, sees nothing; the availability job lands afterwards; the next seal run + still reads that check instead of it falling into a closed calendar day. + """ + self.seed_dataset(TRACKED, self.FAR_FUTURE) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + self.criterion_rows(TRACKED)[ + SealCriterionName.AVAILABLE.value + ].observed_status, + CriterionStatus.UNKNOWN.value, + ) + + self.seed_availability_check( + TRACKED, success=True, checked_at=NOW + timedelta(hours=1) + ) + later = NOW + timedelta(hours=2) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.AVAILABLE.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + + def test_validation_errors_deny_the_seal(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=7) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.FAIL.value, + "a first verdict gets no grace period", + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_dataset_with_no_report_is_unknown_not_compliant(self): + """A missing report is not a clean bill of health.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertIsNone(row.last_verdict_at) + + def test_a_feed_publishing_faster_than_validation_keeps_its_last_verdict(self): + """Daily publishing plus lagging validation, which is the common case. + + Each new dataset arrives unvalidated, so Compliant observes UNKNOWN on the days in + between. That freezes the criterion at the verdict it last earned instead of failing + it, so the feed keeps the seal while the validator catches up. + """ + self.seed_availability_check(TRACKED, success=True) + self.seed_dataset( + TRACKED, + self.FAR_FUTURE, + downloaded_at=NOW - timedelta(days=2), + suffix="_validated", + ) + self.seed_validation_report( + f"{TRACKED}_dataset_validated", total_error=0, suffix="_validated" + ) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + self.criterion_rows(TRACKED)[ + SealCriterionName.COMPLIANT.value + ].observed_status, + CriterionStatus.PASS.value, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Published since, and not validated yet. + later = NOW + timedelta(hours=6) + self.seed_dataset( + TRACKED, + self.FAR_FUTURE, + downloaded_at=NOW + timedelta(hours=1), + suffix="_fresh", + ) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "the verdict on the dataset we did validate still stands", + ) + seal = self.seal_row(TRACKED) + self.assertEqual(self.derived_seal_status(TRACKED), SealStatus.GRANTED.value) + self.assertTrue(seal.has_seal) + + def test_compliant_reads_the_latest_report_of_the_latest_dataset(self): + """Several validator versions run against one dataset; the newest one decides.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.seed_availability_check(TRACKED, success=True) + dataset_id = f"{TRACKED}_dataset" + self.seed_validation_report( + dataset_id, + total_error=9, + validated_at=NOW - timedelta(days=3), + suffix="_old", + ) + self.seed_validation_report( + dataset_id, + total_error=0, + validated_at=NOW - timedelta(hours=1), + suffix="_new", + ) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.COMPLIANT.value] + self.assertEqual(row.observed_status, CriterionStatus.PASS.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + def test_a_feed_new_to_the_database_cannot_hold_the_seal_yet(self): """Stable reads the feed's own age, so a freshly added feed fails it.""" self.set_feed_created_at(TRACKED, NOW - timedelta(days=30)) @@ -933,6 +1544,8 @@ def test_the_seal_arrives_once_the_feed_is_old_enough(self): """ self.set_feed_created_at(TRACKED, NOW) self.seed_dataset(TRACKED, NOW + timedelta(days=400)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertFalse( self.seal_row(TRACKED).has_seal, "in the database for zero days" @@ -949,7 +1562,7 @@ def test_the_seal_arrives_once_the_feed_is_old_enough(self): def test_an_old_feed_qualifies_on_its_very_first_run(self): """The point of reading `feed.created_at`: no six-month wait after deployment for a feed that has already been in the catalog for years.""" - self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -961,7 +1574,7 @@ def test_an_old_feed_qualifies_on_its_very_first_run(self): def test_an_unstable_producer_url_denies_the_seal_immediately(self): """Stable has no grace period, so the flag costs the seal the day it is set.""" - self.seed_dataset(TRACKED, self.FAR_FUTURE) + self.satisfy_everything() update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertTrue(self.seal_row(TRACKED).has_seal) @@ -975,8 +1588,8 @@ def test_an_unstable_producer_url_denies_the_seal_immediately(self): ) self.assertFalse(self.seal_row(TRACKED).has_seal) - def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): - """UNKNOWN is not a failure: the other two criteria still decide the seal.""" + def test_a_feed_with_no_dataset_leaves_the_seal_unknown(self): + """UNKNOWN is not a failure - but it is not a pass to be skipped over either.""" update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -988,10 +1601,13 @@ def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): "no verdict was ever produced, so the criterion is out of service", ) self.assertIsNone(row.last_verdict_at) - self.assertTrue( - self.seal_row(TRACKED).has_seal, - "Official and Stable carry it while Fresh has nothing to say", + seal = self.seal_row(TRACKED) + self.assertEqual( + self.derived_seal_status(TRACKED), + SealStatus.UNKNOWN.value, + "Official and Stable pass, but Fresh has never been judged", ) + self.assertFalse(seal.has_seal) def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): """Fresh has a 14-day grace period, but a criterion that has never passed has not @@ -1007,6 +1623,8 @@ def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): def test_the_grace_period_absorbs_a_lapse_on_a_feed_that_was_passing(self): self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertTrue(self.seal_row(TRACKED).has_seal) @@ -1046,6 +1664,7 @@ def test_a_seasonal_feed_is_not_denied_by_fresh(self): """NOT_APPLICABLE withdraws the criterion instead of failing it, which is the whole point of the value: a seasonal feed keeps the seal on the criteria that do apply. """ + self.satisfy_everything() self.set_seasonal(TRACKED, True) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) @@ -1058,6 +1677,8 @@ def test_a_seasonal_feed_is_not_denied_by_fresh(self): def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self): """A feed marked seasonal after a confirmed Fresh failure stops being judged on it.""" self.seed_dataset(TRACKED, NOW - timedelta(days=1)) + self.seed_availability_check(TRACKED, success=True) + self.seed_validation_report(f"{TRACKED}_dataset", total_error=0) update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) self.assertFalse(self.seal_row(TRACKED).has_seal) @@ -1072,7 +1693,7 @@ def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self) ) self.assertTrue(self.seal_row(TRACKED).has_seal) - def test_a_run_reports_all_three_criteria_with_their_reasons(self): + def test_a_run_reports_every_criterion_with_its_reason(self): self.seed_dataset(TRACKED, self.FAR_FUTURE) report = update_seals(dry_run=True, stable_feed_ids=[TRACKED], now=NOW) @@ -1398,11 +2019,12 @@ def test_going_dark_records_the_attempt_without_moving_the_verdict(self): frozen.last_verdict_at, self.FAILED_AT, "but got no new verdict" ) - def test_going_dark_before_any_verdict_leaves_the_criterion_out_of_service(self): + def test_going_dark_before_any_verdict_leaves_the_seal_unknown(self): """The other half: with no verdict ever, there is nothing to hold in service. A row is written, because the attempt is worth recording, but `confirmed_status` - stays NEVER_EVALUATED so the criterion is skipped rather than denying the seal. + stays NEVER_EVALUATED - and a criterion that has never been judged cannot be skipped + over, so the seal is UNKNOWN rather than granted on the strength of the others. """ self.run_at(DARK_FROM) @@ -1415,9 +2037,10 @@ def test_going_dark_before_any_verdict_leaves_the_criterion_out_of_service(self) "no verdict has ever been produced", ) self.assertIsNone(row.last_verdict_at) - self.assertTrue( - self.seal_row(OFFICIAL).has_seal, - "and the criterion is skipped rather than denying the seal", + seal = self.seal_row(OFFICIAL) + self.assertEqual(self.derived_seal_status(OFFICIAL), SealStatus.UNKNOWN.value) + self.assertFalse( + seal.has_seal, "unknown is not a grant, and not a denial either" ) From d67a4f3c3dc78ecbef457642dbb45925116b3001 Mon Sep 17 00:00:00 2001 From: cka-y Date: Thu, 27 Aug 2026 16:31:48 -0400 Subject: [PATCH 5/5] clean up + logic fix --- api/.openapi-generator/FILES | 22 --------------- api/src/shared/common/seal_criteria.py | 20 +++++++++----- .../db_models/feed_reliability_report_impl.py | 7 +---- docs/DatabaseCatalogAPI.yaml | 17 +++--------- docs/OperationsAPI.yaml | 27 ++++++++++++------- .../test_seal_updater_db.py | 22 +++++++++------ 6 files changed, 50 insertions(+), 65 deletions(-) diff --git a/api/.openapi-generator/FILES b/api/.openapi-generator/FILES index 74129a6b5..afcea0a9d 100644 --- a/api/.openapi-generator/FILES +++ b/api/.openapi-generator/FILES @@ -52,25 +52,3 @@ src/feeds_gen/models/search_feeds200_response.py src/feeds_gen/models/source_info.py src/feeds_gen/models/validation_report.py src/feeds_gen/security_api.py -src/user_service/impl/__init__.py -src/user_service_gen/apis/__init__.py -src/user_service_gen/apis/notifications_api.py -src/user_service_gen/apis/notifications_api_base.py -src/user_service_gen/apis/subscriptions_api.py -src/user_service_gen/apis/subscriptions_api_base.py -src/user_service_gen/apis/users_api.py -src/user_service_gen/apis/users_api_base.py -src/user_service_gen/main.py -src/user_service_gen/models/__init__.py -src/user_service_gen/models/create_notification_subscription_request.py -src/user_service_gen/models/extra_models.py -src/user_service_gen/models/feature_flag.py -src/user_service_gen/models/feed_subscription_summary.py -src/user_service_gen/models/notification_subscription.py -src/user_service_gen/models/notification_type.py -src/user_service_gen/models/subscription_feed.py -src/user_service_gen/models/subscription_feed_group.py -src/user_service_gen/models/update_notification_subscription_request.py -src/user_service_gen/models/update_user_request.py -src/user_service_gen/models/user_profile.py -src/user_service_gen/security_api.py diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index c0fbac829..220b934eb 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -130,9 +130,9 @@ def roll_up_seal_status( ) -> SealStatus: """The feed-level seal outcome from its criteria. * every criterion NEVER_EVALUATED -> seal NEVER_EVALUATED. - * any criterion NEVER_EVALUATED -> seal UNKNOWN. - * otherwise every criterion is a confirmed verdict: GRANTED when they all pass and none is on - probation, NOT_GRANTED otherwise. + * any criterion failing or on probation -> seal NOT_GRANTED, whatever the rest say. + * any remaining criterion NEVER_EVALUATED -> seal UNKNOWN. + * otherwise every criterion is a confirmed pass and none is on probation -> seal GRANTED. """ in_scope = [ (status, on_probation) for status, on_probation in criteria if status is not CriterionStatus.NOT_APPLICABLE @@ -145,12 +145,20 @@ def roll_up_seal_status( if unjudged == len(in_scope): # Every criterion that isn't NOT_APPLICABLE is NEVER_EVALUATED return SealStatus.NEVER_EVALUATED + + # One criterion is enough to deny the seal, so this is decidable even with the rest unjudged. + # A pass on probation denies it too: probation withholds the criterion whatever its status. + denied = any( + status is CriterionStatus.FAIL or (status is CriterionStatus.PASS and on_probation) + for status, on_probation in in_scope + ) + if denied: + return SealStatus.NOT_GRANTED if unjudged: - # At least one criterion is NEVER_EVALUATED + # Nothing denies the seal, but not everything has been judged: it cannot be granted yet. return SealStatus.UNKNOWN - granted = all(status is CriterionStatus.PASS and not on_probation for status, on_probation in in_scope) - return SealStatus.GRANTED if granted else SealStatus.NOT_GRANTED + return SealStatus.GRANTED # Stable: how long we must have been tracking a feed - measured from its diff --git a/api/src/shared/db_models/feed_reliability_report_impl.py b/api/src/shared/db_models/feed_reliability_report_impl.py index b46c1e0e0..3a6c77e7b 100644 --- a/api/src/shared/db_models/feed_reliability_report_impl.py +++ b/api/src/shared/db_models/feed_reliability_report_impl.py @@ -12,12 +12,7 @@ def _seal_status_of(criterion_rows: list[SealCriterionOrm]) -> str: - """The feed-level seal status derived from a feed's `seal_criterion` rows. - - Not stored: the rule lives in `shared.common.seal_criteria` and is shared with the nightly job, - so the status served and the one `has_seal` was decided by cannot drift apart. Probation is read - only for the criteria that serve it, matching `on_probation` below. - """ + """The feed-level seal status derived from a feed's `seal_criterion` rows.""" return roll_up_seal_status( ( CriterionStatus(row.confirmed_status), diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 1eff8d904..bd261877b 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -1121,19 +1121,10 @@ components: example: false seal_status: description: > - Why the feed does or does not hold the seal. `has_seal` is true exactly when this is - `granted`; the other three values are all `has_seal: false` and are not the same thing: - - * `granted` - every criterion in scope is a confirmed pass and none is on probation. - * `not_granted` - every criterion was judged and the feed did not qualify. - * `unknown` - at least one criterion has never produced a verdict, so whether the - feed qualifies cannot be decided yet. The others may all pass. - * `never_evaluated` - no criterion has ever produced a verdict for this feed. - - Derived from the `criteria` below, so it never disagrees with them. It is not on the - embedded `FeedReliabilitySummary`, which is served without the per-criterion rows in - search results - a field that could only ever be filled in on some responses would be - worse than not having one. + Descriptive status of the feed's seal. `has_seal` is true only when this is `granted`. + `not_granted`: at least one criterion is failing. `unknown`: none is failing, but not + every criterion has been evaluated yet. `never_evaluated`: none of the criteria has + been evaluated yet. type: string enum: - granted diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index ed8ec2b42..cf6d6bc1d 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -1213,16 +1213,7 @@ components: example: false seal_status: description: > - Why the feed does or does not hold the seal. `has_seal` is true exactly when this is `granted`; the other three values are all `has_seal: false` and are not the same thing: - - - * `granted` - every criterion in scope is a confirmed pass and none is on probation. - * `not_granted` - every criterion was judged and the feed did not qualify. - * `unknown` - at least one criterion has never produced a verdict, so whether the - feed qualifies cannot be decided yet. The others may all pass. - * `never_evaluated` - no criterion has ever produced a verdict for this feed. - - Derived from the `criteria` below, so it never disagrees with them. It is not on the embedded `FeedReliabilitySummary`, which is served without the per-criterion rows in search results - a field that could only ever be filled in on some responses would be worse than not having one. + Descriptive status of the feed's seal. `has_seal` is true only when this is `granted`. `not_granted`: at least one criterion is failing. `unknown`: none is failing, but not every criterion has been evaluated yet. `never_evaluated`: none of the criteria has been evaluated yet. type: string enum: @@ -2291,6 +2282,10 @@ components: + + + + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2418,6 +2413,10 @@ components: + + + + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2520,6 +2519,10 @@ components: + + + + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2540,6 +2543,10 @@ components: + + + + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index db414c708..d875c4b38 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -1134,13 +1134,14 @@ def test_the_report_counts_and_names_the_outcomes(self): @patch( "tasks.seal_of_reliability.seal_updater.EVALUATORS", OFFICIAL_AND_NEVER_ANSWERS ) - def test_an_unjudged_criterion_outranks_a_failing_one(self): - """A feed failing Official is still UNKNOWN, not NOT_GRANTED, while a criterion has - never been judged. - - Deliberate: the roll-up reports whether the feed could be judged before it reports - the verdict, so `unknown` always means the same thing - the evaluation is incomplete - - rather than meaning it only when nothing else had an opinion. + def test_a_failing_criterion_decides_the_seal_even_with_another_unjudged(self): + """A feed failing Official is NOT_GRANTED, not UNKNOWN, though a criterion has never + been judged. + + One failure is enough to deny the seal, so the outcome is decidable however little we + know about the rest: no verdict the missing criterion could produce would grant it. + UNKNOWN is reserved for the case where nothing is failing and the evaluation is simply + incomplete. """ update_seals(dry_run=False, stable_feed_ids=[NOT_OFFICIAL], now=NOW) @@ -1149,9 +1150,14 @@ def test_an_unjudged_criterion_outranks_a_failing_one(self): rows[SealCriterionName.OFFICIAL.value].confirmed_status, CriterionStatus.FAIL.value, ) + self.assertEqual( + rows[SealCriterionName.AVAILABLE.value].confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + "the other criterion has no verdict at all", + ) seal = self.seal_row(NOT_OFFICIAL) self.assertEqual( - self.derived_seal_status(NOT_OFFICIAL), SealStatus.UNKNOWN.value + self.derived_seal_status(NOT_OFFICIAL), SealStatus.NOT_GRANTED.value ) self.assertFalse(seal.has_seal)