diff --git a/api/src/shared/common/seal_criteria.py b/api/src/shared/common/seal_criteria.py index 2efbbfd17..9c8a14844 100644 --- a/api/src/shared/common/seal_criteria.py +++ b/api/src/shared/common/seal_criteria.py @@ -30,6 +30,54 @@ class SealCriterionName(str, Enum): FRESH_CONTINUOUS = "fresh_continuous" +class CriterionStatus(str, Enum): + """A criterion's status. Values match the `seal_criterion_status` DB enum. + + Only PASS and FAIL are verdicts. The other three say why there is no verdict, and they + are not interchangeable - the job's roll-up sends them in three different directions: + + * UNKNOWN - we could not look; the inputs the check needs were not there. A property of + the run, not of the feed. The criterion keeps its last confirmed verdict and stays in + the roll-up, so an upstream outage freezes a criterion rather than waiving it. + * NOT_APPLICABLE - there is no question to ask; the criterion is deliberately excluded + for this feed (Fresh / future coverage on a seasonal feed). A property of the feed. + The criterion leaves the roll-up entirely. + * NEVER_EVALUATED - never had a verdict, since the feed first appeared. The initial + value, and the only one the job never writes back once a criterion has left it. + + UNKNOWN is never written to `confirmed_status`: a run that could not look does not + change the answer, it leaves the previous one standing. + """ + + PASS = "pass" + FAIL = "fail" + UNKNOWN = "unknown" + NEVER_EVALUATED = "never_evaluated" + NOT_APPLICABLE = "not_applicable" + + @property + def is_verdict(self) -> bool: + """True for PASS and FAIL - the two values that mean the check actually answered.""" + return self in (CriterionStatus.PASS, CriterionStatus.FAIL) + + +class CriterionPhase(str, Enum): + """Which of the two debouncing mechanisms is currently acting on a criterion. + + Derived from the stored row rather than stored itself (see the job's + `state_machine.phase`): it is a pure function of `probation_start`, `confirmed_status` + and `first_observed_failure_at`, all of which are already on the row, so a stored copy + would be a second thing to keep in step for no gain. + + The three values are mutually exclusive: probation suspends the grace period, so a + criterion can never be serving a penalty and holding a failure under grace at once. + """ + + STEADY = "steady" + IN_GRACE_PERIOD = "in_grace_period" + ON_PROBATION = "on_probation" + + # How long a criterion may keep failing its own check before the failure is confirmed and # the seal is withdrawn. None means the status flips on the first failing day. GRACE_PERIODS: Final[Dict[SealCriterionName, Optional[timedelta]]] = { @@ -63,6 +111,33 @@ class SealCriterionName(str, Enum): ) +# Stable: how long we must have been tracking a feed - measured from its +# `feed.created_at` - before it can be called stable. +TRACKING_PERIOD: Final[timedelta] = timedelta(days=180) + +# Fresh / future coverage: how far ahead the latest dataset's service coverage must reach +FUTURE_COVERAGE_HORIZON: Final[timedelta] = timedelta(days=7) + + +def grace_period_for(criterion: str | SealCriterionName) -> Optional[timedelta]: + """How long an observed failure of `criterion` may run before it is confirmed. + + None means the criterion has no grace period and its status flips on the first failing + day. Callers ask for the window rather than declaring their own, so `GRACE_PERIODS` + stays the only place a value can change. + """ + return GRACE_PERIODS[resolve_criterion(criterion)] + + +def probation_period_for(criterion: str | SealCriterionName) -> Optional[timedelta]: + """How long `criterion` must go with no observed failure after a confirmed failure. + + None means the criterion never serves probation (`official` and `stable`, which are + point-in-time state checks). + """ + return PROBATION_PERIODS[resolve_criterion(criterion)] + + def resolve_criterion(criterion: str | SealCriterionName) -> SealCriterionName: """Coerce a stored criterion value to a `SealCriterionName`. diff --git a/api/src/shared/db_models/reliability_criterion_impl.py b/api/src/shared/db_models/reliability_criterion_impl.py index 66e15e34b..a83623c83 100644 --- a/api/src/shared/db_models/reliability_criterion_impl.py +++ b/api/src/shared/db_models/reliability_criterion_impl.py @@ -2,21 +2,18 @@ from feeds_gen.models.reliability_criterion import ReliabilityCriterion from shared.common.seal_criteria import ( - GRACE_PERIODS, - PROBATION_PERIODS, + CriterionStatus, SealCriterionName, + grace_period_for, + probation_period_for, resolve_criterion, window_end, ) from shared.database_gen.sqlacodegen_models import SealCriterion as SealCriterionOrm # The API `status` values are the `seal_criterion_status` DB enum verbatim, so a stored status is -# served as-is with no translation. -STATUS_PASS = "pass" -STATUS_FAIL = "fail" -STATUS_UNKNOWN = "unknown" -STATUS_NEVER_EVALUATED = "never_evaluated" -STATUS_NOT_APPLICABLE = "not_applicable" +# served as-is with no translation - `CriterionStatus` is a `str` enum over exactly those values, +# shared with the nightly job so the two cannot drift apart. class ReliabilityCriterionImpl(ReliabilityCriterion): @@ -42,7 +39,7 @@ def never_evaluated(cls, criterion: SealCriterionName) -> ReliabilityCriterion: """ return cls( criterion=criterion.value, - status=STATUS_NEVER_EVALUATED, + status=CriterionStatus.NEVER_EVALUATED.value, in_grace_period=False, on_probation=False, ) @@ -65,7 +62,7 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # `not_applicable` (withdrawn for this feed) and `never_evaluated` (no verdict ever) do not # participate in the seal, so they carry no grace period, no probation and no windows - just # the flat status, mirroring the row-less `never_evaluated` entry. - if status in (STATUS_NEVER_EVALUATED, STATUS_NOT_APPLICABLE): + if status in (CriterionStatus.NEVER_EVALUATED, CriterionStatus.NOT_APPLICABLE): return cls( criterion=criterion.value, status=status, @@ -76,8 +73,8 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # Criteria exempt from a window (`official` and `stable` from both, `fresh_continuous` from # grace) have their stored values ignored rather than trusted - the policy maps are the # authority on which criteria serve them. - grace_period = GRACE_PERIODS.get(criterion) - probation_period = PROBATION_PERIODS.get(criterion) + grace_period = grace_period_for(criterion) + probation_period = probation_period_for(criterion) probation_start = criterion_row.probation_start if probation_period else None on_probation = probation_start is not None @@ -87,8 +84,8 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio # nothing left for grace to protect. in_grace_period = ( grace_period is not None - and status == STATUS_FAIL - and criterion_row.confirmed_status == STATUS_PASS + and status == CriterionStatus.FAIL + and criterion_row.confirmed_status == CriterionStatus.PASS and not on_probation ) diff --git a/api/tests/unittest/models/test_feed_reliability_report_impl.py b/api/tests/unittest/models/test_feed_reliability_report_impl.py index b6470740f..95d7eaeb5 100644 --- a/api/tests/unittest/models/test_feed_reliability_report_impl.py +++ b/api/tests/unittest/models/test_feed_reliability_report_impl.py @@ -3,10 +3,9 @@ from types import SimpleNamespace from shared.common.error_handling import InternalHTTPException -from shared.common.seal_criteria import PROBATION_PERIOD, SealCriterionName +from shared.common.seal_criteria import PROBATION_PERIOD, CriterionStatus, SealCriterionName from shared.database_gen.sqlacodegen_models import FeedReliabilitySeal, SealCriterion from shared.db_models.feed_reliability_report_impl import FeedReliabilityReportImpl -from shared.db_models.reliability_criterion_impl import STATUS_FAIL, STATUS_NEVER_EVALUATED, STATUS_PASS # Anchored to the real clock because the countdowns are derived against `datetime.now`. Every window # below is at least a day clear of its boundary, so the assertions do not race the wall clock. @@ -68,7 +67,7 @@ def test_never_evaluated_feed(self): assert report.evaluated_at is None assert report.on_probation is False assert len(report.criteria) == 6 - assert all(criterion.status == STATUS_NEVER_EVALUATED for criterion in report.criteria) + assert all(criterion.status == CriterionStatus.NEVER_EVALUATED.value for criterion in report.criteria) def test_all_six_criteria_always_returned_in_order(self): """Criteria with no row are filled in, so a client can render six cards unconditionally.""" @@ -77,8 +76,8 @@ def test_all_six_criteria_always_returned_in_order(self): assert [criterion.criterion for criterion in report.criteria] == [name.value for name in SealCriterionName] criteria = by_criterion(report) - assert criteria["official"].status == STATUS_PASS - assert criteria["compliant"].status == STATUS_NEVER_EVALUATED + assert criteria["official"].status == CriterionStatus.PASS.value + assert criteria["compliant"].status == CriterionStatus.NEVER_EVALUATED.value def test_mixed_criteria(self): """The report carries each criterion's own verdict alongside the stored seal outcome.""" @@ -106,10 +105,10 @@ def test_mixed_criteria(self): criteria = by_criterion(report) assert report.has_seal is False - assert criteria["official"].status == STATUS_PASS - assert criteria["available"].status == STATUS_FAIL + assert criteria["official"].status == CriterionStatus.PASS.value + assert criteria["available"].status == CriterionStatus.FAIL.value assert criteria["available"].in_grace_period is False - assert criteria["compliant"].status == STATUS_FAIL + assert criteria["compliant"].status == CriterionStatus.FAIL.value assert criteria["compliant"].in_grace_period is True def test_evaluated_at_is_latest_across_criteria(self): diff --git a/api/tests/unittest/models/test_reliability_criterion_impl.py b/api/tests/unittest/models/test_reliability_criterion_impl.py index 714ad4590..b495b813d 100644 --- a/api/tests/unittest/models/test_reliability_criterion_impl.py +++ b/api/tests/unittest/models/test_reliability_criterion_impl.py @@ -2,16 +2,14 @@ from datetime import datetime, timedelta, timezone from shared.common.error_handling import InternalHTTPException -from shared.common.seal_criteria import GRACE_PERIODS, PROBATION_PERIOD, SealCriterionName -from shared.database_gen.sqlacodegen_models import SealCriterion -from shared.db_models.reliability_criterion_impl import ( - STATUS_FAIL, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, - STATUS_PASS, - STATUS_UNKNOWN, - ReliabilityCriterionImpl, +from shared.common.seal_criteria import ( + GRACE_PERIODS, + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, ) +from shared.database_gen.sqlacodegen_models import SealCriterion +from shared.db_models.reliability_criterion_impl import ReliabilityCriterionImpl # Anchored to the real clock because the countdowns are derived against `datetime.now`. Every window # below is at least a day clear of its boundary, so the assertions do not race the wall clock. @@ -43,7 +41,7 @@ def test_passing_criterion(self): result = ReliabilityCriterionImpl.from_orm(make_row()) assert result.criterion == "compliant" - assert result.status == STATUS_PASS + assert result.status == CriterionStatus.PASS.value assert result.in_grace_period is False assert result.grace_period_ends_at is None assert result.on_probation is False @@ -73,7 +71,7 @@ def test_never_evaluated_factory(self): result = ReliabilityCriterionImpl.never_evaluated(SealCriterionName.AVAILABLE) assert result.criterion == "available" - assert result.status == STATUS_NEVER_EVALUATED + assert result.status == CriterionStatus.NEVER_EVALUATED.value assert result.in_grace_period is False assert result.on_probation is False @@ -82,7 +80,7 @@ def test_never_evaluated_status_passes_through(self): row = make_row(observed_status="never_evaluated", confirmed_status="never_evaluated") result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_NEVER_EVALUATED + assert result.status == CriterionStatus.NEVER_EVALUATED.value assert result.in_grace_period is False assert result.on_probation is False @@ -91,7 +89,7 @@ def test_unknown_status_passes_through(self): row = make_row(criterion=SealCriterionName.AVAILABLE, observed_status="unknown", confirmed_status="pass") result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_UNKNOWN + assert result.status == CriterionStatus.UNKNOWN.value assert result.in_grace_period is False def test_not_applicable_status_is_withdrawn(self): @@ -107,7 +105,7 @@ def test_not_applicable_status_is_withdrawn(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_NOT_APPLICABLE + assert result.status == CriterionStatus.NOT_APPLICABLE.value assert result.in_grace_period is False assert result.on_probation is False assert result.probation_ends_at is None @@ -126,7 +124,7 @@ def test_failing_inside_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is True assert result.grace_period_ends_at == first_failure + GRACE_PERIODS[SealCriterionName.COMPLIANT] assert result.first_failure_at == first_failure @@ -147,7 +145,7 @@ def test_grace_exempt_criterion_is_never_in_grace(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -162,7 +160,7 @@ def test_failing_beyond_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -175,7 +173,7 @@ def test_passing_while_on_probation(self): row = make_row(criterion=SealCriterionName.AVAILABLE, probation_start=probation_start) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_PASS + assert result.status == CriterionStatus.PASS.value assert result.on_probation is True assert result.probation_ends_at == probation_start + PROBATION_PERIOD @@ -191,7 +189,7 @@ def test_grace_does_not_apply_during_probation(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.on_probation is True assert result.in_grace_period is False assert result.grace_period_ends_at is None @@ -218,7 +216,7 @@ def test_official_and_stable_have_no_grace_period(self): ) result = ReliabilityCriterionImpl.from_orm(row) - assert result.status == STATUS_FAIL + assert result.status == CriterionStatus.FAIL.value assert result.in_grace_period is False assert result.grace_period_ends_at is None diff --git a/api/tests/unittest/models/test_seal_enum_contract.py b/api/tests/unittest/models/test_seal_enum_contract.py index 0db1effb9..39cf3f433 100644 --- a/api/tests/unittest/models/test_seal_enum_contract.py +++ b/api/tests/unittest/models/test_seal_enum_contract.py @@ -11,8 +11,9 @@ mirror of these tests over its own enums, in `functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py`. -Adding a criterion or status means updating, in lockstep: the Liquibase enum, `SealCriterionName` -and the status constants here, `docs/DatabaseCatalogAPI.yaml` (plus a stub regen), and the job enums. +Adding a criterion or status means updating, in lockstep: the Liquibase enum, the enums in +`shared.common.seal_criteria` (which the API and the nightly job both read), and +`docs/DatabaseCatalogAPI.yaml` plus a stub regen. """ import unittest @@ -21,24 +22,12 @@ from shared.common.seal_criteria import ( GRACE_PERIODS, PROBATION_PERIODS, + CriterionStatus, SealCriterionName, ) from shared.database_gen.sqlacodegen_models import SealCriterion -from shared.db_models.reliability_criterion_impl import ( - STATUS_FAIL, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, - STATUS_PASS, - STATUS_UNKNOWN, -) -API_STATUSES = { - STATUS_PASS, - STATUS_FAIL, - STATUS_UNKNOWN, - STATUS_NEVER_EVALUATED, - STATUS_NOT_APPLICABLE, -} +API_STATUSES = {status.value for status in CriterionStatus} def db_enum_values(column_name: str) -> set: @@ -53,13 +42,19 @@ def test_criterion_names_match_db_enum(self): """`SealCriterionName` is the full `seal_criterion_name` type, no more and no less.""" assert {criterion.value for criterion in SealCriterionName} == db_enum_values("criterion") - def test_status_constants_match_db_enum(self): - """The API `status` values are the `seal_criterion_status` type verbatim. + def test_status_values_match_db_enum(self): + """`CriterionStatus` is the `seal_criterion_status` type verbatim. There is no DB-to-API translation left, so any divergence would be served raw to clients. + The nightly job writes these same values from the same enum, so one assertion pins both + sides to the schema. """ assert API_STATUSES == db_enum_values("observed_status") + def test_confirmed_status_shares_the_same_type(self): + """Both status columns are the one enum; a check that only covered one could drift.""" + assert db_enum_values("confirmed_status") == db_enum_values("observed_status") + def test_every_db_status_passes_response_validation(self): """Every status the job can store must be accepted by the generated response model. @@ -84,8 +79,9 @@ def test_every_criterion_passes_response_validation(self): def test_policy_maps_cover_every_criterion(self): """Every criterion needs a grace and probation entry, even if the window is None. - `GRACE_PERIODS.get()` would silently return None for a criterion missing from the map, - turning it into a no-grace criterion by accident rather than by decision. + The maps are the single source for both the API's countdowns and the job's debouncing + (`grace_period_for` / `probation_period_for`), so a criterion missing from one would raise + on lookup rather than quietly becoming a no-grace criterion. """ assert set(GRACE_PERIODS) == set(SealCriterionName) assert set(PROBATION_PERIODS) == set(SealCriterionName) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index f33767660..71380816d 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -18,10 +18,12 @@ The evaluators are pure functions over a `FeedSealContext`, so every DB read for a batch of feeds happens here, in a fixed number of queries regardless of batch size. -Only Official is implemented, so the context currently carries just the feed row. Each new -criterion adds the fields it needs here plus one bulk query to populate them: the latest -dataset for Compliant and Fresh, the day's availability rows for Available, the full -dataset coverage history for Fresh continuous coverage. +Official, Stable and Fresh (future coverage) are implemented. Official and Stable read the +feed row alone; Fresh needs one bulk-loaded extra, the feed's latest dataset. Each new +criterion adds the fields it needs here plus, where they are not already on the feed row, one +bulk query to populate them: the latest dataset's validation report for Compliant, the day's +availability rows for Available, the full dataset coverage history for Fresh continuous +coverage. """ import itertools @@ -29,9 +31,21 @@ from datetime import datetime from typing import Dict, Iterator, List, Optional, Sequence +from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed +from shared.database_gen.sqlacodegen_models import Feed, Gtfsdataset, Gtfsfeed + + +@dataclass(frozen=True) +class LatestDataset: + """ + The feed's latest dataset as of the run's `now`, and the fields criteria read off it. + """ + + dataset_id: str + downloaded_at: datetime + service_date_range_end: Optional[datetime] = None @dataclass @@ -49,6 +63,14 @@ class FeedSealContext: # Feed-level flags official: Optional[bool] = None + is_producer_url_unstable: Optional[bool] = None + seasonal: Optional[bool] = None + + # Stable: when the feed was first added to the database. + feed_created_at: Optional[datetime] = None + + # The feed's latest dataset as of `now` - resolved by `downloaded_at` vs `now` + latest_dataset: Optional[LatestDataset] = None # Feeds in these statuses, or not published, are never eligible for the seal. @@ -130,15 +152,56 @@ def iter_eligible_stable_ids( yield chunk +def _load_latest_datasets( + db_session: Session, feed_ids: Sequence[str], now: datetime +) -> Dict[str, LatestDataset]: + """feed_id -> the feed's latest dataset as of `now`, for feeds that had one. + + "Latest as of `now`" is the most recently downloaded dataset with + `downloaded_at <= now`. + + A feed missing from the result had no dataset at all as of `now`. That is deliberately + distinct from a `LatestDataset` whose `service_date_range_end` is None, which had one + whose coverage was never extracted - the criteria read both as UNKNOWN but report which. + """ + if not feed_ids: + return {} + rows = db_session.execute( + select( + Gtfsdataset.feed_id, + Gtfsdataset.id, + Gtfsdataset.downloaded_at, + Gtfsdataset.service_date_range_end, + ) + .where( + Gtfsdataset.feed_id.in_(list(feed_ids)), + Gtfsdataset.downloaded_at.is_not(None), + Gtfsdataset.downloaded_at <= now, + ) + .distinct(Gtfsdataset.feed_id) + .order_by( + Gtfsdataset.feed_id, + Gtfsdataset.downloaded_at.desc(), + Gtfsdataset.id.desc(), + ) + ).all() + return { + row.feed_id: LatestDataset( + dataset_id=row.id, + downloaded_at=row.downloaded_at, + service_date_range_end=row.service_date_range_end, + ) + for row in rows + } + + def build_contexts( db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime ) -> Dict[str, FeedSealContext]: """Load everything the evaluators need for `feeds`, in a fixed number of queries. Args: - db_session: SQLAlchemy session. Unused while Official is the only criterion, since - everything it needs is already on the feed row, but kept in the signature - because every further criterion needs it. + db_session: SQLAlchemy session. feeds: The batch of feeds to load, already loaded (and eligibility-checked via `is_seal_eligible`) by the caller — `update_seals`. now: The evaluation timestamp. @@ -153,10 +216,10 @@ def build_contexts( below. No query, no cost. 2. Needs its own query. Add the field, then a module-level `_load_*` helper that takes - the whole batch and returns a dict keyed by feed_id, and call it once here. Keeping - the query per batch rather than per feed is what holds the query count proportional - to the number of criteria instead of the number of feeds. For example, Available - (issue #1784) would add: + the whole batch and returns a dict keyed by feed_id, and call it once here, as + `_load_latest_datasets` does. Keeping the query per batch rather than per feed + is what holds the query count proportional to the number of criteria instead of the + number of feeds. For example, Available (issue #1784) would add: def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool]: '''feed_id -> whether any availability check succeeded since day_start. @@ -166,12 +229,20 @@ def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool] called once as `availability = _load_availability_today(...)` and consumed per feed as `availability_success_today=availability.get(feed.id, False)`. """ + latest_datasets = _load_latest_datasets( + db_session, [feed.id for feed in feeds], now + ) + return { feed.id: FeedSealContext( feed_id=feed.id, now=now, stable_id=feed.stable_id, official=feed.official, + is_producer_url_unstable=feed.is_producer_url_unstable, + seasonal=feed.seasonal, + feed_created_at=feed.created_at, + latest_dataset=latest_datasets.get(feed.id), ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py deleted file mode 100644 index ed4574b56..000000000 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/criteria.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# MobilityData 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Seal of Reliability policy values and domain enums. - -The policy values here are the published definition of the seal, so they live in code -rather than in DB config: changing any of them changes which feeds qualify, and that should -go through code review, tests and a deploy. - -Each criterion's grace period belongs with the criterion, as a class attribute on its -evaluator. Only Official is implemented so far and it has none. -""" - -from datetime import timedelta -from enum import Enum -from typing import Final - - -class SealCriterionName(str, Enum): - """The six seal criteria. Values match the seal_criterion_name DB enum. - - All six are listed even though only Official has an evaluator (see #1784 and #1782), - so that the enum stays a faithful mirror of the database type. - """ - - OFFICIAL = "official" - STABLE = "stable" - AVAILABLE = "available" - COMPLIANT = "compliant" - FRESH_COVERAGE = "fresh_coverage" - FRESH_CONTINUOUS = "fresh_continuous" - - -class CriterionStatus(str, Enum): - """A criterion's status. Values match the seal_criterion_status DB enum. - - Only PASS and FAIL are verdicts. The other three say why there is no verdict, and they - are not interchangeable — the roll-up in `seal_updater` sends them in three different - directions: - - * UNKNOWN — we could not look; the inputs the check needs were not there. A property of - the run, not of the feed. The criterion keeps its last confirmed verdict and stays in - the roll-up, so an upstream outage freezes a criterion rather than waiving it. - * NOT_APPLICABLE — there is no question to ask; the criterion is deliberately excluded - for this feed (Fresh / future coverage on a seasonal feed). A property of the feed. - The criterion leaves the roll-up entirely. - * NEVER_EVALUATED — never had a verdict, since the feed first appeared. The initial value, - and the only one the job never writes back once a criterion has left it. - - UNKNOWN is never written to `confirmed_status`: a run that could not look does not - change the answer, it leaves the previous one standing. - """ - - PASS = "pass" - FAIL = "fail" - UNKNOWN = "unknown" - NEVER_EVALUATED = "never_evaluated" - NOT_APPLICABLE = "not_applicable" - - @property - def is_verdict(self) -> bool: - """True for PASS and FAIL — the two values that mean the check actually answered.""" - return self in (CriterionStatus.PASS, CriterionStatus.FAIL) - - -class CriterionPhase(str, Enum): - """Which of the two debouncing mechanisms is currently acting on a criterion. - - Derived from the stored row rather than stored itself (see `state_machine.phase`): it is - a pure function of `probation_start`, `confirmed_status` and `first_observed_failure_at`, - all of which are already on the row, so a stored copy would be a second thing to keep in - step for no gain. - - The three values are mutually exclusive: probation suspends the grace period, so a - criterion can never be serving a penalty and holding a failure under grace at once. - """ - - STEADY = "steady" - IN_GRACE_PERIOD = "in_grace_period" - ON_PROBATION = "on_probation" - - -# A criterion that recovers from a confirmed failure is put on probation: it must then go -# this long with no observed failure before it can contribute to the seal again. It is the -# default for new evaluators; Official is exempt because it is a point-in-time state check -# (see OfficialEvaluator). -# -# Probation is opened only by a recovery. A first evaluation that passes is not a recovery, -# so a feed that has never had a confirmed failure never serves probation at all and can -# hold the seal from its very first evaluation. -PROBATION_PERIOD: Final[timedelta] = timedelta(days=180) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 3ddabbe94..349f33311 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -15,9 +15,12 @@ # """The seal criterion evaluators. -`EVALUATORS` is the registry the job iterates. Only Official is implemented so far -(issue #1783); the remaining criteria are tracked by #1784 and #1782. Adding one means a -new subclass, an entry here, and whatever fields it needs on `FeedSealContext`. +`EVALUATORS` is the registry the job iterates. Official (issue #1783), Stable and Fresh / +future coverage (issue #1784) are implemented; Available and Compliant are the rest of +#1784, and Fresh / continuous coverage is tracked by #1782. Adding one means a new subclass, +an entry here, and whatever fields it needs on `FeedSealContext`. Its windows are not declared +on the subclass: they come from the policy maps in `shared.common.seal_criteria`, which the +read API reads too. `seal_criterion_name` in the database already declares all six values, so a criterion can be added without a schema change. @@ -29,15 +32,21 @@ CriterionEvaluator, CriterionObservation, ) +from tasks.seal_of_reliability.evaluators.fresh_coverage import FreshCoverageEvaluator from tasks.seal_of_reliability.evaluators.official import OfficialEvaluator +from tasks.seal_of_reliability.evaluators.stable import StableEvaluator EVALUATORS: Final[List[CriterionEvaluator]] = [ OfficialEvaluator(), + StableEvaluator(), + FreshCoverageEvaluator(), ] __all__ = [ "EVALUATORS", "CriterionEvaluator", "CriterionObservation", + "FreshCoverageEvaluator", "OfficialEvaluator", + "StableEvaluator", ] diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py index 7291ee2f4..5436557d6 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py @@ -19,12 +19,13 @@ from datetime import timedelta from typing import Optional, Tuple -from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import ( - PROBATION_PERIOD, +from shared.common.seal_criteria import ( CriterionStatus, SealCriterionName, + grace_period_for, + probation_period_for, ) +from tasks.seal_of_reliability.context import FeedSealContext @dataclass(frozen=True) @@ -49,18 +50,33 @@ class CriterionObservation: class CriterionEvaluator: """Evaluates one criterion against a pre-loaded feed context. - Subclasses set `name` and, where they differ from the defaults, `grace_period` and - `probation_period`, then implement `_evaluate`. They never touch the database: all the - data they need is on the context, loaded in bulk by `context.build_contexts`. + A subclass sets `name` and implements `_evaluate`. It does not declare its own windows: + both are resolved from `name` against the policy maps in `shared.common.seal_criteria`, + which the read API reads too, so a criterion cannot debounce one way for the job and + another way for the API. To change a window, change the map. - `grace_period` holds a passing status while an observed failure is still young. - `probation_period` is how long the criterion must go with no observed failure after - recovering from a confirmed failure. None on either means the criterion does not use it. + Evaluators never touch the database: all the data they need is on the context, loaded in + bulk by `context.build_contexts`. """ name: SealCriterionName = None - grace_period: Optional[timedelta] = None - probation_period: Optional[timedelta] = PROBATION_PERIOD + + @property + def grace_period(self) -> Optional[timedelta]: + """How long an observed failure may run before it is confirmed, per the policy map. + + None means the criterion has no grace period and its status flips on the first + failing day. + """ + return grace_period_for(self.name) + + @property + def probation_period(self) -> Optional[timedelta]: + """How long this criterion serves after recovering, per the policy map. + + None means it never serves probation. + """ + return probation_period_for(self.name) def evaluate(self, ctx: FeedSealContext) -> CriterionObservation: """Evaluate the criterion and label the result with this evaluator's name. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py new file mode 100644 index 000000000..c932cb4e1 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/fresh_coverage.py @@ -0,0 +1,71 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Fresh (future coverage) criterion: the latest dataset still covers the near future.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + FUTURE_COVERAGE_HORIZON, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class FreshCoverageEvaluator(CriterionEvaluator): + """`latest dataset.service_date_range_end >= now + 7 days`. + + This is the only implemented criterion that can return NOT_APPLICABLE. A seasonal feed + is expected to have coverage that runs out between seasons, so the question "does this + feed cover the next week" has no meaningful answer for it. + """ + + name = SealCriterionName.FRESH_COVERAGE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + # Applicability is a property of the feed, so it is settled before the inputs are + # looked at: a seasonal feed's missing dataset is not an UNKNOWN worth reporting. + if ctx.seasonal is True: + return ( + CriterionStatus.NOT_APPLICABLE, + "the feed is seasonal, so future coverage is not required", + ) + + # Two different missing inputs, kept apart so the report says which: no dataset at + # all as of this run, or one whose coverage was never extracted. + if ctx.latest_dataset is None: + return CriterionStatus.UNKNOWN, "the feed has no latest dataset" + + coverage_end = ctx.latest_dataset.service_date_range_end + if coverage_end is None: + return ( + CriterionStatus.UNKNOWN, + "the latest dataset has no service_date_range_end", + ) + + horizon = ctx.now + FUTURE_COVERAGE_HORIZON + if coverage_end < horizon: + return ( + CriterionStatus.FAIL, + f"coverage ends {coverage_end.isoformat()}, before the " + f"{FUTURE_COVERAGE_HORIZON.days}-day horizon {horizon.isoformat()}", + ) + return ( + CriterionStatus.PASS, + f"coverage ends {coverage_end.isoformat()}, at or beyond the " + f"{FUTURE_COVERAGE_HORIZON.days}-day horizon {horizon.isoformat()}", + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py index 340a2fc31..d848bb4d1 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/official.py @@ -17,8 +17,8 @@ from typing import Tuple +from shared.common.seal_criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator @@ -26,14 +26,12 @@ class OfficialEvaluator(CriterionEvaluator): """`feed.official IS TRUE`. A point-in-time state check: official at the time of reviewing the dataset, with no - 6-month check. Both the grace period and the probation period are None, so the criterion - fails the same day the flag is lost and clears the same day it comes back, taking the - seal with it in both directions. + 6-month check. The policy maps give it neither a grace period nor a probation period, so + the criterion fails the same day the flag is lost and clears the same day it comes back, + taking the seal with it in both directions. """ name = SealCriterionName.OFFICIAL - grace_period = None - probation_period = None def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: # `is True` rather than a truthiness test: NULL is not an endorsement. It is a FAIL diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py new file mode 100644 index 000000000..0372a700e --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/stable.py @@ -0,0 +1,57 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Stable criterion: the feed is old enough in the database, from a stable producer URL.""" + +from typing import Tuple + +from shared.common.seal_criteria import ( + TRACKING_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.evaluators.base import CriterionEvaluator + + +class StableEvaluator(CriterionEvaluator): + """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable. + + Note the check conflates two things: an observed failure means either "the producer URL + is flagged unstable" or "the feed is younger than 180 days", and the stored state cannot + say which. That is a deliberate. + """ + + name = SealCriterionName.STABLE + + def _evaluate(self, ctx: FeedSealContext) -> Tuple[CriterionStatus, str]: + if ctx.is_producer_url_unstable: + return CriterionStatus.FAIL, "feed.is_producer_url_unstable is True" + + if ctx.feed_created_at is None: + # feed.created_at is NOT NULL, so this is unreachable from the database and means + # a context was built without it. Still a verdict rather than an UNKNOWN. + return CriterionStatus.FAIL, "feed.created_at is missing" + + age = ctx.now - ctx.feed_created_at + if age < TRACKING_PERIOD: + return ( + CriterionStatus.FAIL, + f"in the database for {age.days} day(s), needs {TRACKING_PERIOD.days}", + ) + return ( + CriterionStatus.PASS, + f"in the database for {age.days} day(s) with a stable producer URL", + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 3edf7e21c..37807c1d2 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -37,6 +37,11 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from shared.common.seal_criteria import ( + CriterionPhase, + CriterionStatus, + SealCriterionName, +) from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( FeedReliabilitySeal, @@ -50,11 +55,6 @@ build_contexts, is_seal_eligible, ) -from tasks.seal_of_reliability.criteria import ( - CriterionPhase, - CriterionStatus, - SealCriterionName, -) from tasks.seal_of_reliability.evaluators import EVALUATORS from tasks.seal_of_reliability.state_machine import ( SealCriterionState, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py index 4dc5d37a0..8a3a1405f 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py @@ -38,7 +38,7 @@ from datetime import datetime, timedelta, timezone from typing import Optional -from tasks.seal_of_reliability.criteria import ( +from shared.common.seal_criteria import ( CriterionPhase, CriterionStatus, SealCriterionName, diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py index 2989187eb..1b49964c3 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_end_to_end_db.py @@ -43,13 +43,15 @@ from main import tasks_executor from sqlalchemy import delete, select -from tasks.seal_of_reliability.criteria import CriterionStatus +from shared.common.seal_criteria import CriterionStatus, SealCriterionName +from tasks.seal_of_reliability.evaluators import EVALUATORS from tasks.seal_of_reliability.update_seal_of_reliability import get_parameters from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + Gtfsdataset, Gtfsfeed, SealCriterion, ) @@ -67,6 +69,11 @@ # The task always runs against an explicit feed list. These are the eligible seeded feeds. REQUESTED = [OFFICIAL, NOT_OFFICIAL, UNKNOWN_OFFICIAL] +# Every seeded feed gets a dataset covering the next 400 days, so Fresh passes and `official` +# stays the only criterion that separates the feeds. Stable needs nothing seeded: it reads +# `feed.created_at`, which `_seed` already backdates by 400 days. +COVERAGE_END = NOW + timedelta(days=400) + def _seed(db_session, feed_id, official=True, status="active", operational="published"): db_session.add( @@ -79,9 +86,31 @@ def _seed(db_session, feed_id, official=True, status="active", operational="publ official=official, created_at=NOW - timedelta(days=400), producer_url=f"https://example.com/{feed_id}.zip", + seasonal=False, + ) + ) + db_session.flush() + + # Fresh's input. Seeded rather than produced by the run so the criterion passes and + # `official` stays the only variable. + dataset_id = f"{feed_id}_dataset" + db_session.add( + Gtfsdataset( + id=dataset_id, + feed_id=feed_id, + stable_id=dataset_id, + downloaded_at=NOW - timedelta(days=1), + service_date_range_start=NOW - timedelta(days=30), + service_date_range_end=COVERAGE_END, ) ) db_session.flush() + db_session.execute( + Gtfsfeed.__table__.update() + .where(Gtfsfeed.__table__.c.id == feed_id) + .values(latest_dataset_id=dataset_id) + ) + db_session.flush() def _cleanup(db_session): @@ -162,14 +191,36 @@ def seal_state(db_session): @staticmethod @with_db_session(db_url=default_db_url) def criterion_state(db_session): - """stable_id -> the sealcriterion row, for the seeded feeds.""" + """(stable_id, criterion) -> the seal_criterion row, for the seeded feeds. + + Keyed by the criterion too, not just the feed: the registry holds several + evaluators, so a feed has one row per criterion. + """ criterion = SealCriterion.__table__ rows = db_session.execute( select(Feed.stable_id, criterion) .join(criterion, criterion.c.feed_id == Feed.id) .where(Feed.stable_id.like(f"{PREFIX}%")) ).all() - return {row.stable_id: row for row in rows} + return {(row.stable_id, row.criterion): row for row in rows} + + @classmethod + def official_state(cls): + """stable_id -> the `official` seal_criterion row, the one these tests drive.""" + return { + stable_id: row + for (stable_id, criterion), row in cls.criterion_state().items() + if criterion == SealCriterionName.OFFICIAL.value + } + + @staticmethod + def official_criterion(feed_report: dict) -> dict: + """The `official` entry of a report's per-feed criteria list.""" + return next( + row + for row in feed_report["criteria"] + if row["criterion"] == SealCriterionName.OFFICIAL.value + ) @staticmethod @with_db_session(db_url=default_db_url) @@ -199,7 +250,7 @@ def test_two_runs_with_a_change_in_between(self): # --- first run: writes the initial state first = self.run_task({"dry_run": False, "now": NOW.isoformat()}) self.assertFalse(first["dry_run"]) - self.assertGreaterEqual(first["criterion_rows_written"], 3) + self.assertGreaterEqual(first["criterion_rows_written"], 3 * len(EVALUATORS)) self.assertEqual(first["seals_revoked"], 0, "nothing was held beforehand") self.assertEqual( {row["stable_id"] for row in self.ours(first)}, @@ -217,8 +268,8 @@ def test_two_runs_with_a_change_in_between(self): }, "only the official feed earned it; the others were never granted or lost", ) - criteria = self.criterion_state() - self.assertNotIn(DEPRECATED, criteria) + criteria = self.official_state() + self.assertNotIn(DEPRECATED, criteria, "ineligible feeds are never evaluated") self.assertNotIn(UNPUBLISHED, criteria) self.assertEqual( criteria[OFFICIAL].confirmed_status, CriterionStatus.PASS.value @@ -259,22 +310,22 @@ def test_two_runs_with_a_change_in_between(self): self.assertTrue(moved[OFFICIAL]["had_seal"]) self.assertFalse(moved[OFFICIAL]["has_seal"]) self.assertEqual( - moved[OFFICIAL]["criteria"][0]["confirmed_status"], + self.official_criterion(moved[OFFICIAL])["confirmed_status"], CriterionStatus.FAIL.value, ) self.assertEqual( - moved[OFFICIAL]["criteria"][0]["previously_confirmed_status"], + self.official_criterion(moved[OFFICIAL])["previously_confirmed_status"], CriterionStatus.PASS.value, ) self.assertFalse(moved[NOT_OFFICIAL]["had_seal"]) self.assertTrue(moved[NOT_OFFICIAL]["has_seal"]) self.assertEqual( - moved[NOT_OFFICIAL]["criteria"][0]["confirmed_status"], + self.official_criterion(moved[NOT_OFFICIAL])["confirmed_status"], CriterionStatus.PASS.value, ) self.assertEqual( - moved[NOT_OFFICIAL]["criteria"][0]["previously_confirmed_status"], + self.official_criterion(moved[NOT_OFFICIAL])["previously_confirmed_status"], CriterionStatus.FAIL.value, ) @@ -288,7 +339,7 @@ def test_two_runs_with_a_change_in_between(self): }, "the revoked feed keeps its earned_at; the recovered one gains earned_at", ) - criteria = self.criterion_state() + criteria = self.official_state() self.assertEqual(criteria[OFFICIAL].first_observed_failure_at, later) self.assertIsNone( criteria[NOT_OFFICIAL].first_observed_failure_at, "the streak ended" @@ -313,11 +364,11 @@ def test_third_run_with_no_change_is_a_no_op(self): self.assertEqual(report["seals_before_run"], report["seals_after_run"]) after = self.criterion_state() - for stable_id, row in after.items(): - with self.subTest(stable_id=stable_id): + for (stable_id, criterion), row in after.items(): + with self.subTest(stable_id=stable_id, criterion=criterion): self.assertEqual( row.first_observed_failure_at, - before[stable_id].first_observed_failure_at, + before[(stable_id, criterion)].first_observed_failure_at, "a re-evaluation must not restart a failure streak", ) self.assertEqual(row.evaluated_at, later, "but it is re-evaluated") diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py index 87738fb6f..f94f9938b 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_enum_contract.py @@ -13,24 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # -"""Contract tests pinning the job's seal enums to the database types. No database connection. +"""Contract tests pinning the seal enums to the database types, from the job's tree. -The job writes `seal_criterion.criterion`, `observed_status` and `confirmed_status` from its own -`SealCriterionName` / `CriterionStatus` enums, and the read API serves those stored values straight -through to clients (the API `status` enum is the DB enum verbatim). So a value added here that the -database type does not have, or vice versa, breaks either the write or the read - and the read API -raises on a criterion it does not recognise rather than hiding it. +The job writes `seal_criterion.criterion`, `observed_status` and `confirmed_status` from +`SealCriterionName` / `CriterionStatus`, and the read API serves those stored values straight +through to clients. So a value the enum has that the database type does not, or vice versa, +breaks either the write or the read. -These tests compare against the SQLAlchemy Enum types in the generated models, which are generated -from the Liquibase changelog, so the assertion is against the real schema and needs no live DB. -Adding a criterion or status means updating, in lockstep: the Liquibase enum, the API's -`SealCriterionName` and status constants, `docs/DatabaseCatalogAPI.yaml`, and the enums here. +Both sides now import those enums from one module, `shared.common.seal_criteria`, so this file +asserts the same contract as `api/tests/unittest/models/test_seal_enum_contract.py` rather than a +mirror of it. It is kept because it runs from the function's own tree and venv: it is what catches +the enums failing to resolve at all through the `include_api_folders` symlink, which the API-side +test cannot see. + +The assertions compare against the SQLAlchemy Enum types in the generated models, which come from +the Liquibase changelog, so this checks the real schema with no live DB. Adding a criterion or +status means updating, in lockstep: the Liquibase enum, `shared.common.seal_criteria`, and +`docs/DatabaseCatalogAPI.yaml` plus a stub regen. """ import unittest +from shared.common.seal_criteria import CriterionStatus, SealCriterionName from shared.database_gen.sqlacodegen_models import SealCriterion -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName def db_enum_values(column_name: str) -> set: diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index 190102e29..a37940102 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -16,14 +16,22 @@ """Unit tests for the seal criterion evaluators. No database.""" import unittest -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone -from tasks.seal_of_reliability.context import FeedSealContext -from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName +from shared.common.seal_criteria import ( + FUTURE_COVERAGE_HORIZON, + PROBATION_PERIOD, + TRACKING_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.context import FeedSealContext, LatestDataset from tasks.seal_of_reliability.evaluators import ( EVALUATORS, CriterionEvaluator, + FreshCoverageEvaluator, OfficialEvaluator, + StableEvaluator, ) NOW = datetime(2026, 6, 1, tzinfo=timezone.utc) @@ -133,13 +141,196 @@ def test_never_returns_a_no_verdict_status(self): def test_has_no_grace_or_probation(self): """A point-in-time check: it clears as soon as the feed is official again.""" - self.assertIsNone(OfficialEvaluator.grace_period) - self.assertIsNone(OfficialEvaluator.probation_period) + self.assertIsNone(OfficialEvaluator().grace_period) + self.assertIsNone(OfficialEvaluator().probation_period) def test_reason_names_the_offending_value(self): result = OfficialEvaluator().evaluate(_ctx(official=None)) self.assertIn("None", result.reason) +class TestStable(unittest.TestCase): + """`feed.created_at <= now - 180 days` and the producer URL is not flagged unstable.""" + + def _stable_ctx(self, **overrides): + defaults = {"feed_created_at": NOW - TRACKING_PERIOD - timedelta(days=1)} + defaults.update(overrides) + return _ctx(**defaults) + + def test_an_old_feed_with_a_stable_url_passes(self): + self.assertIs( + StableEvaluator().evaluate(self._stable_ctx()).observed_status, + CriterionStatus.PASS, + ) + + def test_an_unstable_producer_url_fails(self): + result = StableEvaluator().evaluate( + self._stable_ctx(is_producer_url_unstable=True) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("is_producer_url_unstable", result.reason) + + def test_a_null_unstable_flag_is_not_a_claim_of_instability(self): + """The SQL predicate is `IS NOT TRUE`, so NULL and False both leave the check open.""" + for flag in (None, False): + with self.subTest(is_producer_url_unstable=flag): + self.assertIs( + StableEvaluator() + .evaluate(self._stable_ctx(is_producer_url_unstable=flag)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_young_feed_fails(self): + result = StableEvaluator().evaluate( + self._stable_ctx(feed_created_at=NOW - timedelta(days=179)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("179", result.reason) + + def test_a_feed_created_today_fails(self): + result = StableEvaluator().evaluate(self._stable_ctx(feed_created_at=NOW)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("0 day(s)", result.reason) + + def test_the_boundary_day_passes(self): + """Exactly 180 days in the database is enough; the check is `<= now - 180 days`.""" + self.assertIs( + StableEvaluator() + .evaluate(self._stable_ctx(feed_created_at=NOW - TRACKING_PERIOD)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_a_missing_creation_date_fails_rather_than_withholding(self): + """feed.created_at is NOT NULL, so this is unreachable from the database. + + It is still a verdict: an UNKNOWN would freeze the criterion at whatever it last + said, and reporting a feed as not yet stable beats holding a seal on a value nobody + supplied. + """ + result = StableEvaluator().evaluate(self._stable_ctx(feed_created_at=None)) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("created_at", result.reason) + + def test_the_unstable_flag_is_checked_before_the_feed_age(self): + """Both fail, but the reason has to name the one an operator can act on.""" + result = StableEvaluator().evaluate( + _ctx(feed_created_at=NOW, is_producer_url_unstable=True) + ) + self.assertIn("is_producer_url_unstable", result.reason) + + def test_never_returns_a_no_verdict_status(self): + """Both inputs are on the feed row, so Stable can never withhold a verdict.""" + for created_at in (None, NOW, NOW - timedelta(days=400)): + for flag in (None, False, True): + with self.subTest(feed_created_at=created_at, unstable=flag): + status = ( + StableEvaluator() + .evaluate( + _ctx( + feed_created_at=created_at, + is_producer_url_unstable=flag, + ) + ) + .observed_status + ) + self.assertTrue(status.is_verdict) + + def test_has_no_grace_or_probation(self): + """A point-in-time check, like Official: neither input flickers.""" + self.assertIsNone(StableEvaluator().grace_period) + self.assertIsNone(StableEvaluator().probation_period) + + +class TestFreshCoverage(unittest.TestCase): + """`latest dataset.service_date_range_end >= now + 7 days`.""" + + @staticmethod + def _dataset(coverage_end): + return LatestDataset( + dataset_id="mdb-1-202606010000", + downloaded_at=NOW - timedelta(days=1), + service_date_range_end=coverage_end, + ) + + def _fresh_ctx(self, coverage_end=NOW + timedelta(days=90), **overrides): + defaults = {"latest_dataset": self._dataset(coverage_end)} + defaults.update(overrides) + return _ctx(**defaults) + + def test_coverage_beyond_the_horizon_passes(self): + self.assertIs( + FreshCoverageEvaluator().evaluate(self._fresh_ctx()).observed_status, + CriterionStatus.PASS, + ) + + def test_coverage_inside_the_horizon_fails(self): + """It fails before the data runs out, not on the day it does.""" + result = FreshCoverageEvaluator().evaluate( + self._fresh_ctx(NOW + timedelta(days=3)) + ) + self.assertIs(result.observed_status, CriterionStatus.FAIL) + self.assertIn("before the", result.reason) + + def test_the_horizon_itself_passes(self): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(NOW + FUTURE_COVERAGE_HORIZON)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_expired_coverage_fails(self): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(NOW - timedelta(days=1))) + .observed_status, + CriterionStatus.FAIL, + ) + + def test_a_seasonal_feed_is_not_applicable(self): + """Withdrawn from the roll-up rather than failed: the question is meaningless.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(seasonal=True)) + self.assertIs(result.observed_status, CriterionStatus.NOT_APPLICABLE) + self.assertIn("seasonal", result.reason) + + def test_a_seasonal_feed_is_not_applicable_even_with_no_dataset(self): + """Applicability is a property of the feed, so it is settled before the inputs.""" + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(seasonal=True, latest_dataset=None)) + .observed_status, + CriterionStatus.NOT_APPLICABLE, + ) + + def test_a_non_seasonal_feed_is_evaluated(self): + for seasonal in (None, False): + with self.subTest(seasonal=seasonal): + self.assertIs( + FreshCoverageEvaluator() + .evaluate(self._fresh_ctx(seasonal=seasonal)) + .observed_status, + CriterionStatus.PASS, + ) + + def test_no_latest_dataset_is_unknown(self): + """Not a failure: a feed we have never fetched says nothing about its freshness.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(latest_dataset=None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("no latest dataset", result.reason) + + def test_a_dataset_with_no_coverage_end_is_unknown(self): + """The other missing input, and the reason has to tell the two apart.""" + result = FreshCoverageEvaluator().evaluate(self._fresh_ctx(None)) + self.assertIs(result.observed_status, CriterionStatus.UNKNOWN) + self.assertIn("service_date_range_end", result.reason) + + def test_has_a_grace_period_and_serves_probation(self): + """Coverage lapses are the routine failure the grace period exists to absorb.""" + self.assertEqual(FreshCoverageEvaluator().grace_period, timedelta(days=14)) + self.assertEqual(FreshCoverageEvaluator().probation_period, PROBATION_PERIOD) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py index 17bb85518..e70838347 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py @@ -18,7 +18,7 @@ import unittest from datetime import datetime, timedelta, timezone -from tasks.seal_of_reliability.criteria import ( +from shared.common.seal_criteria import ( PROBATION_PERIOD, CriterionPhase, CriterionStatus, diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index aa0e073ca..70af32f04 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -20,18 +20,22 @@ from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock, patch +from shared.common.seal_criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) from tasks.seal_of_reliability.context import ( build_contexts, count_eligible_feeds, is_seal_eligible, iter_eligible_stable_ids, ) -from tasks.seal_of_reliability.criteria import ( - PROBATION_PERIOD, - CriterionStatus, - SealCriterionName, +from tasks.seal_of_reliability.evaluators import ( + EVALUATORS, + CriterionEvaluator, + OfficialEvaluator, ) -from tasks.seal_of_reliability.evaluators import CriterionEvaluator, OfficialEvaluator from tasks.seal_of_reliability.seal_updater import update_seals from tasks.seal_of_reliability.state_machine import SealCriterionState from sqlalchemy import delete, select @@ -40,6 +44,7 @@ from shared.database_gen.sqlacodegen_models import ( Feed, FeedReliabilitySeal, + Gtfsdataset, Gtfsfeed, SealCriterion, SealCriterionSnapshot, @@ -55,6 +60,9 @@ DEPRECATED = f"{PREFIX}deprecated" UNPUBLISHED = f"{PREFIX}unpublished" INACTIVE = f"{PREFIX}inactive" +# An official feed old enough in the database for Stable to pass (every seeded feed is +# backdated 400 days), used by the tests that run the real registry rather than a patched one. +TRACKED = f"{PREFIX}tracked" # The eligible feeds this module seeds. Runs that assert exact counts must be scoped to # these: an unnamed run also covers the fixtures seeded by conftest.pytest_sessionstart and @@ -85,6 +93,13 @@ def _evaluate(self, ctx): # Patched over the registry so the roll-up sees a criterion that can be on probation. WITH_PROBATION = [OfficialEvaluator(), _StandInEvaluator()] +# Official on its own. The classes below that assert seal outcomes are about the roll-up, +# the report and the snapshot table rather than about any particular criterion, and Official +# is the one that reaches a verdict from nothing but the feed row. Stable and Fresh need a +# seal row and a dataset to say anything, and both are exercised against real data in +# TestFullRegistry. +ONLY_OFFICIAL = [OfficialEvaluator()] + DARK_FROM = NOW + timedelta(days=2) @@ -120,13 +135,17 @@ def _evaluate(self, ctx): class _StopsApplyingEvaluator(CriterionEvaluator): """A criterion that stops applying to the feed partway through, standing in for #1782. - Fresh (future coverage) does not apply to a seasonal feed, and a feed can be marked + Fresh / continuous coverage does not apply to a seasonal feed, and a feed can be marked seasonal at any time. Keyed on the clock rather than on `official` so a test can drive it and Official in opposite directions at the same moment. No grace period, so its verdicts land immediately and the tests are about what happens once it withdraws. + + It borrows `fresh_continuous`, which has no evaluator of its own yet, rather than + `fresh_coverage`, whose real evaluator now covers the same seasonal case against real + data in `TestFullRegistry`. """ - name = SealCriterionName.FRESH_COVERAGE + name = SealCriterionName.FRESH_CONTINUOUS grace_period = None def _evaluate(self, ctx): @@ -145,6 +164,8 @@ def _seed_feed( official=True, status="active", operational_status="published", + seasonal=False, + is_producer_url_unstable=None, ): """Insert one GTFS feed.""" db_session.add( @@ -155,6 +176,8 @@ def _seed_feed( status=status, operational_status=operational_status, official=official, + seasonal=seasonal, + is_producer_url_unstable=is_producer_url_unstable, created_at=NOW - timedelta(days=400), producer_url=f"https://example.com/{feed_id}.zip", ) @@ -162,6 +185,45 @@ def _seed_feed( db_session.flush() +def _seed_dataset( + db_session, feed_id: str, coverage_end, downloaded_at=None, suffix="" +): + """Give the feed a dataset covering up to `coverage_end` (None = never extracted). + + `latest_dataset_id` is pointed at it too, so a test that only seeds one dataset matches + what the catalog looks like. The criterion resolves the latest dataset from + `downloaded_at` rather than from that pointer, which is what the two-dataset tests below + exercise. + """ + dataset_id = f"{feed_id}_dataset{suffix}" + db_session.add( + Gtfsdataset( + id=dataset_id, + feed_id=feed_id, + stable_id=dataset_id, + downloaded_at=downloaded_at or NOW - timedelta(days=1), + service_date_range_start=NOW - timedelta(days=30), + service_date_range_end=coverage_end, + ) + ) + db_session.flush() + db_session.execute( + Gtfsfeed.__table__.update() + .where(Gtfsfeed.__table__.c.id == feed_id) + .values(latest_dataset_id=dataset_id) + ) + db_session.commit() + + +def _set_seasonal(db_session, feed_id: str, seasonal): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(seasonal=seasonal) + ) + db_session.commit() + + def _set_official(db_session, feed_id: str, official): db_session.execute( Feed.__table__.update() @@ -195,6 +257,7 @@ def setUp(self, db_session): _seed_feed(db_session, DEPRECATED, status="deprecated") _seed_feed(db_session, UNPUBLISHED, operational_status="unpublished") _seed_feed(db_session, INACTIVE, status="inactive") + _seed_feed(db_session, TRACKED) db_session.commit() @with_db_session(db_url=default_db_url) @@ -236,6 +299,38 @@ def seal_row(feed_id, db_session): def set_official(feed_id, official, db_session): _set_official(db_session, feed_id, official) + @staticmethod + @with_db_session(db_url=default_db_url) + def set_seasonal(feed_id, seasonal, db_session): + _set_seasonal(db_session, feed_id, seasonal) + + @staticmethod + @with_db_session(db_url=default_db_url) + def set_producer_url_unstable(feed_id, unstable, db_session): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(is_producer_url_unstable=unstable) + ) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def seed_dataset( + feed_id, coverage_end, downloaded_at=None, suffix="", db_session=None + ): + _seed_dataset(db_session, feed_id, coverage_end, downloaded_at, suffix) + + @staticmethod + @with_db_session(db_url=default_db_url) + def set_feed_created_at(feed_id, created_at, db_session): + db_session.execute( + Feed.__table__.update() + .where(Feed.__table__.c.id == feed_id) + .values(created_at=created_at) + ) + db_session.commit() + def _feeds_by_stable_id(db_session, *stable_ids): """Plain by-id load — mirrors what `update_seals` does before checking eligibility.""" @@ -310,6 +405,95 @@ def test_loads_the_fields_the_evaluators_need(self, db_session): self.assertEqual(ctx.stable_id, OFFICIAL) self.assertTrue(ctx.official) self.assertEqual(ctx.now, NOW) + self.assertFalse(ctx.seasonal) + self.assertIsNone(ctx.is_producer_url_unstable) + + @with_db_session(db_url=default_db_url) + def test_stables_clock_is_the_feed_row_and_needs_no_query(self, db_session): + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.feed_created_at, NOW - timedelta(days=400)) + + @with_db_session(db_url=default_db_url) + def test_a_feed_with_no_dataset_says_so(self, db_session): + """The bulk load misses, and the context says so rather than guessing a value.""" + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_dataset) + + @with_db_session(db_url=default_db_url) + def test_the_latest_dataset_coverage_is_loaded(self, db_session): + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual(ctx.latest_dataset.dataset_id, f"{TRACKED}_dataset") + self.assertEqual( + ctx.latest_dataset.service_date_range_end, NOW + timedelta(days=90) + ) + + @with_db_session(db_url=default_db_url) + def test_a_dataset_with_no_coverage_end_is_not_a_missing_dataset(self, db_session): + """The two UNKNOWN cases must stay distinguishable at the context layer.""" + _seed_dataset(db_session, TRACKED, coverage_end=None) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNotNone(ctx.latest_dataset, "the dataset is there ...") + self.assertIsNone( + ctx.latest_dataset.service_date_range_end, "... its coverage end is not" + ) + + @with_db_session(db_url=default_db_url) + def test_the_latest_dataset_is_resolved_as_of_now(self, db_session): + """A replay must not see a dataset published after the day it is evaluating. + + `gtfsfeed.latest_dataset_id` points at the newest dataset that exists today, so + reading it would report the feed as fresh on a day when the data covering that day + had not been published yet. + """ + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=5), + downloaded_at=NOW - timedelta(days=10), + suffix="_old", + ) + _seed_dataset( + db_session, + TRACKED, + coverage_end=NOW + timedelta(days=400), + downloaded_at=NOW + timedelta(days=10), + suffix="_new", + ) + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + + as_of_now = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertEqual( + as_of_now.latest_dataset.service_date_range_end, + NOW + timedelta(days=5), + "the newer dataset had not been downloaded yet", + ) + + later = NOW + timedelta(days=20) + as_of_later = build_contexts(db_session, feeds, later)[feeds[0].id] + self.assertEqual( + as_of_later.latest_dataset.service_date_range_end, + NOW + timedelta(days=400), + "by then it had", + ) + + @with_db_session(db_url=default_db_url) + def test_a_dataset_with_no_downloaded_at_cannot_be_placed_in_time(self, db_session): + """It is excluded rather than guessed at: we cannot say whether it existed yet.""" + _seed_dataset(db_session, TRACKED, coverage_end=NOW + timedelta(days=90)) + db_session.execute( + Gtfsdataset.__table__.update() + .where(Gtfsdataset.__table__.c.feed_id == TRACKED) + .values(downloaded_at=None) + ) + db_session.commit() + feeds = list(_feeds_by_stable_id(db_session, TRACKED).values()) + ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + self.assertIsNone(ctx.latest_dataset) @with_db_session(db_url=default_db_url) def test_builds_one_context_per_feed(self, db_session): @@ -319,7 +503,10 @@ def test_builds_one_context_per_feed(self, db_session): self.assertEqual({ctx.official for ctx in contexts.values()}, {True, False}) +@patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestUpdateSeals(SealDbTestCase): + """The report, the roll-up and the two seal transitions, driven by Official alone.""" + def test_dry_run_writes_nothing(self): report = update_seals(dry_run=True, stable_feed_ids=[OFFICIAL], now=NOW) self.assertTrue(report["dry_run"]) @@ -508,7 +695,11 @@ def test_regaining_official_status_clears_the_criterion(self): self.assertEqual(seal.seal_earned_at, later) def test_partial_criteria_run_skips_the_roll_up(self): - """Named explicitly, `official` is still the whole registry, so not partial.""" + """Naming every criterion in the registry is not a partial run. + + Under this class's patch the registry is Official alone; `TestCriteriaSelection` + covers the same distinction against the real, larger registry. + """ report = update_seals( dry_run=False, stable_feed_ids=[OFFICIAL], @@ -526,21 +717,6 @@ def test_a_feed_list_is_required(self): update_seals(dry_run=True, stable_feed_ids=feeds, now=NOW) self.assertIn("stable_feed_ids", str(caught.exception)) - def test_unknown_criterion_raises(self): - with self.assertRaises(ValueError): - update_seals( - stable_feed_ids=[OFFICIAL], criteria=["not_a_criterion"], now=NOW - ) - - def test_criterion_without_an_evaluator_raises(self): - """`stable` is a valid DB enum value but has no evaluator yet (#1784).""" - with self.assertRaises(ValueError): - update_seals( - stable_feed_ids=[OFFICIAL], - criteria=[SealCriterionName.STABLE.value], - now=NOW, - ) - def test_a_run_with_no_usable_feed_raises(self): """Nothing was evaluated, so a report saying so would be too quiet.""" with self.assertRaises(ValueError) as caught: @@ -591,6 +767,7 @@ def test_feeds_omitted_is_zero_when_nothing_was_dropped(self): self.assertEqual(report["feeds_omitted"], 0) +@patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", ONLY_OFFICIAL) class TestCriterionSnapshot(SealDbTestCase): """seal_criterion_snapshot, the per-day record of each criterion (issue #1809). @@ -695,6 +872,259 @@ def test_the_log_covers_every_field_the_state_carries(self): self.assertEqual(carried - recorded, set()) +class TestFullRegistry(SealDbTestCase): + """Official, Stable and Fresh together, against real rows and the real registry. + + The classes above patch the registry down to Official because they are about the + report, the roll-up and the snapshot table rather than about any one criterion. These + tests are the other half: the three implemented criteria, driven by the columns they + actually read. + """ + + FAR_FUTURE = NOW + timedelta(days=90) + + def criteria_of(self, feed_id=TRACKED): + return { + criterion: row.confirmed_status + for criterion, row in self.criterion_rows(feed_id).items() + } + + def test_every_criterion_is_written_for_every_feed(self): + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertEqual( + set(self.criterion_rows(TRACKED)), + {evaluator.name.value for evaluator in EVALUATORS}, + ) + + def test_a_feed_meeting_all_three_earns_the_seal(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of(), + { + SealCriterionName.OFFICIAL.value: CriterionStatus.PASS.value, + SealCriterionName.STABLE.value: CriterionStatus.PASS.value, + SealCriterionName.FRESH_COVERAGE.value: CriterionStatus.PASS.value, + }, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_feed_new_to_the_database_cannot_hold_the_seal_yet(self): + """Stable reads the feed's own age, so a freshly added feed fails it.""" + self.set_feed_created_at(TRACKED, NOW - timedelta(days=30)) + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.FAIL.value, + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_the_seal_arrives_once_the_feed_is_old_enough(self): + """The same feed, evaluated the day it was added and 181 days later. + + Stable's clock is `feed.created_at`, which does not move when the job runs, so the + second run is a plain replay at a later `now` rather than something the first run + had to set up. + """ + self.set_feed_created_at(TRACKED, NOW) + self.seed_dataset(TRACKED, NOW + timedelta(days=400)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertFalse( + self.seal_row(TRACKED).has_seal, "in the database for zero days" + ) + + later = NOW + timedelta(days=181) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.STABLE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.PASS.value) + self.assertIsNone(row.probation_start, "Stable serves no probation") + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_an_old_feed_qualifies_on_its_very_first_run(self): + """The point of reading `feed.created_at`: no six-month wait after deployment for a + feed that has already been in the catalog for years.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.PASS.value, + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_an_unstable_producer_url_denies_the_seal_immediately(self): + """Stable has no grace period, so the flag costs the seal the day it is set.""" + self.seed_dataset(TRACKED, self.FAR_FUTURE) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + self.set_producer_url_unstable(TRACKED, True) + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + self.assertEqual( + self.criteria_of()[SealCriterionName.STABLE.value], + CriterionStatus.FAIL.value, + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_feed_with_no_dataset_leaves_fresh_out_of_the_roll_up(self): + """UNKNOWN is not a failure: the other two criteria still decide the seal.""" + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.NEVER_EVALUATED.value, + "no verdict was ever produced, so the criterion is out of service", + ) + self.assertIsNone(row.last_verdict_at) + self.assertTrue( + self.seal_row(TRACKED).has_seal, + "Official and Stable carry it while Fresh has nothing to say", + ) + + def test_lapsed_coverage_is_confirmed_at_once_on_a_first_evaluation(self): + """Fresh has a 14-day grace period, but a criterion that has never passed has not + earned it, so its first verdict lands as a confirmed failure the same day.""" + self.seed_dataset(TRACKED, NOW + timedelta(days=2)) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_the_grace_period_absorbs_a_lapse_on_a_feed_that_was_passing(self): + self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + # Five days on, the same dataset now covers only five more days: inside the horizon. + within_grace = NOW + timedelta(days=5) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=within_grace) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "the grace period holds the verdict while the producer catches up", + ) + self.assertEqual(row.first_observed_failure_at, within_grace) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_lapse_outlasting_the_grace_period_costs_the_seal(self): + self.seed_dataset(TRACKED, NOW + timedelta(days=10)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + first_failure = NOW + timedelta(days=5) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=first_failure) + + outlasted = first_failure + timedelta(days=15) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=outlasted) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.first_observed_failure_at, + first_failure, + "the streak is measured from its start, not from this run", + ) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + def test_a_seasonal_feed_is_not_denied_by_fresh(self): + """NOT_APPLICABLE withdraws the criterion instead of failing it, which is the whole + point of the value: a seasonal feed keeps the seal on the criteria that do apply. + """ + self.set_seasonal(TRACKED, True) + + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.observed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertEqual(row.confirmed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_becoming_seasonal_freezes_a_failing_fresh_rather_than_carrying_it(self): + """A feed marked seasonal after a confirmed Fresh failure stops being judged on it.""" + self.seed_dataset(TRACKED, NOW - timedelta(days=1)) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=NOW) + self.assertFalse(self.seal_row(TRACKED).has_seal) + + self.set_seasonal(TRACKED, True) + later = NOW + timedelta(days=1) + update_seals(dry_run=False, stable_feed_ids=[TRACKED], now=later) + + row = self.criterion_rows(TRACKED)[SealCriterionName.FRESH_COVERAGE.value] + self.assertEqual(row.confirmed_status, CriterionStatus.NOT_APPLICABLE.value) + self.assertEqual( + row.last_confirmed_failure_at, NOW, "the failure stays on record" + ) + self.assertTrue(self.seal_row(TRACKED).has_seal) + + def test_a_run_reports_all_three_criteria_with_their_reasons(self): + self.seed_dataset(TRACKED, self.FAR_FUTURE) + report = update_seals(dry_run=True, stable_feed_ids=[TRACKED], now=NOW) + + criteria = report["feeds"][0]["criteria"] + self.assertEqual( + [row["criterion"] for row in criteria], + [evaluator.name.value for evaluator in EVALUATORS], + ) + for row in criteria: + with self.subTest(criterion=row["criterion"]): + self.assertTrue(row["reason"]) + + +class TestCriteriaSelection(SealDbTestCase): + """The `criteria` filter, against the real registry rather than a patched one.""" + + def test_unknown_criterion_raises(self): + with self.assertRaises(ValueError): + update_seals( + stable_feed_ids=[OFFICIAL], criteria=["not_a_criterion"], now=NOW + ) + + def test_criterion_without_an_evaluator_raises(self): + """`fresh_continuous` is a valid DB enum value but has no evaluator yet (#1782).""" + with self.assertRaises(ValueError): + update_seals( + stable_feed_ids=[OFFICIAL], + criteria=[SealCriterionName.FRESH_CONTINUOUS.value], + now=NOW, + ) + + def test_naming_every_implemented_criterion_is_not_a_partial_run(self): + report = update_seals( + dry_run=True, + stable_feed_ids=[OFFICIAL], + criteria=[evaluator.name.value for evaluator in EVALUATORS], + now=NOW, + ) + self.assertFalse(report["partial_run"]) + + def test_naming_a_subset_is_a_partial_run(self): + """More than one criterion is implemented now, so a subset is genuinely partial.""" + report = update_seals( + dry_run=True, + stable_feed_ids=[OFFICIAL], + criteria=[SealCriterionName.OFFICIAL.value], + now=NOW, + ) + self.assertTrue(report["partial_run"]) + self.assertIn("note", report) + + @patch("tasks.seal_of_reliability.seal_updater.EVALUATORS", WITH_PROBATION) class TestProbation(SealDbTestCase): """Probation persisted and rolled up, driven by `_StandInEvaluator`. @@ -1003,7 +1433,9 @@ class TestCriterionThatStopsApplying(SealDbTestCase): FAILED_AT = NOW + timedelta(days=1) def stand_in(self): - return self.criterion_rows(OFFICIAL).get(SealCriterionName.FRESH_COVERAGE.value) + return self.criterion_rows(OFFICIAL).get( + SealCriterionName.FRESH_CONTINUOUS.value + ) def run_at(self, moment): update_seals(dry_run=False, stable_feed_ids=[OFFICIAL], now=moment)