From ecde1bd9212e956154c2224dc833d3f6b7d56200 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 25 Aug 2026 13:58:02 -0400 Subject: [PATCH 01/13] Add stubbed backfill_seal_of_reliability task --- functions-python/tasks_executor/src/main.py | 22 ++ .../backfill/backfill_seal_of_reliability.py | 135 +++++++ .../backfill/seal_backfill.py | 339 ++++++++++++++++++ .../backfill/test_seal_backfill.py | 317 ++++++++++++++++ 4 files changed, 813 insertions(+) create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py diff --git a/functions-python/tasks_executor/src/main.py b/functions-python/tasks_executor/src/main.py index f1ff61b85..c05bce528 100644 --- a/functions-python/tasks_executor/src/main.py +++ b/functions-python/tasks_executor/src/main.py @@ -76,6 +76,10 @@ notifications_dispatch_monitor_handler, ) from tasks.changelog.backfill_changelog import backfill_changelog_handler +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import ( + backfill_seal_of_reliability_handler, +) + from tasks.seal_of_reliability.update_seal_of_reliability import ( update_seal_of_reliability_handler, ) @@ -302,6 +306,24 @@ ), "handler": update_seal_of_reliability_handler, }, + "backfill_seal_of_reliability": { + "description": ( + "Establishes a starting Seal of Reliability state for feeds that have none " + "(issue #1763), by cold-starting each feed 12 months back and replaying the " + "nightly evaluation forward one day at a time to end_date, writing only the " + "final day. NOT YET IMPLEMENTED: the day march raises, so dry_run=true is " + "the only mode that returns — it resolves and reports the plan. " + "Parameters: stable_feed_ids (required, non-empty), start_date (ISO date, " + "default end_date minus days_back; clamped up to each feed's created_at), " + "end_date (ISO date, default yesterday UTC), days_back (default 365), " + "dry_run (default true), limit (default null), criteria (default null " + "meaning every implemented criterion), batch_size (default 200), " + "only_missing (default true; skips feeds that already have seal state), " + "snapshot_mode (final|all|none, default final), resume_from_snapshot " + "(default false; the #1803 hook), max_reported_feeds (default 50)." + ), + "handler": backfill_seal_of_reliability_handler, + }, "seal_orchestrator": { "description": ( "Cloud Tasks producer for the nightly Seal of Reliability run across the " diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py new file mode 100644 index 000000000..753b16457 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -0,0 +1,135 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Task entry point for the Seal of Reliability backfill (issue #1763).""" + +from datetime import date, datetime +from typing import Optional + +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + DEFAULT_SNAPSHOT_MODE, + backfill_seals, +) +from tasks.seal_of_reliability.seal_updater import ( + DEFAULT_BATCH_SIZE, + DEFAULT_MAX_REPORTED_FEEDS, +) + + +def _parse_day(value: Optional[str], field: str) -> Optional[date]: + """Parse a payload date string to a `date`, or None when it is absent. + + A plain date is what the window is expressed in, but an operator copying a value from a + log or from the nightly task's `now` will paste a full timestamp. Accept both rather than + failing on a value whose meaning is unambiguous; the time of day is dropped either way, + since the march is day-granular. + """ + if value is None: + return None + try: + return date.fromisoformat(value) + except ValueError: + pass + try: + return datetime.fromisoformat(value).date() + except ValueError: + raise ValueError( + f"{field} must be an ISO date such as 2026-01-31, got {value!r}" + ) + + +def get_parameters(payload: dict): + """Read the task parameters from the payload, applying defaults.""" + payload = payload or {} + return ( + payload.get("stable_feed_ids"), + _parse_day(payload.get("start_date"), "start_date"), + _parse_day(payload.get("end_date"), "end_date"), + payload.get("days_back", DEFAULT_DAYS_BACK), + payload.get("dry_run", True), + payload.get("limit", None), + payload.get("criteria", None), + payload.get("batch_size", DEFAULT_BATCH_SIZE), + payload.get("only_missing", True), + payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE), + payload.get("resume_from_snapshot", False), + payload.get("max_reported_feeds", DEFAULT_MAX_REPORTED_FEEDS), + ) + + +def backfill_seal_of_reliability_handler(payload: dict) -> dict: + """ + Handler for the Seal of Reliability backfill. + + The day march is not implemented yet: a dry run returns the resolved plan, and a non-dry + run raises rather than reporting a success that wrote nothing. + + Payload parameters: + stable_feed_ids (list[str]): Required and non-empty. The feeds to backfill; there is + no run-the-whole-catalogue mode. Ineligible ids are skipped with a + logged warning, and it raises if none can be used. + start_date (str | None): First day of the window, ISO date. Clamped up to each feed's + own created_at. Default: end_date minus days_back. + end_date (str | None): Last day simulated, and the day the written state belongs to. + Resolved once for the whole run. Default: yesterday UTC. + days_back (int): Window length used when start_date is absent. Default: 365 — the + "12 months" of #1763, a cost/coverage default rather than a + correctness threshold. + dry_run (bool): Resolve and return the plan without marching or writing. + Default: True. + limit (int | None): Cap the number of feeds, from the list. Default: no limit. + criteria (list[str] | None): Backfill only these criteria. Default: None, meaning + every implemented criterion. + batch_size (int): Feeds loaded and marched per batch. Default: 200. + only_missing (bool): Skip feeds that already have seal state, which is #1763's stated + scope. False re-backfills them, overwriting what is stored. + Default: True. + snapshot_mode (str): "final" (only the last day, per #1763), "all" (every simulated + day — millions of rows over a year, but what would let #1803 resume + inside the backfilled window), or "none". Default: "final". + resume_from_snapshot (bool): Seed each criterion from its snapshot at march_start - 1 + rather than cold-starting empty. The #1803 hook. Default: False. + max_reported_feeds (int): Cap on the `feeds` list in the response; `feeds_omitted` + reports how many entries were left out. Default: 50. + """ + ( + stable_feed_ids, + start_date, + end_date, + days_back, + dry_run, + limit, + criteria, + batch_size, + only_missing, + snapshot_mode, + resume_from_snapshot, + max_reported_feeds, + ) = get_parameters(payload) + return backfill_seals( + stable_feed_ids=stable_feed_ids, + start_date=start_date, + end_date=end_date, + days_back=days_back, + dry_run=dry_run, + limit=limit, + criteria=criteria, + batch_size=batch_size, + only_missing=only_missing, + snapshot_mode=snapshot_mode, + resume_from_snapshot=resume_from_snapshot, + max_reported_feeds=max_reported_feeds, + ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py new file mode 100644 index 000000000..af79e3808 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -0,0 +1,339 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Seal of Reliability backfill (issue #1763). + +Establishes a starting seal state for feeds that have none, so the nightly job (#1761) has a +"yesterday" to step from. For each feed it cold-starts at `march_start`, replays the nightly +evaluation forward one day at a time to `end_date`, and writes only the final day. The +intermediate days are held in memory and discarded — marching forward is what builds up the +path-dependent state (grace-period streaks, probation) that makes the final state right. + +STATUS: the invocation is complete and validated; **the day march itself is not implemented**. +A dry run resolves and returns the full plan — which feeds, which window per feed, how many +days — and a non-dry run raises rather than silently writing nothing. See `_march` below. + +Per-feed window +--------------- +`march_start = max(start_date, feed.created_at)`. Clamping to the feed's own creation date +does two things: it skips days before the feed existed, and it is the value the Stable +criterion measures its 180 days from. A feed younger than the window therefore gets an exact +cold start rather than a guessed one — there is no history before its creation to be wrong +about. + +`end_date` is resolved once by the caller and passed down, never recomputed per feed. Two +workers of the same run started either side of midnight would otherwise march to different +final days. + +What the backfill cannot know +----------------------------- +Official and Stable have no historical record, so they can only be evaluated against their +current values. Neither has a grace period or probation, so a wrong value on a past day does +not propagate into the days after it (see #1763). + +The cold start assumes an empty prior state — no failure streak, no probation — which may not +match reality for a feed whose history is truncated by `start_date`. Errors from that +assumption are not bounded by the window: a single observed failure inside it can extend the +divergence by another probation period, and repeatedly. The window is therefore a +cost/coverage default, not a correctness guarantee. +""" + +import logging +from datetime import date, datetime, timedelta, timezone +from typing import Dict, List, Optional, Sequence, Set, Tuple + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Gtfsfeed, SealCriterion + +from tasks.seal_of_reliability.context import is_seal_eligible +from tasks.seal_of_reliability.seal_updater import ( + DEFAULT_BATCH_SIZE, + DEFAULT_MAX_REPORTED_FEEDS, + _resolve_evaluators, + _validate_requested_feed_ids, +) + +logger = logging.getLogger(__name__) + +# How far back the window reaches when `start_date` is not given. Expressed in days rather +# than months so the arithmetic is exact and needs no calendar library: 365 is the "12 months" +# of #1763. It is roughly twice the 180-day probation period, which is where the number came +# from — but see the module docstring: that reasoning bounds nothing, so treat this as a +# default for how much history to replay rather than as a correctness threshold. +DEFAULT_DAYS_BACK: int = 365 + +# What to record in seal_criterion_snapshot. +# final — only the last day's state, per #1763. The intermediate days are discarded. +# all — every simulated day. Costs len(days) x feeds x criteria rows, which is millions +# over a year, but it is what would let #1803 resume inside the backfilled window +# instead of cold-starting again. +# none — write nothing to the snapshot table. +SNAPSHOT_MODES: Tuple[str, ...] = ("final", "all", "none") +DEFAULT_SNAPSHOT_MODE: str = "final" + +CRITERION_TABLE = SealCriterion.__table__ + + +def yesterday_utc() -> date: + """The default `end_date`: the last day that is fully over in UTC.""" + return datetime.now(timezone.utc).date() - timedelta(days=1) + + +def resolve_window( + start_date: Optional[date], + end_date: Optional[date], + days_back: int, +) -> Tuple[date, date]: + """Resolve the run-wide window, applying defaults and rejecting a nonsensical one. + + Resolved once for the whole run rather than per feed, so every feed of a run marches to + the same final day whatever time the run started or how long it takes. + """ + if days_back <= 0: + raise ValueError("days_back must be a positive integer") + + resolved_end = end_date or yesterday_utc() + resolved_start = start_date or (resolved_end - timedelta(days=days_back)) + + if resolved_start > resolved_end: + raise ValueError( + f"start_date ({resolved_start.isoformat()}) is after end_date " + f"({resolved_end.isoformat()})" + ) + return resolved_start, resolved_end + + +def march_start_for(feed: Gtfsfeed, start_date: date) -> date: + """Where this feed's march begins: the later of the window start and its creation. + + Clamping to `feed.created_at` is not only an optimisation. It is also the value the + Stable criterion counts its 180 days from, and it is what makes the cold start exact for + a feed younger than the window: such a feed has no history before its creation, so the + empty starting state is the truth rather than an assumption. + """ + created = feed.created_at + if created is None: + # created_at is NOT NULL in the schema, so this is defensive only: a feed with no + # creation date gets the full window rather than being skipped. + return start_date + created_day = ( + created.astimezone(timezone.utc).date() + if created.tzinfo is not None + else created.date() + ) + return max(start_date, created_day) + + +def _feeds_with_seal_state(db_session: Session, feed_ids: Sequence[str]) -> Set[str]: + """The subset of `feed_ids` that already has at least one seal_criterion row. + + `only_missing` filters on this: #1763 backfills feeds that have no stored state to carry + forward, and re-running the march over a feed the nightly job already owns would throw + away real history in favour of a simulation of it. + """ + if not feed_ids: + return set() + rows = db_session.execute( + select(CRITERION_TABLE.c.feed_id) + .where(CRITERION_TABLE.c.feed_id.in_(list(feed_ids))) + .distinct() + ).all() + return {row.feed_id for row in rows} + + +def _march( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + evaluators: Sequence, + snapshot_mode: str, + resume_from_snapshot: bool, +) -> List[dict]: + """Replay the nightly evaluation day by day and write the final state. NOT IMPLEMENTED. + + What this has to do, once built: + + 1. Ask each criterion to bulk-load its inputs for the whole day range at once — one load + per criterion per batch, never one per day. A year marched with per-day queries turns + a handful of queries into several thousand. + 2. Seed each (feed, criterion) with an empty `SealCriterionState`, or with the state read + from `seal_criterion_snapshot` at `march_start - 1` when `resume_from_snapshot` is set + (#1803). + 3. For each day in order, evaluate every criterion and apply `state_machine.transition`, + threading the returned state into the next day in memory. Nothing is written per day. + 4. Roll up `has_seal` on the final day and upsert `seal_criterion` and + `feed_reliability_seal`, plus `seal_criterion_snapshot` per `snapshot_mode`. + 5. Write `feed_reliability_seal.created_at = march_start` on insert only, so Stable counts + from the right day and a re-backfill cannot reset a countdown already running. + + Two behaviours are still undecided and must be settled before this is built: + + * `seal_earned_at` — as `_upsert_seals` stands it would be stamped with the write time, + so every backfilled feed would look like it earned its seal on backfill day. The march + knows the day the roll-up actually flipped and could stamp that instead. + * Whether the evaluators can answer for a past day at all. They currently read the + current feed row only; each criterion needs to own its own historical lookup first. + """ + raise NotImplementedError( + "The Seal of Reliability day march is not implemented yet (#1763). Run with " + "dry_run=true to resolve and inspect the plan." + ) + + +@with_db_session +def backfill_seals( + db_session: Session, + stable_feed_ids: Sequence[str], + start_date: Optional[date] = None, + end_date: Optional[date] = None, + days_back: int = DEFAULT_DAYS_BACK, + dry_run: bool = True, + limit: Optional[int] = None, + criteria: Optional[Sequence[str]] = None, + batch_size: int = DEFAULT_BATCH_SIZE, + only_missing: bool = True, + snapshot_mode: str = DEFAULT_SNAPSHOT_MODE, + resume_from_snapshot: bool = False, + max_reported_feeds: int = DEFAULT_MAX_REPORTED_FEEDS, +) -> dict: + """Plan and (once `_march` exists) run the backfill for the requested feeds. + + Like `update_seals`, this always runs against an explicit list of feeds — enumerating the + catalogue is a producer's job, not this function's. + + Args: + db_session: SQLAlchemy session, injected by @with_db_session. + stable_feed_ids: The feeds to backfill. Required and non-empty. Unknown or ineligible + ids are skipped with a logged warning; it raises only if none can be used. + start_date: First day of the window. Clamped up to each feed's `created_at`. Defaults + to `end_date - days_back`. + end_date: Last day simulated, and the day the written state belongs to. Defaults to + yesterday UTC. Resolved once here so every feed of a run ends on the same day. + days_back: Window length used when `start_date` is absent. Default 365. + dry_run: Resolve and return the plan without marching or writing. Default True. + limit: Cap the number of feeds, applied to the requested list. + criteria: Backfill only these criteria. Same names as the nightly task. + batch_size: Feeds loaded and marched per batch. + only_missing: Skip feeds that already have seal state, which is #1763's stated scope. + Set False to re-backfill a feed and overwrite what is stored. + snapshot_mode: One of `SNAPSHOT_MODES` — how much of the march to record in + seal_criterion_snapshot. Default "final". + resume_from_snapshot: Seed each criterion from its snapshot at `march_start - 1` + rather than cold-starting empty. The #1803 hook; requires snapshots to exist. + max_reported_feeds: Cap on the `feeds` list in the report. + + Returns: + A plan report. `days` is the longest march in the run; feeds clamped to their own + `created_at` march fewer. + """ + if not stable_feed_ids: + raise ValueError("stable_feed_ids is required and must be non-empty") + if snapshot_mode not in SNAPSHOT_MODES: + raise ValueError( + f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" + ) + if batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + + window_start, window_end = resolve_window(start_date, end_date, days_back) + evaluators = _resolve_evaluators(criteria) + + # Plain by-id load, then eligibility in Python on the loaded rows — the same shape as + # `update_seals`, so a feed that does not exist can be told apart from one that exists + # but is not eligible without a second query. + query = db_session.query(Gtfsfeed).filter( + Gtfsfeed.stable_id.in_(list(stable_feed_ids)) + ) + if limit is not None: + query = query.limit(limit) + feeds = query.all() + eligible = [feed for feed in feeds if is_seal_eligible(feed)] + + already_backfilled = ( + _feeds_with_seal_state(db_session, [feed.id for feed in eligible]) + if only_missing + else set() + ) + selected = [feed for feed in eligible if feed.id not in already_backfilled] + + _validate_requested_feed_ids( + stable_feed_ids, + found={feed.stable_id for feed in feeds}, + evaluated={feed.stable_id for feed in eligible}, + ) + + windows = { + feed.id: (march_start_for(feed, window_start), window_end) for feed in selected + } + longest_march = ( + max((end - start).days + 1 for start, end in windows.values()) if windows else 0 + ) + + feed_plans = [ + { + "stable_id": feed.stable_id, + "march_start": windows[feed.id][0].isoformat(), + "end_date": window_end.isoformat(), + "days": (window_end - windows[feed.id][0]).days + 1, + # The march start doubles as the Stable criterion's anchor, and is what + # feed_reliability_seal.created_at will be set to on insert. + "tracking_start": windows[feed.id][0].isoformat(), + } + for feed in selected + ] + + report = { + "message": ( + f"Planned a backfill of {len(selected)} feed(s) across " + f"{len(evaluators)} criterion/criteria, ending {window_end.isoformat()}." + ), + "dry_run": dry_run, + "implemented": False, + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "days": longest_march, + "total_feeds": len(selected), + "skipped_already_backfilled": len(already_backfilled), + "criteria": [evaluator.name.value for evaluator in evaluators], + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + "batch_size": batch_size, + } + report["feeds"] = feed_plans[:max_reported_feeds] + report["feeds_omitted"] = max(0, len(feed_plans) - max_reported_feeds) + + logger.info( + "Backfill plan: %s", + {key: value for key, value in report.items() if key != "feeds"}, + ) + + if not dry_run: + # Raise rather than return a report that looks like a completed run. Until `_march` + # exists there is nothing to write, and reporting success for that would be worse + # than failing. + _march( + db_session, + selected, + windows, + evaluators, + snapshot_mode, + resume_from_snapshot, + ) + + return report diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py new file mode 100644 index 000000000..e5142166c --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -0,0 +1,317 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for the Seal of Reliability backfill (#1763): parameters, window, and plan. + +The day march is not implemented, so these cover the invocation surface — what the payload +resolves to, which feeds are selected, and that a non-dry run refuses rather than reporting +a success that wrote nothing. +""" + +import unittest +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Optional + +from sqlalchemy import delete, insert + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import ( + _parse_day, + backfill_seal_of_reliability_handler, + get_parameters, +) +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + backfill_seals, + march_start_for, + resolve_window, + yesterday_utc, +) +from test_shared.test_utils.database_utils import default_db_url + +PREFIX = "seal_bf_" +OLD = f"{PREFIX}old" # created well before any window we test +YOUNG = f"{PREFIX}young" # created inside the window, so its march is clamped +DEPRECATED = f"{PREFIX}deprecated" # not seal-eligible +ALREADY = f"{PREFIX}already" # already has seal state + +END = date(2026, 6, 1) +START = date(2025, 6, 1) + +NOW = datetime(2026, 6, 2, 12, 0, tzinfo=timezone.utc) +OLD_CREATED = NOW - timedelta(days=800) +YOUNG_CREATED = NOW - timedelta(days=90) + + +@dataclass +class _FeedStub: + """Just enough of a feed row for `march_start_for`, which reads only `created_at`.""" + + created_at: Optional[datetime] + + +class TestParseDay(unittest.TestCase): + def test_plain_iso_date(self): + self.assertEqual(_parse_day("2026-01-31", "start_date"), date(2026, 1, 31)) + + def test_timestamp_is_accepted_and_truncated(self): + """An operator pasting the nightly task's `now` should not hit a parse error. + + The march is day-granular, so the time of day is dropped either way. + """ + for value in ("2026-01-31T12:00:00", "2026-01-31T12:00:00+00:00"): + with self.subTest(value=value): + self.assertEqual(_parse_day(value, "end_date"), date(2026, 1, 31)) + + def test_absent_stays_none(self): + self.assertIsNone(_parse_day(None, "start_date")) + + def test_garbage_names_the_field(self): + with self.assertRaises(ValueError) as caught: + _parse_day("last tuesday", "start_date") + self.assertIn("start_date", str(caught.exception)) + + +class TestGetParameters(unittest.TestCase): + def test_defaults(self): + ( + stable_feed_ids, + start_date, + end_date, + days_back, + dry_run, + limit, + criteria, + batch_size, + only_missing, + snapshot_mode, + resume_from_snapshot, + max_reported_feeds, + ) = get_parameters({"stable_feed_ids": ["a"]}) + + self.assertEqual(stable_feed_ids, ["a"]) + self.assertIsNone(start_date) + self.assertIsNone(end_date) + self.assertEqual(days_back, DEFAULT_DAYS_BACK) + self.assertTrue(dry_run, "a backfill must not write unless asked to") + self.assertIsNone(limit) + self.assertIsNone(criteria) + self.assertEqual(batch_size, 200) + self.assertTrue(only_missing, "#1763 backfills feeds that have no state yet") + self.assertEqual(snapshot_mode, "final") + self.assertFalse(resume_from_snapshot) + self.assertEqual(max_reported_feeds, 50) + + def test_empty_payload_does_not_raise_here(self): + """Validation belongs to the engine, so the parser stays a plain reader.""" + self.assertIsNone(get_parameters({})[0]) + + +class TestResolveWindow(unittest.TestCase): + def test_both_given_are_kept(self): + self.assertEqual(resolve_window(START, END, DEFAULT_DAYS_BACK), (START, END)) + + def test_end_defaults_to_yesterday(self): + _, resolved_end = resolve_window(START, None, DEFAULT_DAYS_BACK) + self.assertEqual(resolved_end, yesterday_utc()) + + def test_start_defaults_to_days_back_from_end(self): + resolved_start, _ = resolve_window(None, END, 365) + self.assertEqual(resolved_start, END - timedelta(days=365)) + + def test_start_after_end_is_rejected(self): + with self.assertRaises(ValueError) as caught: + resolve_window(END + timedelta(days=1), END, DEFAULT_DAYS_BACK) + self.assertIn("after end_date", str(caught.exception)) + + def test_non_positive_days_back_is_rejected(self): + with self.assertRaises(ValueError): + resolve_window(None, END, 0) + + +class TestMarchStart(unittest.TestCase): + def test_feed_older_than_the_window_starts_at_the_window(self): + feed = _FeedStub(created_at=datetime(2020, 1, 1, tzinfo=timezone.utc)) + self.assertEqual(march_start_for(feed, START), START) + + def test_feed_younger_than_the_window_starts_at_its_creation(self): + """A feed with no history before its creation gets an exact cold start, not a guess.""" + created = datetime(2025, 9, 15, 8, 30, tzinfo=timezone.utc) + feed = _FeedStub(created_at=created) + self.assertEqual(march_start_for(feed, START), date(2025, 9, 15)) + + def test_naive_created_at_is_read_as_utc(self): + feed = _FeedStub(created_at=datetime(2025, 9, 15, 8, 30)) + self.assertEqual(march_start_for(feed, START), date(2025, 9, 15)) + + def test_missing_created_at_falls_back_to_the_window(self): + self.assertEqual(march_start_for(_FeedStub(created_at=None), START), START) + + +def _seed_feed(db_session, feed_id, created_at, status="active"): + db_session.add( + Gtfsfeed( + id=feed_id, + stable_id=feed_id, + data_type="gtfs", + status=status, + operational_status="published", + official=True, + created_at=created_at, + producer_url=f"https://example.com/{feed_id}.zip", + ) + ) + db_session.flush() + + +def _cleanup(db_session): + """Delete from `feed`, not `gtfsfeed`. + + Gtfsfeed is a joined-table subclass, so deleting the subclass leaves the parent row and + the next insert collides on feed_pkey. The seal tables are ON DELETE CASCADE. + """ + db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) + db_session.commit() + + +class BackfillDbTestCase(unittest.TestCase): + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + _cleanup(db_session) + _seed_feed(db_session, OLD, OLD_CREATED) + _seed_feed(db_session, YOUNG, YOUNG_CREATED) + _seed_feed(db_session, DEPRECATED, OLD_CREATED, status="deprecated") + _seed_feed(db_session, ALREADY, OLD_CREATED) + db_session.execute( + insert(SealCriterion.__table__).values( + feed_id=ALREADY, criterion="official" + ) + ) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + _cleanup(db_session) + + +class TestBackfillPlan(BackfillDbTestCase): + def test_dry_run_reports_the_resolved_window(self): + report = backfill_seals( + stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=True + ) + self.assertTrue(report["dry_run"]) + self.assertFalse(report["implemented"]) + self.assertEqual(report["start_date"], START.isoformat()) + self.assertEqual(report["end_date"], END.isoformat()) + self.assertEqual(report["days"], (END - START).days + 1) + self.assertEqual(report["total_feeds"], 1) + + def test_each_feed_marches_from_its_own_start(self): + """The window start is run-wide; the march start is per feed.""" + report = backfill_seals( + stable_feed_ids=[OLD, YOUNG], start_date=START, end_date=END, dry_run=True + ) + by_id = {entry["stable_id"]: entry for entry in report["feeds"]} + + self.assertEqual(by_id[OLD]["march_start"], START.isoformat()) + self.assertEqual(by_id[YOUNG]["march_start"], YOUNG_CREATED.date().isoformat()) + self.assertLess(by_id[YOUNG]["days"], by_id[OLD]["days"]) + + def test_march_start_is_also_the_stable_anchor(self): + report = backfill_seals( + stable_feed_ids=[YOUNG], start_date=START, end_date=END, dry_run=True + ) + entry = report["feeds"][0] + self.assertEqual(entry["tracking_start"], entry["march_start"]) + + def test_ineligible_feeds_are_left_out(self): + report = backfill_seals( + stable_feed_ids=[OLD, DEPRECATED], + start_date=START, + end_date=END, + dry_run=True, + ) + self.assertEqual( + [entry["stable_id"] for entry in report["feeds"]], + [OLD], + ) + + def test_only_missing_skips_a_feed_that_already_has_state(self): + report = backfill_seals( + stable_feed_ids=[OLD, ALREADY], + start_date=START, + end_date=END, + dry_run=True, + ) + self.assertEqual(report["total_feeds"], 1) + self.assertEqual(report["skipped_already_backfilled"], 1) + self.assertEqual([entry["stable_id"] for entry in report["feeds"]], [OLD]) + + def test_only_missing_false_re_backfills(self): + report = backfill_seals( + stable_feed_ids=[OLD, ALREADY], + start_date=START, + end_date=END, + dry_run=True, + only_missing=False, + ) + self.assertEqual(report["total_feeds"], 2) + self.assertEqual(report["skipped_already_backfilled"], 0) + + +class TestBackfillValidation(BackfillDbTestCase): + def test_a_real_run_refuses_rather_than_writing_nothing(self): + """Until the march exists, reporting success would be worse than failing.""" + with self.assertRaises(NotImplementedError) as caught: + backfill_seals( + stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=False + ) + self.assertIn("#1763", str(caught.exception)) + + def test_empty_feed_list_is_rejected(self): + with self.assertRaises(ValueError): + backfill_seals(stable_feed_ids=[], dry_run=True) + + def test_unknown_snapshot_mode_is_rejected(self): + with self.assertRaises(ValueError) as caught: + backfill_seals( + stable_feed_ids=[OLD], dry_run=True, snapshot_mode="occasionally" + ) + self.assertIn("occasionally", str(caught.exception)) + + def test_unknown_criteria_are_rejected(self): + with self.assertRaises(ValueError) as caught: + backfill_seals(stable_feed_ids=[OLD], dry_run=True, criteria=["punctual"]) + self.assertIn("punctual", str(caught.exception)) + + def test_handler_threads_the_payload_through(self): + report = backfill_seal_of_reliability_handler( + { + "stable_feed_ids": [OLD], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + "snapshot_mode": "all", + "resume_from_snapshot": True, + } + ) + self.assertEqual(report["snapshot_mode"], "all") + self.assertTrue(report["resume_from_snapshot"]) + self.assertEqual(report["start_date"], START.isoformat()) + + +if __name__ == "__main__": + unittest.main() From 4fdc30c760628ffbb0d4aa8fdbede129584ee75d Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 25 Aug 2026 14:07:45 -0400 Subject: [PATCH 02/13] Add scripted Compliant stand-in for path-dependent criteria --- .../backfill/test_scripted_compliant.py | 340 ++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py new file mode 100644 index 000000000..71336e787 --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py @@ -0,0 +1,340 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A scripted stand-in for the Compliant criterion, and the tests proving it drives. + +Compliant (#1761) has no evaluator yet, and Official — the only one that does — has neither +a grace period nor probation. So nothing currently exercises the path-dependent behaviour a +backfill (#1763) exists to reconstruct: a failure streak debounced by a grace period, a +confirmed failure, the probation that follows recovery, and an UNKNOWN day that freezes the +lot. This stand-in supplies it, with Compliant's real policy values. + +Why scripted by day rather than driven through the database, as the stand-ins in +`test_seal_updater_db.py` are: those read `ctx.official` and a test moves them by issuing an +UPDATE between runs. A backfill marches its days in memory with no writes in between, so a +criterion it can drive has to answer from `ctx.now` alone. `ComplianceScript` is that — a set +of failing days and a set of unknown days, fixed up front, replayed by simply advancing the +clock. +""" + +import unittest +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta, timezone +from typing import FrozenSet, Tuple +from unittest.mock import patch + +from sqlalchemy import delete, select + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion +from tasks.seal_of_reliability.criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.evaluators import CriterionEvaluator +from tasks.seal_of_reliability.seal_updater import update_seals +from test_shared.test_utils.database_utils import default_db_url + +# Compliant's published policy (#1761): a failure streak is held for 30 days before the +# status flips, and recovery from a confirmed failure serves the standard 180-day probation. +COMPLIANT_GRACE = timedelta(days=30) + +DAY_ZERO = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) + +PREFIX = "seal_sc_" +FEED = f"{PREFIX}compliant" + + +def day(offset: int) -> datetime: + """The run timestamp `offset` days after day zero.""" + return DAY_ZERO + timedelta(days=offset) + + +@dataclass(frozen=True) +class ComplianceScript: + """What the stand-in answers on each day, fixed before the run starts. + + Days named in neither set pass. Keyed by `date` rather than by offset so a script stays + readable next to the assertions that check it. + """ + + failing: FrozenSet[date] = field(default_factory=frozenset) + unknown: FrozenSet[date] = field(default_factory=frozenset) + + @classmethod + def failing_on(cls, *offsets: int) -> "ComplianceScript": + return cls(failing=frozenset(day(offset).date() for offset in offsets)) + + @classmethod + def failing_between(cls, first: int, last: int) -> "ComplianceScript": + """Inclusive of both ends, which is how a failure streak is described.""" + return cls.failing_on(*range(first, last + 1)) + + def with_unknown_on(self, *offsets: int) -> "ComplianceScript": + return ComplianceScript( + failing=self.failing, + unknown=frozenset(day(offset).date() for offset in offsets), + ) + + +class ScriptedCompliantEvaluator(CriterionEvaluator): + """A Compliant stand-in whose verdict is a pure function of the day being evaluated. + + Borrows the `compliant` enum value, which has no evaluator of its own yet. Carries + Compliant's real grace period and inherits the default probation period, so the + debouncing under test is the one that will actually ship. + """ + + name = SealCriterionName.COMPLIANT + grace_period = COMPLIANT_GRACE + + def __init__(self, script: ComplianceScript): + self.script = script + + def _evaluate(self, ctx) -> Tuple[CriterionStatus, str]: + today = ctx.now.astimezone(timezone.utc).date() + if today in self.script.unknown: + # No validation report for the latest dataset — the real Compliant's UNKNOWN + # case, and the reason a never-validated feed does not sit at a confirmed + # failure. + return CriterionStatus.UNKNOWN, f"scripted: no report on {today}" + if today in self.script.failing: + return CriterionStatus.FAIL, f"scripted: errors on {today}" + return CriterionStatus.PASS, f"scripted: clean on {today}" + + +@contextmanager +def registry(script: ComplianceScript): + """Run with the scripted stand-in as the only criterion. + + Sole occupant on purpose: `update_seals` treats a shorter evaluator list as a partial run + and skips the has_seal roll-up, so patching the registry itself rather than filtering it + is what keeps the seal in play. + """ + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedCompliantEvaluator(script)], + ): + yield + + +def _cleanup(db_session): + db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) + db_session.commit() + + +class ScriptedCompliantTestCase(unittest.TestCase): + """Seeds one eligible feed and replays scripted days against it.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + _cleanup(db_session) + db_session.add( + Gtfsfeed( + id=FEED, + stable_id=FEED, + data_type="gtfs", + status="active", + operational_status="published", + official=True, + created_at=DAY_ZERO - timedelta(days=400), + producer_url=f"https://example.com/{FEED}.zip", + ) + ) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + _cleanup(db_session) + + @staticmethod + def run_days(script: ComplianceScript, offsets) -> None: + """Evaluate the feed once per day, in order, writing each day's state. + + This is a march done the slow way — through the database, one `update_seals` call per + day — which is exactly what #1763's in-memory march has to reproduce. + """ + with registry(script): + for offset in offsets: + update_seals(stable_feed_ids=[FEED], dry_run=False, now=day(offset)) + + @staticmethod + @with_db_session(db_url=default_db_url) + def state(db_session=None): + return db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == FEED, + SealCriterion.__table__.c.criterion + == SealCriterionName.COMPLIANT.value, + ) + ).one() + + +class TestScriptDrivesTheEvaluator(unittest.TestCase): + """The fixture itself, with no database in the way.""" + + class _Ctx: + def __init__(self, now): + self.now = now + self.feed_id = FEED + self.stable_id = FEED + + def observed(self, script: ComplianceScript, offset: int) -> CriterionStatus: + evaluator = ScriptedCompliantEvaluator(script) + return evaluator.evaluate(self._Ctx(day(offset))).observed_status + + def test_unnamed_days_pass(self): + self.assertIs( + self.observed(ComplianceScript.failing_on(3), 0), CriterionStatus.PASS + ) + + def test_named_days_fail(self): + self.assertIs( + self.observed(ComplianceScript.failing_on(3), 3), CriterionStatus.FAIL + ) + + def test_a_range_is_inclusive_of_both_ends(self): + script = ComplianceScript.failing_between(5, 7) + for offset, expected in ( + (4, CriterionStatus.PASS), + (5, CriterionStatus.FAIL), + (7, CriterionStatus.FAIL), + (8, CriterionStatus.PASS), + ): + with self.subTest(offset=offset): + self.assertIs(self.observed(script, offset), expected) + + def test_unknown_days_win_over_failing_days(self): + """An input we could not read is not a failure, whatever else the script says.""" + script = ComplianceScript.failing_on(3).with_unknown_on(3) + self.assertIs(self.observed(script, 3), CriterionStatus.UNKNOWN) + + def test_it_carries_compliant_policy(self): + self.assertEqual(ScriptedCompliantEvaluator.grace_period, timedelta(days=30)) + self.assertEqual( + ScriptedCompliantEvaluator.probation_period, + PROBATION_PERIOD, + "the default 180 days, not an override", + ) + + +class TestGracePeriod(ScriptedCompliantTestCase): + def test_a_short_streak_is_absorbed(self): + """29 failing days is inside the 30-day grace period, so the status holds.""" + script = ComplianceScript.failing_between(1, 29) + self.run_days(script, range(0, 30)) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "still inside the grace period on day 29", + ) + self.assertIsNone(row.probation_start, "an absorbed failure opens no probation") + + def test_a_streak_past_the_grace_period_confirms(self): + script = ComplianceScript.failing_between(1, 40) + self.run_days(script, range(0, 41)) + + row = self.state() + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertIsNotNone(row.last_confirmed_failure_at) + + def test_the_first_evaluation_gets_no_grace(self): + """A criterion that has never passed has no track record to hold.""" + self.run_days(ComplianceScript.failing_on(0), [0]) + self.assertEqual(self.state().confirmed_status, CriterionStatus.FAIL.value) + + +class TestProbationFollowsRecovery(ScriptedCompliantTestCase): + def test_recovery_from_a_confirmed_failure_opens_probation(self): + script = ComplianceScript.failing_between(1, 40) + self.run_days(script, list(range(0, 42))) # day 41 is the repair + + row = self.state() + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "the check passes again", + ) + self.assertIsNotNone( + row.probation_start, "but it is serving probation for the failure" + ) + + def test_probation_suspends_the_grace_period(self): + """One bad day during probation confirms at once, and restarts the count. + + Off probation, a single failing day sits well inside the 30-day grace period and + would confirm nothing. This is the ratchet that makes a cold start's error persist. + """ + script = ComplianceScript( + failing=frozenset( + [day(offset).date() for offset in range(1, 41)] + [day(60).date()] + ) + ) + self.run_days(script, list(range(0, 62))) + + row = self.state() + self.assertEqual( + row.last_confirmed_failure_at.astimezone(timezone.utc).date(), + day(60).date(), + "the single day confirmed because probation had suspended the grace period", + ) + self.assertEqual( + row.probation_start.astimezone(timezone.utc).date(), + day(61).date(), + "and probation restarted from the day after it", + ) + + def test_probation_clears_once_served(self): + script = ComplianceScript.failing_between(1, 40) + served = 41 + PROBATION_PERIOD.days + self.run_days(script, [*range(0, 42), served]) + + self.assertIsNone( + self.state().probation_start, "the full stretch has been served" + ) + + +class TestUnknownFreezesTheState(ScriptedCompliantTestCase): + def test_an_unknown_day_leaves_the_verdict_standing(self): + script = ComplianceScript().with_unknown_on(1) + self.run_days(script, [0, 1]) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "a missing input must never read as a failure", + ) + + def test_an_unknown_day_does_not_advance_a_failure_streak(self): + """The streak keeps its start, so the grace period is not quietly extended.""" + script = ComplianceScript.failing_between(1, 5).with_unknown_on(3) + self.run_days(script, range(0, 6)) + + row = self.state() + self.assertEqual( + row.first_observed_failure_at.astimezone(timezone.utc).date(), + day(1).date(), + ) + + +if __name__ == "__main__": + unittest.main() From fb493c9dab7dbd7f4ea936e223db175627e64b22 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 25 Aug 2026 21:23:27 -0400 Subject: [PATCH 03/13] Let each criterion load its own day-varying inputs --- .../src/tasks/seal_of_reliability/context.py | 130 +++++++++++++----- .../evaluators/__init__.py | 4 +- .../seal_of_reliability/evaluators/base.py | 43 +++++- .../tasks/seal_of_reliability/seal_updater.py | 14 +- .../test_seal_evaluators.py | 58 +++++++- .../test_seal_updater_db.py | 37 ++++- 6 files changed, 233 insertions(+), 53 deletions(-) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index f33767660..b2d2c94cd 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -18,25 +18,33 @@ The evaluators are pure functions over a `FeedSealContext`, so every DB read for a batch of feeds happens here, in a fixed number of queries regardless of batch size. -Only Official is implemented, so the context currently carries just the feed row. Each new -criterion adds the fields it needs here plus one bulk query to populate them: the latest -dataset for Compliant and Fresh, the day's availability rows for Available, the full -dataset coverage history for Fresh continuous coverage. +A criterion's inputs reach it one of two ways, and the split is deliberate: + +* Day-invariant feed facts — `official`, and later `seasonal`, `is_producer_unstable`, + `created_at` — are fields on `FeedSealContext`, read straight off the feed row the caller + has already loaded. They cost no query and they have no history to read: the same value + answers every day of a backfill. +* Anything that varies by day is loaded by the criterion itself, through + `CriterionEvaluator.load_inputs`. Only the criterion knows what its own inputs look like, + and keeping that knowledge there is what stops this module from having to grow a field + and a query for every criterion added. """ import itertools -from dataclasses import dataclass -from datetime import datetime -from typing import Dict, Iterator, List, Optional, Sequence +from dataclasses import dataclass, field +from datetime import date, datetime, timezone +from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence from sqlalchemy.orm import Session from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed +from tasks.seal_of_reliability.criteria import SealCriterionName + @dataclass class FeedSealContext: - """Everything the evaluators need for one feed. + """Everything the evaluators need for one feed on one day. Built by `build_contexts`. Evaluators read from this and never query. """ @@ -50,6 +58,23 @@ class FeedSealContext: # Feed-level flags official: Optional[bool] = None + # Each criterion's own bulk-loaded inputs, keyed by criterion name — see + # `collect_inputs`. Opaque here: this module never looks inside a criterion's payload, + # and an evaluator reaches only its own through `inputs_for(self.name)`. + # + # The whole batch's inputs are shared by reference across every context, rather than + # sliced per feed and per day. Slicing would force every criterion into one storage + # shape, and it would copy a year of history once per (feed, day) during a backfill. + inputs: Mapping[SealCriterionName, Any] = field(default_factory=dict) + + def inputs_for(self, criterion: SealCriterionName) -> Any: + """This criterion's loaded inputs, or None if its loader returned nothing. + + None is the normal answer for a criterion whose inputs are day-invariant fields on + this context, so it means "nothing to load", not "the load failed". + """ + return self.inputs.get(criterion) + # Feeds in these statuses, or not published, are never eligible for the seal. # `inactive` and `future` feeds are deliberately kept eligible. @@ -130,48 +155,91 @@ def iter_eligible_stable_ids( yield chunk +def snapshot_date_of(now: datetime) -> date: + """The UTC day a run evaluating at `now` belongs to. + + The day a run's snapshots are keyed under, and the day its criteria are evaluated + against. Naive values are read as UTC rather than rejected: the task entry points + normalize what an operator passes, but `update_seals` is also called directly. + """ + if now.tzinfo is None: + return now.date() + return now.astimezone(timezone.utc).date() + + +def collect_inputs( + db_session: Session, + feeds: Sequence[Gtfsfeed], + days: Sequence[date], + evaluators: Sequence, +) -> Dict[SealCriterionName, Any]: + """Ask each evaluator to bulk-load its own day-varying inputs for the whole batch. + + Called once per batch whatever the number of days: a criterion loads its full history + for `days` in one go and answers each day from memory afterwards. That is what holds the + query count proportional to the number of criteria rather than to feeds x days — the + difference between a handful of queries and several thousand once a backfill marches a + year (#1763). + + Args: + db_session: SQLAlchemy session. + feeds: The batch of feeds, already loaded and eligibility-checked by the caller. + days: Every UTC day that will be evaluated, ascending. One entry for a nightly run. + evaluators: The `CriterionEvaluator` instances this run will apply. Not annotated as + such because `evaluators.base` imports this module for `FeedSealContext`. + + Returns: + criterion name -> whatever that criterion's loader returned. Opaque to this module; + an evaluator reaches its own with `ctx.inputs_for(self.name)`. + """ + return { + evaluator.name: evaluator.load_inputs(db_session, feeds, days) + for evaluator in evaluators + } + + def build_contexts( - db_session: Session, feeds: Sequence[Gtfsfeed], now: datetime + db_session: Session, + feeds: Sequence[Gtfsfeed], + now: datetime, + evaluators: Sequence, ) -> Dict[str, FeedSealContext]: - """Load everything the evaluators need for `feeds`, in a fixed number of queries. + """Build one context per feed for a single day — the nightly run's case. + + This is `collect_inputs` over a one-day range, plus the day-invariant feed fields. A + backfill marching a year calls `collect_inputs` once for the whole range and then builds + its contexts per day from that same result, so both paths load a criterion's inputs + through the criterion itself and there is only ever one place they come from. Args: - db_session: SQLAlchemy session. Unused while Official is the only criterion, since - everything it needs is already on the feed row, but kept in the signature - because every further criterion needs it. - feeds: The batch of feeds to load, already loaded (and eligibility-checked via + db_session: SQLAlchemy session, passed on to the evaluators' loaders. + feeds: The batch of feeds to build for, already loaded (and eligibility-checked via `is_seal_eligible`) by the caller — `update_seals`. now: The evaluation timestamp. + evaluators: The evaluators this run will apply. Required rather than defaulted: an + omitted list would leave every criterion with no inputs and quietly turn its + verdicts into UNKNOWN. Returns: feed_id -> FeedSealContext. - How to add a criterion's data. Two kinds: + Adding a criterion's data. Two kinds: 1. Already on the selected feed row (`official`, `created_at`, `seasonal`, - `is_producer_url_unstable`). Add the field to FeedSealContext and read it off `feed` - below. No query, no cost. - - 2. Needs its own query. Add the field, then a module-level `_load_*` helper that takes - the whole batch and returns a dict keyed by feed_id, and call it once here. Keeping - the query per batch rather than per feed is what holds the query count proportional - to the number of criteria instead of the number of feeds. For example, Available - (issue #1784) would add: - - def _load_availability_today(db_session, feed_ids, day_start) -> Dict[str, bool]: - '''feed_id -> whether any availability check succeeded since day_start. - Feeds absent from the result had no check at all, which the criterion reads - as "not evaluable" rather than "failing".''' - - called once as `availability = _load_availability_today(...)` and consumed per feed - as `availability_success_today=availability.get(feed.id, False)`. + `is_producer_url_unstable`). Add the field to `FeedSealContext` and read it off `feed` + below. No query, no cost, and it answers for any day. + + 2. Varies by day. Nothing changes here: override `load_inputs` on the criterion's own + evaluator and read it back in `_evaluate` with `ctx.inputs_for(self.name)`. """ + inputs = collect_inputs(db_session, feeds, [snapshot_date_of(now)], evaluators) return { feed.id: FeedSealContext( feed_id=feed.id, now=now, stable_id=feed.stable_id, official=feed.official, + inputs=inputs, ) for feed in feeds } diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py index 3ddabbe94..9893e2402 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/__init__.py @@ -17,7 +17,9 @@ `EVALUATORS` is the registry the job iterates. Only Official is implemented so far (issue #1783); the remaining criteria are tracked by #1784 and #1782. Adding one means a -new subclass, an entry here, and whatever fields it needs on `FeedSealContext`. +new subclass and an entry here, plus — for whatever inputs it needs — either a +day-invariant field on `FeedSealContext`, or its own `load_inputs` override when the +inputs vary by day. `seal_criterion_name` in the database already declares all six values, so a criterion can be added without a schema change. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py index 7291ee2f4..bd73e6446 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/evaluators/base.py @@ -16,8 +16,10 @@ """Base class for the per-criterion evaluators.""" from dataclasses import dataclass -from datetime import timedelta -from typing import Optional, Tuple +from datetime import date, timedelta +from typing import Any, Optional, Sequence, Tuple + +from sqlalchemy.orm import Session from tasks.seal_of_reliability.context import FeedSealContext from tasks.seal_of_reliability.criteria import ( @@ -50,8 +52,12 @@ class CriterionEvaluator: """Evaluates one criterion against a pre-loaded feed context. Subclasses set `name` and, where they differ from the defaults, `grace_period` and - `probation_period`, then implement `_evaluate`. They never touch the database: all the - data they need is on the context, loaded in bulk by `context.build_contexts`. + `probation_period`, then implement `_evaluate`. They never touch the database at + evaluation time: everything they need is already on the context, either as a + day-invariant feed field or as the inputs their own `load_inputs` bulk-loaded. + + A criterion that has to look backwards owns that lookup itself, rather than the context + builder growing a field and a query per criterion. `load_inputs` is where it goes. `grace_period` holds a passing status while an observed failure is still young. `probation_period` is how long the criterion must go with no observed failure after @@ -62,6 +68,35 @@ class CriterionEvaluator: grace_period: Optional[timedelta] = None probation_period: Optional[timedelta] = PROBATION_PERIOD + def load_inputs( + self, + db_session: Session, + feeds: Sequence, + days: Sequence[date], + ) -> Any: + """Bulk-load this criterion's day-varying inputs for a whole batch of feeds, at once. + + Returns an object of the criterion's own choosing — nothing outside the criterion + looks inside it. The caller stashes it on every context in the batch, and `_evaluate` + reads it back with `ctx.inputs_for(self.name)`, indexing by `ctx.feed_id` and the + day of `ctx.now`. + + The default returns None, which is the right answer for a criterion whose inputs are + day-invariant fields already on the context — Official and Stable read the feed row + and have nothing of their own to load. + + Override it for any criterion that does, and load the whole of `days` in one query + rather than one query per day: a nightly run passes a single day, but a backfill + (#1763) passes a year, and a per-day query there turns a handful of queries into + several thousand. + + Args: + db_session: SQLAlchemy session. + feeds: The batch of feeds to load for, already loaded by the caller. + days: Every UTC day that will be evaluated, ascending. + """ + return None + def evaluate(self, ctx: FeedSealContext) -> CriterionObservation: """Evaluate the criterion and label the result with this evaluator's name. diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index 3edf7e21c..c80fd3e3a 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -49,6 +49,7 @@ batched, build_contexts, is_seal_eligible, + snapshot_date_of, ) from tasks.seal_of_reliability.criteria import ( CriterionPhase, @@ -243,17 +244,6 @@ def _upsert_criteria( ) -def snapshot_date_of(now: datetime) -> date: - """The UTC day a run evaluating at `now` takes its snapshots under. - - Naive values are read as UTC rather than rejected: the entry point normalizes what an - operator passes, but `update_seals` is also called directly. - """ - if now.tzinfo is None: - return now.date() - return now.astimezone(timezone.utc).date() - - def _snapshot_row(state: SealCriterionState, snapshot_date: date) -> dict: """One seal_criterion_snapshot row: the key, then the state columns read off by name. @@ -414,7 +404,7 @@ def update_seals( for batch in batched(eligible_feeds, batch_size): batch_ids = [feed.id for feed in batch] - contexts = build_contexts(db_session, batch, now) + contexts = build_contexts(db_session, batch, now, evaluators) previous_states = _load_previous_states(db_session, batch_ids) previous_seals = _load_previous_seals(db_session, batch_ids) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py index 190102e29..05ce9c0a2 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_evaluators.py @@ -16,9 +16,9 @@ """Unit tests for the seal criterion evaluators. No database.""" import unittest -from datetime import datetime, timezone +from datetime import date, datetime, timezone -from tasks.seal_of_reliability.context import FeedSealContext +from tasks.seal_of_reliability.context import FeedSealContext, collect_inputs from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.evaluators import ( EVALUATORS, @@ -141,5 +141,59 @@ def test_reason_names_the_offending_value(self): self.assertIn("None", result.reason) +class TestLoadInputs(unittest.TestCase): + """The `load_inputs` hook, and how what it loads reaches `_evaluate`.""" + + def test_default_loader_loads_nothing(self): + """A criterion reading only day-invariant context fields has nothing to load. + + None here means "nothing to load", not "the load failed" — Official reads + `feed.official` off the context and never needs a query. + """ + for evaluator in EVALUATORS: + with self.subTest(criterion=evaluator.name): + self.assertIsNone(evaluator.load_inputs(object(), [], [NOW.date()])) + + def test_each_criterion_is_asked_once_for_the_whole_batch(self): + """One call per criterion, carrying every feed and every day. + + This is the property the backfill depends on: a criterion loading per day instead + would turn a year's march into several thousand queries. + """ + calls = [] + + class Recording(CriterionEvaluator): + name = SealCriterionName.AVAILABLE + + def load_inputs(self, db_session, feeds, days): + calls.append((tuple(feeds), tuple(days))) + return {"loaded": True} + + def _evaluate(self, ctx): + return CriterionStatus.PASS, "recorded" + + feeds = ["feed-1", "feed-2"] + days = [date(2026, 5, 30), date(2026, 5, 31), NOW.date()] + inputs = collect_inputs(object(), feeds, days, [Recording()]) + + self.assertEqual(calls, [(("feed-1", "feed-2"), tuple(days))]) + self.assertEqual(inputs, {SealCriterionName.AVAILABLE: {"loaded": True}}) + + def test_a_criterion_reaches_only_its_own_inputs(self): + ctx = _ctx( + inputs={ + SealCriterionName.AVAILABLE: "available-inputs", + SealCriterionName.COMPLIANT: "compliant-inputs", + } + ) + self.assertEqual( + ctx.inputs_for(SealCriterionName.AVAILABLE), "available-inputs" + ) + self.assertIsNone(ctx.inputs_for(SealCriterionName.OFFICIAL)) + + def test_context_defaults_to_no_inputs(self): + self.assertIsNone(_ctx().inputs_for(SealCriterionName.AVAILABLE)) + + if __name__ == "__main__": unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py index aa0e073ca..70b5ba9f5 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_updater_db.py @@ -31,7 +31,11 @@ CriterionStatus, SealCriterionName, ) -from tasks.seal_of_reliability.evaluators import CriterionEvaluator, OfficialEvaluator +from tasks.seal_of_reliability.evaluators import ( + EVALUATORS, + CriterionEvaluator, + OfficialEvaluator, +) from tasks.seal_of_reliability.seal_updater import update_seals from tasks.seal_of_reliability.state_machine import SealCriterionState from sqlalchemy import delete, select @@ -306,7 +310,7 @@ class TestBuildContexts(SealDbTestCase): @with_db_session(db_url=default_db_url) def test_loads_the_fields_the_evaluators_need(self, db_session): feeds = list(_feeds_by_stable_id(db_session, OFFICIAL).values()) - ctx = build_contexts(db_session, feeds, NOW)[feeds[0].id] + ctx = build_contexts(db_session, feeds, NOW, EVALUATORS)[feeds[0].id] self.assertEqual(ctx.stable_id, OFFICIAL) self.assertTrue(ctx.official) self.assertEqual(ctx.now, NOW) @@ -314,10 +318,37 @@ def test_loads_the_fields_the_evaluators_need(self, db_session): @with_db_session(db_url=default_db_url) def test_builds_one_context_per_feed(self, db_session): feeds = list(_feeds_by_stable_id(db_session, OFFICIAL, NOT_OFFICIAL).values()) - contexts = build_contexts(db_session, feeds, NOW) + contexts = build_contexts(db_session, feeds, NOW, EVALUATORS) self.assertEqual(len(contexts), 2) self.assertEqual({ctx.official for ctx in contexts.values()}, {True, False}) + @with_db_session(db_url=default_db_url) + def test_a_criterion_inputs_reach_every_context_in_the_batch(self, db_session): + """The nightly run is the one-day case: the loader is asked for `now`'s day only. + + The payload is shared by reference across the batch's contexts, so a criterion + indexes it by feed itself rather than the builder slicing it per feed. + """ + seen_days = [] + + class Loading(CriterionEvaluator): + name = SealCriterionName.AVAILABLE + + def load_inputs(self, db_session, feeds, days): + seen_days.append(list(days)) + return {feed.id: feed.stable_id for feed in feeds} + + def _evaluate(self, ctx): + return CriterionStatus.PASS, "loaded" + + feeds = list(_feeds_by_stable_id(db_session, OFFICIAL, NOT_OFFICIAL).values()) + contexts = build_contexts(db_session, feeds, NOW, [Loading()]) + + self.assertEqual(seen_days, [[NOW.date()]]) + for feed in feeds: + inputs = contexts[feed.id].inputs_for(SealCriterionName.AVAILABLE) + self.assertEqual(inputs[feed.id], feed.stable_id) + class TestUpdateSeals(SealDbTestCase): def test_dry_run_writes_nothing(self): From 90834f0f642b1456b797f3f689b62660209eeea9 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 25 Aug 2026 21:42:13 -0400 Subject: [PATCH 04/13] Implement the backfill day march --- functions-python/tasks_executor/src/main.py | 4 +- .../backfill/backfill_seal_of_reliability.py | 3 +- .../backfill/seal_backfill.py | 379 +++++++++++++++--- .../tasks/seal_of_reliability/seal_updater.py | 12 +- .../backfill/test_seal_backfill.py | 279 ++++++++++++- 5 files changed, 608 insertions(+), 69 deletions(-) diff --git a/functions-python/tasks_executor/src/main.py b/functions-python/tasks_executor/src/main.py index c05bce528..354aecda8 100644 --- a/functions-python/tasks_executor/src/main.py +++ b/functions-python/tasks_executor/src/main.py @@ -311,8 +311,8 @@ "Establishes a starting Seal of Reliability state for feeds that have none " "(issue #1763), by cold-starting each feed 12 months back and replaying the " "nightly evaluation forward one day at a time to end_date, writing only the " - "final day. NOT YET IMPLEMENTED: the day march raises, so dry_run=true is " - "the only mode that returns — it resolves and reports the plan. " + "final day. The intermediate days are held in memory and discarded unless " + "snapshot_mode says otherwise. " "Parameters: stable_feed_ids (required, non-empty), start_date (ISO date, " "default end_date minus days_back; clamped up to each feed's created_at), " "end_date (ISO date, default yesterday UTC), days_back (default 365), " diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py index 753b16457..636e5a0f2 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -74,8 +74,7 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: """ Handler for the Seal of Reliability backfill. - The day march is not implemented yet: a dry run returns the resolved plan, and a non-dry - run raises rather than reporting a success that wrote nothing. + A dry run returns the resolved plan without marching or writing. Payload parameters: stable_feed_ids (list[str]): Required and non-empty. The feeds to backfill; there is diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index af79e3808..5f5cfe8a3 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -21,9 +21,8 @@ intermediate days are held in memory and discarded — marching forward is what builds up the path-dependent state (grace-period streaks, probation) that makes the final state right. -STATUS: the invocation is complete and validated; **the day march itself is not implemented**. -A dry run resolves and returns the full plan — which feeds, which window per feed, how many -days — and a non-dry run raises rather than silently writing nothing. See `_march` below. +A dry run resolves and returns the plan — which feeds, which window per feed, how many days — +without marching or writing anything. Per-feed window --------------- @@ -51,22 +50,39 @@ """ import logging -from datetime import date, datetime, timedelta, timezone +import time as clock +from datetime import date, datetime, time, timedelta, timezone from typing import Dict, List, Optional, Sequence, Set, Tuple -from sqlalchemy import select +from sqlalchemy import and_, or_, select +from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from shared.database.database import with_db_session from shared.database_gen.sqlacodegen_models import Gtfsfeed, SealCriterion -from tasks.seal_of_reliability.context import is_seal_eligible +from tasks.seal_of_reliability.context import ( + FeedSealContext, + batched, + collect_inputs, + is_seal_eligible, +) +from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.seal_updater import ( DEFAULT_BATCH_SIZE, DEFAULT_MAX_REPORTED_FEEDS, + SEAL_TABLE, + SNAPSHOT_STATE_COLUMNS, + SNAPSHOT_TABLE, + _load_previous_seals, _resolve_evaluators, + _roll_up_has_seal, + _upsert_criteria, + _upsert_criterion_snapshot, _validate_requested_feed_ids, + is_partial_run, ) +from tasks.seal_of_reliability.state_machine import SealCriterionState, transition logger = logging.getLogger(__name__) @@ -156,43 +172,269 @@ def _feeds_with_seal_state(db_session: Session, feed_ids: Sequence[str]) -> Set[ return {row.feed_id for row in rows} +def day_start(day: date) -> datetime: + """The `now` a simulated day is evaluated at: midnight UTC. + + A fixed time of day, so that `snapshot_date_of(now)` is the day itself and + `_next_day_start(now)` — which probation uses — lands on the following midnight with no + rounding to reason about. The nightly job's own `now` is whatever time it ran; the march + only has to be consistent with itself and day-aligned. + """ + return datetime.combine(day, time.min, tzinfo=timezone.utc) + + +def days_between(first: date, last: date) -> List[date]: + """Every day from `first` to `last`, ascending, both ends included.""" + return [first + timedelta(days=offset) for offset in range((last - first).days + 1)] + + +def _state_from_snapshot(row) -> SealCriterionState: + """Rebuild a `SealCriterionState` from one seal_criterion_snapshot row. + + The state columns are taken from the table rather than listed, the mirror of + `seal_updater._snapshot_row` which writes them: a column added to the snapshot table is + read back without touching this function, and fails loudly here if `SealCriterionState` + has no field for it rather than being silently dropped. + """ + values = {} + for column in SNAPSHOT_STATE_COLUMNS: + value = getattr(row, column) + if column in ("observed_status", "confirmed_status"): + value = CriterionStatus(value) + values[column] = value + return SealCriterionState( + feed_id=row.feed_id, + criterion=SealCriterionName(row.criterion), + **values, + ) + + +def _seed_states( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + resume_from_snapshot: bool, +) -> Dict[Tuple[str, str], SealCriterionState]: + """The state each (feed, criterion) enters its first simulated day with. + + Empty unless `resume_from_snapshot`, in which case each pair is seeded from its latest + snapshot strictly before that feed's march start — a complete state, which is what turns + a cold start into a resume (#1803). + + Pairs with no snapshot are simply absent from the result, and `transition` builds them + from nothing on their first day, exactly as a cold start would. A resume that reaches + further back than the snapshots go therefore degrades to a cold start for those criteria + rather than failing. + """ + if not resume_from_snapshot or not feeds: + return {} + + # One query for the batch. Each feed has its own cut-off, so the conditions are OR-ed + # rather than sharing a single date; DISTINCT ON keeps the latest row per pair. + cutoffs = [ + and_( + SNAPSHOT_TABLE.c.feed_id == feed.id, + SNAPSHOT_TABLE.c.snapshot_date < windows[feed.id][0], + ) + for feed in feeds + if feed.id in windows + ] + if not cutoffs: + return {} + + rows = db_session.execute( + select(SNAPSHOT_TABLE) + .where(or_(*cutoffs)) + .distinct(SNAPSHOT_TABLE.c.feed_id, SNAPSHOT_TABLE.c.criterion) + .order_by( + SNAPSHOT_TABLE.c.feed_id, + SNAPSHOT_TABLE.c.criterion, + SNAPSHOT_TABLE.c.snapshot_date.desc(), + ) + ).all() + return {(row.feed_id, row.criterion): _state_from_snapshot(row) for row in rows} + + +def _upsert_seals_from_backfill( + db_session: Session, + outcomes: Sequence[dict], + now: datetime, +) -> None: + """Write feed_reliability_seal for the marched feeds. + + Two things differ from the nightly job's `_upsert_seals`, and both are about + `created_at`: + + * It is written explicitly, as the feed's march start rather than the write time. That + column is what the Stable criterion counts its 180 days from, so left at its + `DEFAULT now()` every backfilled feed would fail Stable on every simulated day and the + backfill would grant no seals at all. + * It is **insert-only**, absent from the conflict clause. A re-backfill of a feed the + nightly job already owns must not reset a countdown that has been running for real. + + `seal_earned_at` is stamped with `now` — the run's end_date — for a feed the backfill + grants. The march does know the day the roll-up flipped, but under a cold start that day + is often the very first simulated one, which would claim a feed earned its seal a year + ago on the strength of a single simulated day. `end_date` says only that the seal record + begins here, which is exactly what a backfill establishes. + """ + for outcome in outcomes: + row = { + "feed_id": outcome["feed_id"], + "has_seal": outcome["has_seal"], + "created_at": day_start(outcome["tracking_start"]), + "updated_at": now, + } + if outcome["granted"]: + row["seal_earned_at"] = now + elif outcome["revoked"]: + row["seal_lost_at"] = now + + statement = insert(SEAL_TABLE).values(**row) + update_set = { + "has_seal": statement.excluded.has_seal, + "updated_at": statement.excluded.updated_at, + } + # created_at is deliberately not in update_set — see the docstring. + if "seal_earned_at" in row: + update_set["seal_earned_at"] = statement.excluded.seal_earned_at + if "seal_lost_at" in row: + update_set["seal_lost_at"] = statement.excluded.seal_lost_at + db_session.execute( + statement.on_conflict_do_update( + index_elements=[SEAL_TABLE.c.feed_id], set_=update_set + ) + ) + + def _march( db_session: Session, feeds: Sequence[Gtfsfeed], windows: Dict[str, Tuple[date, date]], evaluators: Sequence, + end_date: date, snapshot_mode: str, resume_from_snapshot: bool, + partial_run: bool, +) -> dict: + """Replay the nightly evaluation day by day for one batch, and write the final day. + + The evaluation itself is the nightly job's, unmodified: `transition` is called once per + feed, criterion and day with `now` set to that day. What this adds is that the returned + state is threaded into the next day in memory instead of being written, so a year's march + costs one write per feed rather than three hundred and sixty-five. + + Ascending day order is not a convenience, it is the algorithm — each day's state is the + input to the next. + """ + if not feeds: + return {"feeds": 0, "criterion_rows": 0, "snapshot_rows": 0, "outcomes": []} + + marched_days = days_between(min(start for start, _ in windows.values()), end_date) + + # One load per criterion for the whole batch and the whole range. A criterion querying + # per day would turn this into several thousand queries; see `CriterionEvaluator.load_inputs`. + inputs = collect_inputs(db_session, feeds, marched_days, evaluators) + + states = _seed_states(db_session, feeds, windows, resume_from_snapshot) + snapshot_rows = 0 + + for today in marched_days: + now = day_start(today) + # A feed whose march starts later is simply not evaluated yet: its window was + # clamped to its own created_at, and days before that have nothing to say about it. + active = [feed for feed in feeds if windows[feed.id][0] <= today] + if not active: + continue + + days_states: List[SealCriterionState] = [] + for feed in active: + ctx = FeedSealContext( + feed_id=feed.id, + now=now, + stable_id=feed.stable_id, + official=feed.official, + inputs=inputs, + ) + for evaluator in evaluators: + key = (feed.id, evaluator.name.value) + states[key] = transition( + prev=states.get(key), + observation=evaluator.evaluate(ctx), + grace_period=evaluator.grace_period, + probation_period=evaluator.probation_period, + now=now, + feed_id=feed.id, + ) + days_states.append(states[key]) + + if snapshot_mode == "all": + # The expensive mode, and the only one that writes inside the loop. Flushed per + # day rather than accumulated so a year's march does not hold every day's state + # in memory at once. + _upsert_criterion_snapshot(db_session, days_states, today) + snapshot_rows += len(days_states) + db_session.commit() + + final_states = list(states.values()) + outcomes = _final_outcomes(db_session, feeds, windows, states, partial_run) + + _upsert_criteria(db_session, final_states, day_start(end_date)) + if snapshot_mode == "final": + _upsert_criterion_snapshot(db_session, final_states, end_date) + snapshot_rows += len(final_states) + if outcomes: + _upsert_seals_from_backfill(db_session, outcomes, day_start(end_date)) + db_session.commit() + + return { + "feeds": len(feeds), + "criterion_rows": len(final_states), + "snapshot_rows": snapshot_rows, + "outcomes": outcomes, + } + + +def _final_outcomes( + db_session: Session, + feeds: Sequence[Gtfsfeed], + windows: Dict[str, Tuple[date, date]], + states: Dict[Tuple[str, str], SealCriterionState], + partial_run: bool, ) -> List[dict]: - """Replay the nightly evaluation day by day and write the final state. NOT IMPLEMENTED. - - What this has to do, once built: - - 1. Ask each criterion to bulk-load its inputs for the whole day range at once — one load - per criterion per batch, never one per day. A year marched with per-day queries turns - a handful of queries into several thousand. - 2. Seed each (feed, criterion) with an empty `SealCriterionState`, or with the state read - from `seal_criterion_snapshot` at `march_start - 1` when `resume_from_snapshot` is set - (#1803). - 3. For each day in order, evaluate every criterion and apply `state_machine.transition`, - threading the returned state into the next day in memory. Nothing is written per day. - 4. Roll up `has_seal` on the final day and upsert `seal_criterion` and - `feed_reliability_seal`, plus `seal_criterion_snapshot` per `snapshot_mode`. - 5. Write `feed_reliability_seal.created_at = march_start` on insert only, so Stable counts - from the right day and a re-backfill cannot reset a countdown already running. - - Two behaviours are still undecided and must be settled before this is built: - - * `seal_earned_at` — as `_upsert_seals` stands it would be stamped with the write time, - so every backfilled feed would look like it earned its seal on backfill day. The march - knows the day the roll-up actually flipped and could stamp that instead. - * Whether the evaluators can answer for a past day at all. They currently read the - current feed row only; each criterion needs to own its own historical lookup first. + """Roll `has_seal` up from the final day's state, one entry per marched feed. + + Skipped entirely on a partial criteria run, mirroring `update_seals`: criteria that were + not evaluated cannot be judged, so the roll-up would be answering a question it has only + part of the evidence for. """ - raise NotImplementedError( - "The Seal of Reliability day march is not implemented yet (#1763). Run with " - "dry_run=true to resolve and inspect the plan." - ) + if partial_run: + return [] + + previous = _load_previous_seals(db_session, [feed.id for feed in feeds]) + outcomes = [] + for feed in feeds: + feed_states = { + criterion: state + for (owner_id, criterion), state in states.items() + if owner_id == feed.id + } + had_seal = bool(previous.get(feed.id)) + has_seal = _roll_up_has_seal(feed_states) + outcomes.append( + { + "feed_id": feed.id, + "stable_id": feed.stable_id, + "tracking_start": windows[feed.id][0], + "had_seal": had_seal, + "has_seal": has_seal, + # A first evaluation is a grant if it passes, but not a loss if it fails: + # nothing was held, so nothing was lost. + "granted": has_seal and not had_seal, + "revoked": had_seal and not has_seal, + } + ) + return outcomes @with_db_session @@ -211,7 +453,7 @@ def backfill_seals( resume_from_snapshot: bool = False, max_reported_feeds: int = DEFAULT_MAX_REPORTED_FEEDS, ) -> dict: - """Plan and (once `_march` exists) run the backfill for the requested feeds. + """Plan and run the backfill for the requested feeds. Like `update_seals`, this always runs against an explicit list of feeds — enumerating the catalogue is a producer's job, not this function's. @@ -238,9 +480,10 @@ def backfill_seals( max_reported_feeds: Cap on the `feeds` list in the report. Returns: - A plan report. `days` is the longest march in the run; feeds clamped to their own + A report. `days` is the longest march in the run; feeds clamped to their own `created_at` march fewer. """ + started = clock.monotonic() if not stable_feed_ids: raise ValueError("stable_feed_ids is required and must be non-empty") if snapshot_mode not in SNAPSHOT_MODES: @@ -297,43 +540,73 @@ def backfill_seals( for feed in selected ] + partial_run = is_partial_run(evaluators) + report = { "message": ( - f"Planned a backfill of {len(selected)} feed(s) across " - f"{len(evaluators)} criterion/criteria, ending {window_end.isoformat()}." + f"{'Planned' if dry_run else 'Ran'} a backfill of {len(selected)} feed(s) " + f"across {len(evaluators)} criterion/criteria, ending " + f"{window_end.isoformat()}." ), "dry_run": dry_run, - "implemented": False, "start_date": window_start.isoformat(), "end_date": window_end.isoformat(), "days": longest_march, "total_feeds": len(selected), "skipped_already_backfilled": len(already_backfilled), "criteria": [evaluator.name.value for evaluator in evaluators], + "partial_run": partial_run, "only_missing": only_missing, "snapshot_mode": snapshot_mode, "resume_from_snapshot": resume_from_snapshot, "batch_size": batch_size, + "criterion_rows_written": 0, + "snapshot_rows_written": 0, + "seals_granted": 0, + "seals_after_run": 0, + "granted_stable_ids": [], } + + if not dry_run: + outcomes: List[dict] = [] + for batch in batched(selected, batch_size): + result = _march( + db_session, + batch, + windows, + evaluators, + window_end, + snapshot_mode, + resume_from_snapshot, + partial_run, + ) + report["criterion_rows_written"] += result["criterion_rows"] + report["snapshot_rows_written"] += result["snapshot_rows"] + outcomes.extend(result["outcomes"]) + + granted = [outcome for outcome in outcomes if outcome["granted"]] + report["seals_granted"] = len(granted) + report["seals_after_run"] = sum( + 1 for outcome in outcomes if outcome["has_seal"] + ) + report["granted_stable_ids"] = [outcome["stable_id"] for outcome in granted] + + if partial_run: + report["note"] = ( + "Partial criteria run: has_seal was not recalculated because the criteria " + "that were not evaluated cannot be judged." + ) + + report["elapsed_seconds"] = round(clock.monotonic() - started, 2) report["feeds"] = feed_plans[:max_reported_feeds] report["feeds_omitted"] = max(0, len(feed_plans) - max_reported_feeds) + # Logged without `feeds`: Cloud Logging drops a LogEntry over 256 KB, so a run naming a + # few hundred feeds would lose the whole entry. logger.info( - "Backfill plan: %s", + "Backfill %s: %s", + "plan" if dry_run else "complete", {key: value for key, value in report.items() if key != "feeds"}, ) - if not dry_run: - # Raise rather than return a report that looks like a completed run. Until `_march` - # exists there is nothing to write, and reporting success for that would be worse - # than failing. - _march( - db_session, - selected, - windows, - evaluators, - snapshot_mode, - resume_from_snapshot, - ) - return report diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py index c80fd3e3a..65c3f4a0f 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/seal_updater.py @@ -97,6 +97,16 @@ def _resolve_evaluators(criteria: Optional[Sequence[str]]) -> List: return [evaluator for evaluator in EVALUATORS if evaluator.name.value in wanted] +def is_partial_run(evaluators: Sequence) -> bool: + """Whether `evaluators` is only part of the registry, so has_seal cannot be rolled up. + + Kept here, next to the registry it compares against, so that both the nightly job and + the backfill answer the question the same way — and so a test patching `EVALUATORS` in + this module alone moves both. + """ + return len(evaluators) < len(EVALUATORS) + + def _validate_requested_feed_ids( requested: Sequence[str], found: Set[str], @@ -369,7 +379,7 @@ def update_seals( started = time.monotonic() now = now or datetime.now(timezone.utc) evaluators = _resolve_evaluators(criteria) - partial_run = len(evaluators) < len(EVALUATORS) + partial_run = is_partial_run(evaluators) # Plain by-id load: no eligibility predicate here, since these ids were already # explicitly requested. Eligibility is checked in Python below, on the loaded rows. diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py index e5142166c..5c161b48b 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -24,11 +24,18 @@ from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone from typing import Optional +from unittest.mock import patch -from sqlalchemy import delete, insert +from sqlalchemy import delete, insert, select from shared.database.database import with_db_session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion +from shared.database_gen.sqlacodegen_models import ( + Feed, + FeedReliabilitySeal, + Gtfsfeed, + SealCriterion, + SealCriterionSnapshot, +) from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import ( _parse_day, backfill_seal_of_reliability_handler, @@ -37,10 +44,15 @@ from tasks.seal_of_reliability.backfill.seal_backfill import ( DEFAULT_DAYS_BACK, backfill_seals, + day_start, + days_between, march_start_for, resolve_window, yesterday_utc, ) +from tasks.seal_of_reliability.criteria import SealCriterionName +from tasks.seal_of_reliability.seal_updater import update_seals +from test_scripted_compliant import ComplianceScript, ScriptedCompliantEvaluator from test_shared.test_utils.database_utils import default_db_url PREFIX = "seal_bf_" @@ -214,7 +226,7 @@ def test_dry_run_reports_the_resolved_window(self): stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=True ) self.assertTrue(report["dry_run"]) - self.assertFalse(report["implemented"]) + self.assertEqual(report["criterion_rows_written"], 0) self.assertEqual(report["start_date"], START.isoformat()) self.assertEqual(report["end_date"], END.isoformat()) self.assertEqual(report["days"], (END - START).days + 1) @@ -274,14 +286,6 @@ def test_only_missing_false_re_backfills(self): class TestBackfillValidation(BackfillDbTestCase): - def test_a_real_run_refuses_rather_than_writing_nothing(self): - """Until the march exists, reporting success would be worse than failing.""" - with self.assertRaises(NotImplementedError) as caught: - backfill_seals( - stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=False - ) - self.assertIn("#1763", str(caught.exception)) - def test_empty_feed_list_is_rejected(self): with self.assertRaises(ValueError): backfill_seals(stable_feed_ids=[], dry_run=True) @@ -298,6 +302,13 @@ def test_unknown_criteria_are_rejected(self): backfill_seals(stable_feed_ids=[OLD], dry_run=True, criteria=["punctual"]) self.assertIn("punctual", str(caught.exception)) + def test_dry_run_writes_nothing(self): + backfill_seals( + stable_feed_ids=[OLD], start_date=START, end_date=END, dry_run=True + ) + self.assertEqual(criterion_rows(OLD), {}) + self.assertIsNone(seal_row(OLD)) + def test_handler_threads_the_payload_through(self): report = backfill_seal_of_reliability_handler( { @@ -313,5 +324,251 @@ def test_handler_threads_the_payload_through(self): self.assertEqual(report["start_date"], START.isoformat()) +MARCHED = f"{PREFIX}marched" +REPLAYED = f"{PREFIX}replayed" + +# A short window, so the equivalence test replays a tractable number of days through the +# database. The failing run is long enough to outlast the stand-in's 30-day grace period, +# so the comparison covers a confirmed failure and the probation that follows it. +MARCH_START = date(2026, 1, 1) +MARCH_END = date(2026, 3, 15) +FAILING = ComplianceScript.failing_between(10, 45) + +STATE_COLUMNS = ( + "observed_status", + "confirmed_status", + "evaluated_at", + "last_verdict_at", + "first_observed_failure_at", + "last_observed_failure_at", + "last_confirmed_failure_at", + "probation_start", +) + + +def _script_for(offsets_from_march_start): + """A ComplianceScript whose failing days are offsets from MARCH_START. + + `ComplianceScript` counts from its own day zero, so the offsets are rebased here rather + than the fixture being reconfigured. + """ + return ComplianceScript( + failing=frozenset( + MARCH_START + timedelta(days=offset) for offset in offsets_from_march_start + ) + ) + + +@with_db_session(db_url=default_db_url) +def criterion_rows(stable_id, db_session=None): + rows = db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == stable_id + ) + ).all() + return {row.criterion: row for row in rows} + + +@with_db_session(db_url=default_db_url) +def seal_row(stable_id, db_session=None): + return db_session.execute( + select(FeedReliabilitySeal.__table__).where( + FeedReliabilitySeal.__table__.c.feed_id == stable_id + ) + ).one_or_none() + + +@with_db_session(db_url=default_db_url) +def snapshot_days(stable_id, db_session=None): + rows = db_session.execute( + select(SealCriterionSnapshot.__table__.c.snapshot_date).where( + SealCriterionSnapshot.__table__.c.feed_id == stable_id + ) + ).all() + return sorted({row.snapshot_date for row in rows}) + + +class MarchTestCase(unittest.TestCase): + """Two identically-aged feeds: one marched in memory, one replayed through the database.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + _cleanup(db_session) + _seed_feed(db_session, MARCHED, OLD_CREATED) + _seed_feed(db_session, REPLAYED, OLD_CREATED) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + _cleanup(db_session) + + @staticmethod + def registry(script): + """Patch the registry `_resolve_evaluators` reads, which is the one both paths use.""" + return patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedCompliantEvaluator(script)], + ) + + @staticmethod + def march(stable_id, script, **kwargs): + with MarchTestCase.registry(script): + return backfill_seals( + stable_feed_ids=[stable_id], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + **kwargs, + ) + + @staticmethod + def replay_through_db(stable_id, script): + """The same days, evaluated one `update_seals` call at a time. + + Uses the same midnight-UTC timestamps the march uses, so any difference in the final + state is the march's doing and not a difference in `now`. + """ + with MarchTestCase.registry(script): + for day in days_between(MARCH_START, MARCH_END): + update_seals( + stable_feed_ids=[stable_id], dry_run=False, now=day_start(day) + ) + + def state_of(self, stable_id): + row = criterion_rows(stable_id)[SealCriterionName.COMPLIANT.value] + return {column: getattr(row, column) for column in STATE_COLUMNS} + + +class TestMarchMatchesTheDatabaseReplay(MarchTestCase): + def test_a_clean_run_agrees(self): + script = _script_for([]) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + self.assertEqual(self.state_of(MARCHED), self.state_of(REPLAYED)) + + def test_a_confirmed_failure_and_its_probation_agree(self): + """The case the backfill exists for: state that depends on the whole path.""" + script = _script_for(range(10, 46)) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + + marched = self.state_of(MARCHED) + self.assertEqual(marched, self.state_of(REPLAYED)) + self.assertIsNotNone( + marched["last_confirmed_failure_at"], + "the 36-day streak must have outlasted the 30-day grace period", + ) + self.assertIsNotNone( + marched["probation_start"], "and recovery must have opened probation" + ) + + def test_an_absorbed_blip_agrees(self): + script = _script_for([20]) + self.march(MARCHED, script) + self.replay_through_db(REPLAYED, script) + + marched = self.state_of(MARCHED) + self.assertEqual(marched, self.state_of(REPLAYED)) + self.assertIsNone( + marched["last_confirmed_failure_at"], + "one day is well inside the grace period", + ) + + +class TestMarchWrites(MarchTestCase): + def test_only_the_final_day_is_snapshotted_by_default(self): + self.march(MARCHED, _script_for([20])) + self.assertEqual(snapshot_days(MARCHED), [MARCH_END]) + + def test_snapshot_mode_all_records_every_day(self): + self.march(MARCHED, _script_for([20]), snapshot_mode="all") + self.assertEqual(snapshot_days(MARCHED), days_between(MARCH_START, MARCH_END)) + + def test_snapshot_mode_none_records_nothing(self): + self.march(MARCHED, _script_for([20]), snapshot_mode="none") + self.assertEqual(snapshot_days(MARCHED), []) + + def test_the_seal_row_created_at_is_the_march_start(self): + """Left at its DEFAULT now(), Stable would fail on every simulated day.""" + self.march(MARCHED, _script_for([])) + self.assertEqual( + seal_row(MARCHED).created_at.astimezone(timezone.utc).date(), MARCH_START + ) + + def test_created_at_survives_a_re_backfill(self): + """Insert-only: a re-run must not reset a countdown already running.""" + self.march(MARCHED, _script_for([])) + first = seal_row(MARCHED).created_at + + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=30), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + ) + self.assertEqual(seal_row(MARCHED).created_at, first) + + def test_seal_earned_at_is_the_end_of_the_window(self): + report = self.march(MARCHED, _script_for([])) + self.assertEqual(report["seals_granted"], 1) + self.assertEqual( + seal_row(MARCHED).seal_earned_at.astimezone(timezone.utc).date(), MARCH_END + ) + + def test_the_report_counts_what_was_written(self): + report = self.march(MARCHED, _script_for([20])) + self.assertEqual(report["criterion_rows_written"], 1) + self.assertEqual(report["snapshot_rows_written"], 1) + self.assertEqual(report["granted_stable_ids"], [MARCHED]) + self.assertFalse(report["dry_run"]) + + +class TestResumeFromSnapshot(MarchTestCase): + def test_a_resume_starts_from_the_stored_snapshot(self): + """Seeded from the day before, the march inherits an open probation. + + Without the seed the same window is a clean cold start, so the difference is + entirely the snapshot's doing. + """ + # A first march that ends on probation, snapshotting every day. + self.march(MARCHED, _script_for(range(10, 46)), snapshot_mode="all") + self.assertIsNotNone(self.state_of(MARCHED)["probation_start"]) + + # Resume the tail of the window, with no failures in it at all. + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=60), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + resume_from_snapshot=True, + ) + + self.assertIsNotNone( + self.state_of(MARCHED)["probation_start"], + "the probation carried over from the seeded snapshot", + ) + + def test_without_the_flag_the_same_window_cold_starts(self): + self.march(MARCHED, _script_for(range(10, 46)), snapshot_mode="all") + + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START + timedelta(days=60), + end_date=MARCH_END, + dry_run=False, + only_missing=False, + ) + + self.assertIsNone( + self.state_of(MARCHED)["probation_start"], + "a cold start carries no probation forward", + ) + + if __name__ == "__main__": unittest.main() From bd642ebc9807cdd997aa66671d225771cc13c1c3 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Tue, 25 Aug 2026 21:57:43 -0400 Subject: [PATCH 05/13] Fan the backfill out across the catalog --- functions-python/tasks_executor/src/main.py | 43 ++- .../backfill/seal_backfill.py | 8 + .../backfill/seal_backfill_orchestrator.py | 318 ++++++++++++++++ .../backfill/seal_backfill_worker.py | 129 +++++++ .../src/tasks/seal_of_reliability/context.py | 31 +- .../orchestrator/seal_orchestrator_monitor.py | 27 +- .../backfill/test_seal_backfill_fanout.py | 349 ++++++++++++++++++ 7 files changed, 890 insertions(+), 15 deletions(-) create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py diff --git a/functions-python/tasks_executor/src/main.py b/functions-python/tasks_executor/src/main.py index 354aecda8..0c1e534a8 100644 --- a/functions-python/tasks_executor/src/main.py +++ b/functions-python/tasks_executor/src/main.py @@ -80,6 +80,14 @@ backfill_seal_of_reliability_handler, ) +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, +) + +from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, +) + from tasks.seal_of_reliability.update_seal_of_reliability import ( update_seal_of_reliability_handler, ) @@ -324,6 +332,35 @@ ), "handler": backfill_seal_of_reliability_handler, }, + "seal_backfill_orchestrator": { + "description": ( + "Cloud Tasks producer for the Seal of Reliability backfill across the whole " + "catalog (issue #1763). Resolves every seal-eligible GTFS feed that has no " + "seal state yet, chunks it, registers a run in TaskExecutionTracker (feeds " + "DB), and enqueues one 'seal_backfill_worker' task per batch plus a single " + "'seal_orchestrator_monitor' barrier task. The window is resolved here once " + "and passed to every worker, so all batches of a run end on the same day. " + "Parameters: dry_run (default true), batch_size (default 100), start_date " + "(ISO date, default end_date minus days_back), end_date (ISO date, default " + "yesterday UTC), days_back (default 365), criteria (default null), limit " + "(default null), stable_feed_ids (restrict eligibility to these ids, default " + "null), only_missing (default true), snapshot_mode (final|all|none, default " + "final), resume_from_snapshot (default false), deadline_seconds (default " + "7200), monitor_delay_seconds (default 300)." + ), + "handler": seal_backfill_orchestrator_handler, + }, + "seal_backfill_worker": { + "description": ( + "Cloud Tasks worker: march one batch's worth of feeds for the Seal of " + "Reliability backfill and report completion/failure to TaskExecutionTracker. " + "Parameters: run_id (required), batch_id (required), stable_feed_ids " + "(required, non-empty), start_date (required, ISO date), end_date (required, " + "ISO date), criteria (default null), only_missing (default true), " + "snapshot_mode (default final), resume_from_snapshot (default false)." + ), + "handler": seal_backfill_worker_handler, + }, "seal_orchestrator": { "description": ( "Cloud Tasks producer for the nightly Seal of Reliability run across the " @@ -355,8 +392,10 @@ "batch of a seal orchestrator run has reported, or the run's " "deadline_seconds passes, then aggregates each batch's report and marks " "the run completed (every batch succeeded) or failed (any batch failed, " - "or the deadline was reached with batches still unaccounted for). " - "Parameters: run_id (required)." + "or the deadline was reached with batches still unaccounted for). Settles " + "both the nightly fan-out and the backfill fan-out; task_name selects which. " + "Parameters: run_id (required), task_name (default 'seal_orchestrator_run'; " + "pass 'seal_backfill_run' for a backfill run)." ), "handler": seal_orchestrator_monitor_handler, }, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index 5f5cfe8a3..d8f549715 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -563,8 +563,10 @@ def backfill_seals( "criterion_rows_written": 0, "snapshot_rows_written": 0, "seals_granted": 0, + "seals_revoked": 0, "seals_after_run": 0, "granted_stable_ids": [], + "revoked_stable_ids": [], } if not dry_run: @@ -585,11 +587,17 @@ def backfill_seals( outcomes.extend(result["outcomes"]) granted = [outcome for outcome in outcomes if outcome["granted"]] + # A backfill can only revoke when only_missing is False, since a feed with no + # stored seal held nothing to lose. Reported anyway so the run-level aggregate in + # `seal_orchestrator_monitor` reads the same keys from both fan-outs. + revoked = [outcome for outcome in outcomes if outcome["revoked"]] report["seals_granted"] = len(granted) + report["seals_revoked"] = len(revoked) report["seals_after_run"] = sum( 1 for outcome in outcomes if outcome["has_seal"] ) report["granted_stable_ids"] = [outcome["stable_id"] for outcome in granted] + report["revoked_stable_ids"] = [outcome["stable_id"] for outcome in revoked] if partial_run: report["note"] = ( diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py new file mode 100644 index 000000000..b90d0086c --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py @@ -0,0 +1,318 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Cloud Tasks producer: fan the Seal of Reliability backfill out across the catalog (#1763). + +`backfill_seal_of_reliability` only ever marches an explicit `stable_feed_ids` list. This +producer is what enumerates the catalog and chunks it, the same shape as the nightly +`seal_orchestrator`: + + 1. resolves every seal-eligible GTFS feed that has no seal state yet; + 2. splits the stable_ids into batches of `batch_size`; + 3. registers the run + one entry per batch in TaskExecutionTracker and enqueues one + `seal_backfill_worker` Cloud Task per batch; + 4. enqueues a single `seal_orchestrator_monitor` barrier task, carrying this run's + `task_name` so the shared monitor settles the right tracker. + +Three things differ from the nightly producer, and all three follow from a march being +long where a nightly evaluation is a single day: + +* **`end_date` is resolved here, once**, and passed to every worker. Left to each worker to + default, two workers of the same run started either side of midnight would march to + different final days and write states that do not correspond to the same moment. +* **Batches are smaller** and the **deadline is longer**. A batch marches a year for each of + its feeds, not one day. +* **`only_missing` is the eligibility predicate, not a filter the worker applies.** A feed + the nightly job already owns has real accumulated history, and must not have a simulation + written over it. + +Payload (all optional):: + + { + "dry_run": bool, # default True + "batch_size": int, # default 100 + "start_date": str | None, # ISO date, default end_date - days_back + "end_date": str | None, # ISO date, default yesterday UTC + "days_back": int, # default 365 + "criteria": [str] | None, # default None (every implemented criterion) + "limit": int | None, # cap total feeds considered, default None + "stable_feed_ids": [str] | None, # restrict eligibility to these ids, default None + "only_missing": bool, # default True + "snapshot_mode": str, # final | all | none, default final + "resume_from_snapshot": bool, # default False + "deadline_seconds": int, # default 7200 (2h) + "monitor_delay_seconds": int, # default 300 + } +""" + +import logging +import math +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from shared.database.database import with_db_session +from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker + +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import _parse_day +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_DAYS_BACK, + DEFAULT_SNAPSHOT_MODE, + SNAPSHOT_MODES, + resolve_window, +) +from tasks.seal_of_reliability.context import ( + count_eligible_feeds, + iter_eligible_stable_ids, +) +from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + _enqueue, + _safe_task_name, +) + +logger = logging.getLogger(__name__) + +# TaskExecutionTracker task_name for a backfill run. Distinct from the nightly run's, so +# the two never share a tracker and the monitor aggregates only its own batches. +SEAL_BACKFILL_TASK_NAME = "seal_backfill_run" + +# Smaller than the nightly default of 250: a batch marches every one of its feeds across the +# whole window, so the per-batch cost scales with days as well as feeds. +DEFAULT_BATCH_SIZE = 100 +DEFAULT_DEADLINE_SECONDS = 2 * 60 * 60 # 2h wall-clock cap for a run +DEFAULT_MONITOR_DELAY_SECONDS = 300 + + +def seal_backfill_orchestrator_handler(payload: dict) -> dict: + """Entry point for the `seal_backfill_orchestrator` task.""" + payload = payload or {} + snapshot_mode = payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE) + if snapshot_mode not in SNAPSHOT_MODES: + raise ValueError( + f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" + ) + + # Validated and resolved here rather than in the workers, so a bad date fails the run at + # the producer instead of once per batch. + window_start, window_end = resolve_window( + _parse_day(payload.get("start_date"), "start_date"), + _parse_day(payload.get("end_date"), "end_date"), + int(payload.get("days_back", DEFAULT_DAYS_BACK)), + ) + + return _plan_run( + dry_run=bool(payload.get("dry_run", True)), + batch_size=int(payload.get("batch_size", DEFAULT_BATCH_SIZE)), + window_start=window_start, + window_end=window_end, + criteria=payload.get("criteria"), + limit=payload.get("limit"), + stable_feed_ids=payload.get("stable_feed_ids"), + only_missing=bool(payload.get("only_missing", True)), + snapshot_mode=snapshot_mode, + resume_from_snapshot=bool(payload.get("resume_from_snapshot", False)), + deadline_seconds=int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)), + monitor_delay_seconds=int( + payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) + ), + ) + + +@with_db_session +def _plan_run( + dry_run: bool, + batch_size: int, + window_start, + window_end, + criteria: Optional[List[str]], + limit: Optional[int], + stable_feed_ids: Optional[List[str]], + only_missing: bool, + snapshot_mode: str, + resume_from_snapshot: bool, + deadline_seconds: int, + monitor_delay_seconds: int, + db_session=None, +) -> Dict[str, Any]: + """Resolve the feeds to backfill, chunk them, and (unless dry_run) fan the run out.""" + if batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + + run_started_at = datetime.now(timezone.utc) + total_feeds = count_eligible_feeds( + db_session, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ) + num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 + run_id = f"seal-backfill-{run_started_at.strftime('%Y%m%dT%H%M%S')}" + + logger.info( + "seal_backfill_orchestrator: run=%s total_feeds=%d batch_size=%d batches=%d " + "window=%s..%s dry_run=%s", + run_id, + total_feeds, + batch_size, + num_batches, + window_start.isoformat(), + window_end.isoformat(), + dry_run, + ) + + plan = { + "run_id": run_id, + "total_feeds": total_feeds, + "batch_size": batch_size, + "batches": num_batches, + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + "enqueued": 0, + "dry_run": dry_run, + } + if dry_run or not num_batches: + return plan + + run_params = { + "dry_run": False, + "batch_size": batch_size, + "criteria": criteria, + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + "run_started_at": run_started_at.isoformat(), + "deadline_seconds": deadline_seconds, + } + batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] + _start_run(run_id, batch_ids, run_params) + + enqueued = 0 + consumed = 0 + stable_id_batches = iter_eligible_stable_ids( + db_session, + batch_size, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ) + for batch_id, batch_stable_ids in zip(batch_ids, stable_id_batches): + consumed += 1 + worker_payload = { + "run_id": run_id, + "batch_id": batch_id, + "stable_feed_ids": batch_stable_ids, + "criteria": criteria, + # Both ends are explicit, so a worker never re-derives the window. + "start_date": window_start.isoformat(), + "end_date": window_end.isoformat(), + "only_missing": only_missing, + "snapshot_mode": snapshot_mode, + "resume_from_snapshot": resume_from_snapshot, + } + if _enqueue( + in_body_task="seal_backfill_worker", + payload=worker_payload, + queue_env="SEAL_ORCHESTRATOR_QUEUE", + task_name=_safe_task_name(f"seal-backfill-{run_id}-{batch_id}"), + ): + enqueued += 1 + else: + # Dead on arrival: don't leave this batch as `triggered` for the monitor to + # only notice once the deadline passes. + _mark_enqueue_failed(run_id, batch_id) + + if consumed < len(batch_ids): + # The eligible-feed stream yielded fewer chunks than the plan-time count implied — + # eligibility narrowed in the gap between the two queries. Fail the leftovers + # immediately rather than leaving them `triggered` until the deadline. + missing = batch_ids[consumed:] + logger.error( + "seal_backfill_orchestrator: run=%s stream yielded %d batch(es), expected %d " + "— marking %d failed: %s", + run_id, + consumed, + len(batch_ids), + len(missing), + missing, + ) + for batch_id in missing: + _mark_enqueue_failed( + run_id, + batch_id, + error_message="no eligible-feed data for this batch (count/stream mismatch)", + ) + else: + extra_chunk = next(stable_id_batches, None) + if extra_chunk is not None: + # More chunks than planned: feeds became newly eligible in the gap. Log-only — + # a backfill is operator-triggered, so the fix is to run it again. + logger.error( + "seal_backfill_orchestrator: run=%s stream had more batches than the " + "plan-time count of %d expected (>=%d additional feed(s)) — those feeds " + "were not backfilled; re-run to pick them up", + run_id, + len(batch_ids), + len(extra_chunk), + ) + + _enqueue( + in_body_task="seal_orchestrator_monitor", + payload={"run_id": run_id, "task_name": SEAL_BACKFILL_TASK_NAME}, + queue_env="SEAL_ORCHESTRATOR_MONITOR_QUEUE", + task_name=_safe_task_name(f"seal-backfill-monitor-{run_id}"), + schedule_seconds=monitor_delay_seconds, + ) + + plan["enqueued"] = enqueued + return plan + + +@with_db_session +def _start_run( + run_id: str, + batch_ids: List[str], + run_params: dict, + db_session=None, +) -> None: + """Register the run and one tracked entry per batch.""" + tracker = TaskExecutionTracker( + task_name=SEAL_BACKFILL_TASK_NAME, + run_id=run_id, + db_session=db_session, + ) + tracker.start_run(total_count=len(batch_ids), params=run_params) + for batch_id in batch_ids: + tracker.mark_triggered(batch_id) + db_session.commit() + + +@with_db_session +def _mark_enqueue_failed( + run_id: str, + batch_id: str, + error_message: str = "enqueue failed", + db_session=None, +) -> None: + tracker = TaskExecutionTracker( + task_name=SEAL_BACKFILL_TASK_NAME, + run_id=run_id, + db_session=db_session, + ) + tracker.mark_failed(batch_id, error_message=error_message) + db_session.commit() diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py new file mode 100644 index 000000000..2d395e56a --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py @@ -0,0 +1,129 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Cloud Tasks worker: march one batch of the Seal of Reliability backfill (#1763). + +One `seal_backfill_worker` task is enqueued per batch by `seal_backfill_orchestrator`. It +calls `backfill_seals` for its slice of stable_ids and reports the outcome to the shared +`TaskExecutionTracker` so the monitor knows when the run has drained. + +`backfill_seals` writes via upsert, so a Cloud Tasks redelivery of the same batch — a +timeout on the response after the write already committed, say — is safe to reprocess: the +same window over the same source data produces the same final state. `created_at` on +`feed_reliability_seal` is insert-only, so even a redelivery cannot move a feed's tracking +start. + +`start_date` and `end_date` both arrive explicit. A worker never defaults them: the run's +window belongs to the run, not to the moment a particular batch happened to execute. + +Payload:: + + { + "run_id": str, # required — TaskExecutionTracker run id + "batch_id": str, # required — e.g. "batch-0003" + "stable_feed_ids": [str], # required, non-empty + "start_date": str, # required, ISO date + "end_date": str, # required, ISO date + "criteria": [str] | None, # optional + "only_missing": bool, # optional, default True + "snapshot_mode": str, # optional, default final + "resume_from_snapshot": bool # optional, default False + } +""" + +import logging +from typing import Optional + +from shared.database.database import with_db_session +from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker + +from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import _parse_day +from tasks.seal_of_reliability.backfill.seal_backfill import ( + DEFAULT_SNAPSHOT_MODE, + backfill_seals, +) +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, +) + +logger = logging.getLogger(__name__) + + +def seal_backfill_worker_handler(payload: dict) -> dict: + """Entry point for the `seal_backfill_worker` task.""" + payload = payload or {} + run_id = payload.get("run_id") + batch_id = payload.get("batch_id") + stable_feed_ids = payload.get("stable_feed_ids") + if not run_id or not batch_id: + raise ValueError("run_id and batch_id are required") + if not stable_feed_ids: + raise ValueError("stable_feed_ids is required and must be non-empty") + + start_date = _parse_day(payload.get("start_date"), "start_date") + end_date = _parse_day(payload.get("end_date"), "end_date") + if start_date is None or end_date is None: + # The producer resolves the window for the whole run. A worker that defaulted it + # could march to a different final day than its siblings. + raise ValueError("start_date and end_date are required") + + try: + result = backfill_seals( + stable_feed_ids=stable_feed_ids, + start_date=start_date, + end_date=end_date, + dry_run=False, + criteria=payload.get("criteria"), + only_missing=bool(payload.get("only_missing", True)), + snapshot_mode=payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE), + resume_from_snapshot=bool(payload.get("resume_from_snapshot", False)), + ) + except ( + Exception + ) as error: # infra failure, or every id in the batch turned ineligible + logger.exception( + "seal_backfill_worker failed for run=%s batch=%s", run_id, batch_id + ) + _mark_entry(run_id, batch_id, error=str(error)) + raise + + _mark_entry(run_id, batch_id, result=result) + return {"status": "ok", "batch_id": batch_id, **result} + + +@with_db_session +def _mark_entry( + run_id: str, + batch_id: str, + result: Optional[dict] = None, + error: Optional[str] = None, + db_session=None, +) -> None: + """Record this batch's completion in the run's TaskExecutionTracker. + + The stored metadata is what `seal_orchestrator_monitor` aggregates into the run-level + report, so the keys it reads — total_feeds, criterion_rows_written, seals_granted / + seals_revoked and their stable_id lists — must survive here. + """ + tracker = TaskExecutionTracker( + task_name=SEAL_BACKFILL_TASK_NAME, + run_id=run_id, + db_session=db_session, + ) + if error is None: + tracker.mark_completed(batch_id, metadata=result) + else: + tracker.mark_failed(batch_id, error_message=error) + db_session.commit() diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py index b2d2c94cd..cd61da558 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/context.py @@ -35,9 +35,10 @@ from datetime import date, datetime, timezone from typing import Any, Dict, Iterator, List, Mapping, Optional, Sequence +from sqlalchemy import select from sqlalchemy.orm import Session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion from tasks.seal_of_reliability.criteria import SealCriterionName @@ -97,13 +98,20 @@ def is_seal_eligible(feed) -> bool: def _eligible_stable_ids_query( - db_session: Session, stable_feed_ids: Optional[Sequence[str]] = None + db_session: Session, + stable_feed_ids: Optional[Sequence[str]] = None, + exclude_backfilled: bool = False, ): """Base query: `stable_id` of every seal-eligible GTFS feed. `stable_feed_ids`, if given, narrows the candidate set without changing the predicate. Left as `None`, every eligible feed in the catalog is returned; this is what the seal orchestrator (issue #1800) uses to enumerate the full batch to fan out. + + `exclude_backfilled` drops feeds that already have seal state, which is the backfill + producer's candidate set (#1763): a feed the nightly job already owns has real history + to carry forward and must not have a simulation written over it. It lives here rather + than in the producer so both the count and the stream apply one predicate. """ query = db_session.query(Gtfsfeed.stable_id).filter( Feed.status.notin_(INELIGIBLE_STATUSES), @@ -111,6 +119,11 @@ def _eligible_stable_ids_query( ) if stable_feed_ids is not None: query = query.filter(Feed.stable_id.in_(list(stable_feed_ids))) + if exclude_backfilled: + has_state = select(SealCriterion.__table__.c.feed_id).where( + SealCriterion.__table__.c.feed_id == Feed.id + ) + query = query.filter(~has_state.exists()) return query @@ -118,9 +131,14 @@ def count_eligible_feeds( db_session: Session, stable_feed_ids: Optional[Sequence[str]] = None, limit: Optional[int] = None, + exclude_backfilled: bool = False, ) -> int: """Cheap `COUNT(*)` of eligible feeds — no rows loaded.""" - query = _eligible_stable_ids_query(db_session, stable_feed_ids=stable_feed_ids) + query = _eligible_stable_ids_query( + db_session, + stable_feed_ids=stable_feed_ids, + exclude_backfilled=exclude_backfilled, + ) if limit is not None: query = query.limit(limit) return query.count() @@ -131,6 +149,7 @@ def iter_eligible_stable_ids( batch_size: int, stable_feed_ids: Optional[Sequence[str]] = None, limit: Optional[int] = None, + exclude_backfilled: bool = False, ) -> Iterator[List[str]]: """Stream eligible feeds' `stable_id`s in chunks of at most `batch_size`. @@ -141,7 +160,11 @@ def iter_eligible_stable_ids( if batch_size <= 0: raise ValueError("batch_size must be a positive integer") query = ( - _eligible_stable_ids_query(db_session, stable_feed_ids=stable_feed_ids) + _eligible_stable_ids_query( + db_session, + stable_feed_ids=stable_feed_ids, + exclude_backfilled=exclude_backfilled, + ) .order_by(Gtfsfeed.stable_id) .execution_options(stream_results=True) ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py index a7168be18..b3eae033a 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py @@ -31,9 +31,15 @@ a clear failure: the whole point of tracking start/end is to know when a nightly run did NOT fully update the seal for every feed. +The same monitor settles the backfill fan-out (#1763). Everything it does — poll, honour +the deadline, aggregate each batch's stored report — is identical for both; only the +TaskExecutionTracker `task_name` differs, so it is a payload parameter rather than a second +copy of this file. + Payload:: - { "run_id": str } # required + { "run_id": str, # required + "task_name": str } # optional, defaults to the nightly run's task name """ import logging @@ -64,16 +70,19 @@ def seal_orchestrator_monitor_handler(payload: dict) -> dict: """Entry point for the `seal_orchestrator_monitor` task.""" - run_id = (payload or {}).get("run_id") + payload = payload or {} + run_id = payload.get("run_id") if not run_id: raise ValueError("run_id is required") - return _monitor(run_id) + return _monitor(run_id, payload.get("task_name") or SEAL_ORCHESTRATOR_TASK_NAME) @with_db_session -def _monitor(run_id: str, db_session=None) -> dict: +def _monitor( + run_id: str, task_name: str = SEAL_ORCHESTRATOR_TASK_NAME, db_session=None +) -> dict: tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, + task_name=task_name, run_id=run_id, db_session=db_session, ) @@ -89,7 +98,7 @@ def _monitor(run_id: str, db_session=None) -> dict: # report the same aggregate (read-only, no mutation) rather than a bare status string: # this is the only way to see a settled run's feed-processing totals after the fact. if summary["run_status"] in _SETTLED_STATUSES: - aggregated = _aggregate_batches(db_session, run_id) + aggregated = _aggregate_batches(db_session, run_id, task_name) return { "run_id": run_id, "status": ( @@ -121,7 +130,7 @@ def _monitor(run_id: str, db_session=None) -> dict: f"run {run_id} still in progress: {summary['triggered']} batch(es) pending" ) - aggregated = _aggregate_batches(db_session, run_id) + aggregated = _aggregate_batches(db_session, run_id, task_name) incomplete = summary["triggered"] # > 0 only if the deadline was reached first final_status = ( STATUS_FAILED if summary["failed"] > 0 or incomplete > 0 else STATUS_COMPLETED @@ -152,12 +161,12 @@ def _monitor(run_id: str, db_session=None) -> dict: return result -def _aggregate_batches(db_session, run_id: str) -> Dict[str, Any]: +def _aggregate_batches(db_session, run_id: str, task_name: str) -> Dict[str, Any]: """Sum each completed batch's stored `update_seals` report into one run-level report.""" rows = ( db_session.query(TaskExecutionLog.metadata_) .filter( - TaskExecutionLog.task_name == SEAL_ORCHESTRATOR_TASK_NAME, + TaskExecutionLog.task_name == task_name, TaskExecutionLog.run_id == run_id, TaskExecutionLog.metadata_.isnot(None), ) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py new file mode 100644 index 000000000..5aed77384 --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py @@ -0,0 +1,349 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Unit tests for the seal backfill Cloud Tasks fan-out (issue #1763). + +Mirrors test_seal_orchestrator_fanout.py: the producer/worker orchestration with the DB and +Cloud Tasks boundaries mocked. The march itself is covered against the real database by +test_seal_backfill.py. +""" + +import unittest +from datetime import date +from unittest.mock import patch + +_PLAN = "tasks.seal_of_reliability.backfill.seal_backfill_orchestrator" +_WORKER = "tasks.seal_of_reliability.backfill.seal_backfill_worker" + +START = date(2025, 6, 1) +END = date(2026, 6, 1) + + +class TestBackfillOrchestrator(unittest.TestCase): + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=5) + def test_enqueues_worker_per_batch_plus_monitor( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter( + [["mdb-1", "mdb-2"], ["mdb-3", "mdb-4"], ["mdb-5"]] + ) + + result = seal_backfill_orchestrator_handler( + { + "dry_run": False, + "batch_size": 2, + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + start_run_mock.assert_called_once() + in_body = [c.kwargs["in_body_task"] for c in enqueue_mock.call_args_list] + self.assertEqual(in_body.count("seal_backfill_worker"), 3) + self.assertEqual(in_body.count("seal_orchestrator_monitor"), 1) + self.assertEqual(result["total_feeds"], 5) + self.assertEqual(result["batches"], 3) + self.assertEqual(result["enqueued"], 3) + + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) + def test_every_worker_gets_the_same_explicit_window( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """The whole reason the producer resolves the window. + + Left to each worker to default, two started either side of midnight would march to + different final days. + """ + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"], ["mdb-2"]]) + seal_backfill_orchestrator_handler( + { + "dry_run": False, + "batch_size": 1, + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + windows = [ + (c.kwargs["payload"]["start_date"], c.kwargs["payload"]["end_date"]) + for c in enqueue_mock.call_args_list + if c.kwargs["in_body_task"] == "seal_backfill_worker" + ] + self.assertEqual(windows, [(START.isoformat(), END.isoformat())] * 2) + + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) + def test_the_monitor_is_told_which_tracker_to_settle( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """The monitor is shared with the nightly run, so the task_name has to travel.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat()} + ) + + monitor = next( + c + for c in enqueue_mock.call_args_list + if c.kwargs["in_body_task"] == "seal_orchestrator_monitor" + ) + self.assertEqual( + monitor.kwargs["payload"]["task_name"], SEAL_BACKFILL_TASK_NAME + ) + + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) + def test_only_missing_narrows_the_candidate_set( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + """#1763's scope: feeds with no stored state. It is the eligibility predicate.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1", "mdb-2", "mdb-3"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat()} + ) + + self.assertTrue(count_mock.call_args.kwargs["exclude_backfilled"]) + self.assertTrue(iter_mock.call_args.kwargs["exclude_backfilled"]) + + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) + def test_only_missing_false_widens_it( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1", "mdb-2", "mdb-3"]]) + seal_backfill_orchestrator_handler( + {"dry_run": False, "end_date": END.isoformat(), "only_missing": False} + ) + + self.assertFalse(count_mock.call_args.kwargs["exclude_backfilled"]) + + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue") + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=4) + def test_dry_run_enqueues_nothing( + self, count_mock, iter_mock, enqueue_mock, start_run_mock + ): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + result = seal_backfill_orchestrator_handler({"end_date": END.isoformat()}) + + enqueue_mock.assert_not_called() + start_run_mock.assert_not_called() + self.assertTrue(result["dry_run"]) + self.assertEqual(result["enqueued"], 0) + self.assertEqual(result["total_feeds"], 4) + + @patch(f"{_PLAN}._mark_enqueue_failed") + @patch(f"{_PLAN}._start_run") + @patch(f"{_PLAN}._enqueue", return_value=False) + @patch(f"{_PLAN}.iter_eligible_stable_ids") + @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) + def test_a_failed_enqueue_fails_its_batch_immediately( + self, count_mock, iter_mock, enqueue_mock, start_run_mock, failed_mock + ): + """Otherwise the batch sits `triggered` until the deadline forces the run failed.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + iter_mock.return_value = iter([["mdb-1"], ["mdb-2"]]) + result = seal_backfill_orchestrator_handler( + {"dry_run": False, "batch_size": 1, "end_date": END.isoformat()} + ) + + self.assertEqual(result["enqueued"], 0) + self.assertEqual(failed_mock.call_count, 2) + + def test_a_bad_window_fails_at_the_producer(self): + """One failure at the producer beats the same failure once per batch.""" + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + with self.assertRaises(ValueError): + seal_backfill_orchestrator_handler( + { + "start_date": END.isoformat(), + "end_date": START.isoformat(), + } + ) + + def test_an_unknown_snapshot_mode_fails_at_the_producer(self): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + seal_backfill_orchestrator_handler, + ) + + with self.assertRaises(ValueError): + seal_backfill_orchestrator_handler({"snapshot_mode": "occasionally"}) + + +class TestBackfillWorker(unittest.TestCase): + @patch(f"{_WORKER}._mark_entry") + @patch(f"{_WORKER}.backfill_seals", return_value={"total_feeds": 2}) + def test_marks_the_batch_completed(self, backfill_mock, mark_mock): + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + result = seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1", "mdb-2"], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + + self.assertEqual(result["status"], "ok") + self.assertFalse(backfill_mock.call_args.kwargs["dry_run"]) + self.assertEqual(backfill_mock.call_args.kwargs["start_date"], START) + self.assertEqual(backfill_mock.call_args.kwargs["end_date"], END) + mark_mock.assert_called_once() + self.assertEqual(mark_mock.call_args.kwargs["result"], {"total_feeds": 2}) + + @patch(f"{_WORKER}._mark_entry") + @patch(f"{_WORKER}.backfill_seals", side_effect=RuntimeError("db down")) + def test_marks_the_batch_failed_and_re_raises(self, backfill_mock, mark_mock): + """Re-raised so Cloud Tasks retries; the tracker entry records the reason.""" + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + with self.assertRaises(RuntimeError): + seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1"], + "start_date": START.isoformat(), + "end_date": END.isoformat(), + } + ) + self.assertIn("db down", mark_mock.call_args.kwargs["error"]) + + def test_a_worker_never_defaults_the_window(self): + """The run's window belongs to the run, not to when a batch happened to execute.""" + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + with self.assertRaises(ValueError) as caught: + seal_backfill_worker_handler( + { + "run_id": "r1", + "batch_id": "batch-0000", + "stable_feed_ids": ["mdb-1"], + "start_date": START.isoformat(), + } + ) + self.assertIn("end_date", str(caught.exception)) + + def test_required_fields_are_checked(self): + from tasks.seal_of_reliability.backfill.seal_backfill_worker import ( + seal_backfill_worker_handler, + ) + + for payload in ( + {"batch_id": "b", "stable_feed_ids": ["mdb-1"]}, + {"run_id": "r", "stable_feed_ids": ["mdb-1"]}, + {"run_id": "r", "batch_id": "b", "stable_feed_ids": []}, + ): + with self.subTest(payload=payload): + with self.assertRaises(ValueError): + seal_backfill_worker_handler(payload) + + +class TestSharedMonitor(unittest.TestCase): + def test_it_defaults_to_the_nightly_tracker(self): + from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, + ) + from tasks.seal_of_reliability.orchestrator import seal_orchestrator_monitor + + with patch.object(seal_orchestrator_monitor, "_monitor") as monitor_mock: + seal_orchestrator_monitor.seal_orchestrator_monitor_handler( + {"run_id": "r1"} + ) + self.assertEqual(monitor_mock.call_args.args[1], SEAL_ORCHESTRATOR_TASK_NAME) + + def test_it_settles_the_backfill_tracker_when_told_to(self): + from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, + ) + from tasks.seal_of_reliability.orchestrator import seal_orchestrator_monitor + + with patch.object(seal_orchestrator_monitor, "_monitor") as monitor_mock: + seal_orchestrator_monitor.seal_orchestrator_monitor_handler( + {"run_id": "r1", "task_name": SEAL_BACKFILL_TASK_NAME} + ) + self.assertEqual(monitor_mock.call_args.args[1], SEAL_BACKFILL_TASK_NAME) + + +class TestBatchSizeDefault(unittest.TestCase): + def test_backfill_batches_are_smaller_than_nightly_ones(self): + """A batch marches a year per feed, so its cost scales with days as well as feeds.""" + from tasks.seal_of_reliability.backfill import seal_backfill_orchestrator + from tasks.seal_of_reliability.orchestrator import seal_orchestrator + + self.assertLess( + seal_backfill_orchestrator.DEFAULT_BATCH_SIZE, + seal_orchestrator.DEFAULT_BATCH_SIZE, + ) + self.assertGreater( + seal_backfill_orchestrator.DEFAULT_DEADLINE_SECONDS, + seal_orchestrator.DEFAULT_DEADLINE_SECONDS, + ) + + +if __name__ == "__main__": + unittest.main() From 4e2089e8b4ce49bcc3a241220671b2d65d476a22 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 26 Aug 2026 11:56:41 -0400 Subject: [PATCH 06/13] Drop pytest -s so log output no longer breaks the progress display --- scripts/api-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/api-tests.sh b/scripts/api-tests.sh index 621e3efc4..06ee85ca1 100755 --- a/scripts/api-tests.sh +++ b/scripts/api-tests.sh @@ -100,7 +100,7 @@ execute_tests() { # Run tests with coverage. Add the path to the main file and the shared packages that were linked. PT="src:tests:$PYTHONPATH" - PYTHONPATH="$PT" venv/bin/coverage run --branch -m pytest -s -W 'ignore::DeprecationWarning' tests + PYTHONPATH="$PT" venv/bin/coverage run --branch -m pytest -W 'ignore::DeprecationWarning' tests # Fail if tests fail if [ $? -ne 0 ]; then printf "\n${RED}Tests failed in $1${NC}\n" From 8b8fa166f3f64783d609e5396e9c390901858f7b Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 26 Aug 2026 11:57:21 -0400 Subject: [PATCH 07/13] Improved tests. --- .../backfill/test_backfill_matrix.py | 306 +++++++++++++ .../backfill/test_scripted_compliant.py | 340 -------------- .../backfill/test_scripted_evaluator.py | 420 ++++++++++++++++++ .../backfill/test_seal_backfill.py | 29 +- 4 files changed, 739 insertions(+), 356 deletions(-) create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py delete mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py new file mode 100644 index 000000000..778514d0a --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_backfill_matrix.py @@ -0,0 +1,306 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""End-state matrix for the backfill march (#1763): feed age x observation pattern. + +Every cell runs one real backfill over a 365-day window and asserts the three things that +survive it — `confirmed_status`, whether probation is open, and whether the feed holds the +seal. The point is not the state machine itself (covered day by day in +test_scripted_evaluator.py) but that a *march* of the right length lands on the right answer. + +Feed age is the second axis because the march start is clamped to `created_at`, so a younger +feed marches fewer days. Where that matters it is called out per cell: a criterion needs 180 +clean days to serve probation, and a feed younger than that simply cannot finish it inside +its own march however clean it is. + +Day 0 of each scenario is that feed's own march start, not the window start. + +**The criterion under test is not the real Official.** Every cell runs with `EVALUATORS` +patched to a single `ScriptedEvaluator`, which files its rows under the `official` name but +carries a 30-day grace period and the standard 180-day probation — for the duration of the +test only. The real `OfficialEvaluator` has neither and would flip the same day the flag +moves, making every row of this table identical and the age axis meaningless. The whole +matrix is about the debouncing mechanisms, so it has to supply a criterion that has them; +see test_scripted_evaluator.py. +""" + +import unittest +from datetime import date, datetime, timedelta, timezone + +from sqlalchemy import select +from unittest.mock import patch + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import FeedReliabilitySeal, SealCriterion +from tasks.seal_of_reliability.backfill.seal_backfill import backfill_seals +from tasks.seal_of_reliability.criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) +from test_scripted_evaluator import ( + TEST_GRACE, + Script, + ScriptedEvaluator, + cleanup, + seed_feed, +) +from test_shared.test_utils.database_utils import default_db_url + +PREFIX = "seal_mx_" + +WINDOW_DAYS = 365 +END = date(2026, 6, 1) +START = END - timedelta(days=WINDOW_DAYS) + +GRACE_DAYS = TEST_GRACE.days # 30 +PROBATION_DAYS = PROBATION_PERIOD.days # 180 + +# How long before `END` each band's feed was created. The march start is the later of the +# window start and the creation date, so only `old` is clamped by the window. +BANDS = { + "old": 800, # older than the window: marches all 366 days + "middle": 270, # inside the window, comfortably past the probation period + "young": 90, # inside the window, and shorter than probation +} + +PASS = CriterionStatus.PASS.value +FAIL = CriterionStatus.FAIL.value + +# Each pattern is a function of the march length, returning the failing day offsets. +PATTERNS = { + # Fails from the very first day and never recovers. + "all_fail": lambda n: range(0, n), + # Passes from the very first day and never fails. + "all_pass": lambda n: (), + # One bad first day, clean ever after. The first evaluation gets no grace, so it + # confirms — and recovery from a confirmed failure opens probation on day 1. + "fail_first_then_clean": lambda n: (0,), + # Clean, then fails from day 5 to the end. Confirms once the streak outlasts grace. + "clean_then_fails_to_the_end": lambda n: range(5, n), + # A single failing day, well inside the grace period. + "absorbed_blip": lambda n: (20,), + # A confirmed failure repaired too late for probation to be served by `end_date`. + "late_recovery": lambda n: range(n - 40, n - 4), + # The same failure repaired early enough that probation *may* be served, depending on + # how many days the feed's march actually has left. + "early_recovery": lambda n: range(5, 5 + GRACE_DAYS + 6), +} + +# (pattern, band) -> (confirmed_status, probation_open, has_seal) +EXPECTED = { + ("all_fail", "old"): (FAIL, True, False), + ("all_fail", "middle"): (FAIL, True, False), + ("all_fail", "young"): (FAIL, True, False), + ("all_pass", "old"): (PASS, False, True), + ("all_pass", "middle"): (PASS, False, True), + ("all_pass", "young"): (PASS, False, True), + # Probation opens on day 1 and needs 180 clean days. Only the young feed runs out of + # march before it can serve them. + ("fail_first_then_clean", "old"): (PASS, False, True), + ("fail_first_then_clean", "middle"): (PASS, False, True), + ("fail_first_then_clean", "young"): (PASS, True, False), + ("clean_then_fails_to_the_end", "old"): (FAIL, True, False), + ("clean_then_fails_to_the_end", "middle"): (FAIL, True, False), + ("clean_then_fails_to_the_end", "young"): (FAIL, True, False), + ("absorbed_blip", "old"): (PASS, False, True), + ("absorbed_blip", "middle"): (PASS, False, True), + ("absorbed_blip", "young"): (PASS, False, True), + ("late_recovery", "old"): (PASS, True, False), + ("late_recovery", "middle"): (PASS, True, False), + ("late_recovery", "young"): (PASS, True, False), + ("early_recovery", "old"): (PASS, False, True), + ("early_recovery", "middle"): (PASS, False, True), + # Same repair, same clean run afterwards — but 90 days of march cannot contain a + # 180-day probation, so the young feed ends still serving it. + ("early_recovery", "young"): (PASS, True, False), +} + + +def _created_at(age_days: int) -> datetime: + return datetime.combine( + END - timedelta(days=age_days), datetime.min.time(), tzinfo=timezone.utc + ) + + +def _march_start(age_days: int) -> date: + return max(START, END - timedelta(days=age_days)) + + +def _march_length(age_days: int) -> int: + return (END - _march_start(age_days)).days + 1 + + +class TestBackfillEndStateMatrix(unittest.TestCase): + """One real backfill per cell, asserting the state it leaves behind.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + @staticmethod + @with_db_session(db_url=default_db_url) + def _seed(stable_id, age_days, db_session=None): + seed_feed(db_session, stable_id, _created_at(age_days)) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _final_state(stable_id, db_session=None): + criterion = db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == stable_id, + SealCriterion.__table__.c.criterion == SealCriterionName.OFFICIAL.value, + ) + ).one() + seal = db_session.execute( + select(FeedReliabilitySeal.__table__).where( + FeedReliabilitySeal.__table__.c.feed_id == stable_id + ) + ).one() + return ( + criterion.confirmed_status, + criterion.probation_start is not None, + bool(seal.has_seal), + ) + + def _run_cell(self, pattern_name: str, band: str): + age = BANDS[band] + length = _march_length(age) + stable_id = f"{PREFIX}{band}_{pattern_name}"[:255] + + self._seed(stable_id, age) + script = Script.from_offsets( + _march_start(age), failing=PATTERNS[pattern_name](length) + ) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + return self._final_state(stable_id) + + def test_every_cell(self): + for (pattern_name, band), expected in sorted(EXPECTED.items()): + with self.subTest(pattern=pattern_name, band=band): + self.assertEqual( + self._run_cell(pattern_name, band), + expected, + f"{pattern_name} / {band}: expected " + f"(confirmed, probation_open, has_seal) = {expected}", + ) + + def test_the_bands_really_do_march_different_lengths(self): + """Guards the matrix: if the clamp broke, every band would march the same window.""" + self.assertEqual(_march_length(BANDS["old"]), WINDOW_DAYS + 1) + self.assertEqual(_march_length(BANDS["middle"]), BANDS["middle"] + 1) + self.assertEqual(_march_length(BANDS["young"]), BANDS["young"] + 1) + self.assertLess(_march_length(BANDS["young"]), PROBATION_DAYS) + + +class TestProbationBoundaryAcrossAges(unittest.TestCase): + """The exact age at which a feed becomes able to serve probation inside its own march. + + A bad first day opens probation on day 1, which clears on the first passing day at or + after day 1 + 180. So the feed needs a march reaching day 181 — one created 181 days + before `end_date` clears it on the very last day, and one created 180 days before does + not. Nothing else in the suite pins this. + """ + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + def _seal_after_bad_first_day(self, age_days: int) -> bool: + stable_id = f"{PREFIX}boundary_{age_days}" + TestBackfillEndStateMatrix._seed(stable_id, age_days) + script = Script.from_offsets(_march_start(age_days), failing=[0]) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + return TestBackfillEndStateMatrix._final_state(stable_id)[2] + + def test_one_day_short_of_serving_probation(self): + self.assertFalse(self._seal_after_bad_first_day(PROBATION_DAYS)) + + def test_exactly_long_enough_to_serve_probation(self): + self.assertTrue(self._seal_after_bad_first_day(PROBATION_DAYS + 1)) + + +class TestMarchEndingInsideTheGracePeriod(unittest.TestCase): + """A march can run out before a failure streak outlasts its grace period. + + A feed failing every observed day since day 5 still ends holding the seal, because the + 30-day grace period has not expired by `end_date`. Correct, and the one case where a + shorter march is *more* generous rather than less. + """ + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session, PREFIX) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session, PREFIX) + + def test_a_newborn_feed_keeps_the_seal_mid_streak(self): + age = 20 # marches 21 days; the streak from day 5 is 15 days old at the end + stable_id = f"{PREFIX}newborn" + TestBackfillEndStateMatrix._seed(stable_id, age) + script = Script.from_offsets( + _march_start(age), failing=range(5, _march_length(age)) + ) + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [ScriptedEvaluator(script)], + ): + backfill_seals( + stable_feed_ids=[stable_id], + start_date=START, + end_date=END, + dry_run=False, + ) + + confirmed, probation_open, has_seal = TestBackfillEndStateMatrix._final_state( + stable_id + ) + self.assertEqual(confirmed, PASS, "the streak is still inside the grace period") + self.assertFalse(probation_open) + self.assertTrue(has_seal) + self.assertLess(_march_length(age), 5 + GRACE_DAYS) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py deleted file mode 100644 index 71336e787..000000000 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_compliant.py +++ /dev/null @@ -1,340 +0,0 @@ -# -# MobilityData 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""A scripted stand-in for the Compliant criterion, and the tests proving it drives. - -Compliant (#1761) has no evaluator yet, and Official — the only one that does — has neither -a grace period nor probation. So nothing currently exercises the path-dependent behaviour a -backfill (#1763) exists to reconstruct: a failure streak debounced by a grace period, a -confirmed failure, the probation that follows recovery, and an UNKNOWN day that freezes the -lot. This stand-in supplies it, with Compliant's real policy values. - -Why scripted by day rather than driven through the database, as the stand-ins in -`test_seal_updater_db.py` are: those read `ctx.official` and a test moves them by issuing an -UPDATE between runs. A backfill marches its days in memory with no writes in between, so a -criterion it can drive has to answer from `ctx.now` alone. `ComplianceScript` is that — a set -of failing days and a set of unknown days, fixed up front, replayed by simply advancing the -clock. -""" - -import unittest -from contextlib import contextmanager -from dataclasses import dataclass, field -from datetime import date, datetime, timedelta, timezone -from typing import FrozenSet, Tuple -from unittest.mock import patch - -from sqlalchemy import delete, select - -from shared.database.database import with_db_session -from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion -from tasks.seal_of_reliability.criteria import ( - PROBATION_PERIOD, - CriterionStatus, - SealCriterionName, -) -from tasks.seal_of_reliability.evaluators import CriterionEvaluator -from tasks.seal_of_reliability.seal_updater import update_seals -from test_shared.test_utils.database_utils import default_db_url - -# Compliant's published policy (#1761): a failure streak is held for 30 days before the -# status flips, and recovery from a confirmed failure serves the standard 180-day probation. -COMPLIANT_GRACE = timedelta(days=30) - -DAY_ZERO = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) - -PREFIX = "seal_sc_" -FEED = f"{PREFIX}compliant" - - -def day(offset: int) -> datetime: - """The run timestamp `offset` days after day zero.""" - return DAY_ZERO + timedelta(days=offset) - - -@dataclass(frozen=True) -class ComplianceScript: - """What the stand-in answers on each day, fixed before the run starts. - - Days named in neither set pass. Keyed by `date` rather than by offset so a script stays - readable next to the assertions that check it. - """ - - failing: FrozenSet[date] = field(default_factory=frozenset) - unknown: FrozenSet[date] = field(default_factory=frozenset) - - @classmethod - def failing_on(cls, *offsets: int) -> "ComplianceScript": - return cls(failing=frozenset(day(offset).date() for offset in offsets)) - - @classmethod - def failing_between(cls, first: int, last: int) -> "ComplianceScript": - """Inclusive of both ends, which is how a failure streak is described.""" - return cls.failing_on(*range(first, last + 1)) - - def with_unknown_on(self, *offsets: int) -> "ComplianceScript": - return ComplianceScript( - failing=self.failing, - unknown=frozenset(day(offset).date() for offset in offsets), - ) - - -class ScriptedCompliantEvaluator(CriterionEvaluator): - """A Compliant stand-in whose verdict is a pure function of the day being evaluated. - - Borrows the `compliant` enum value, which has no evaluator of its own yet. Carries - Compliant's real grace period and inherits the default probation period, so the - debouncing under test is the one that will actually ship. - """ - - name = SealCriterionName.COMPLIANT - grace_period = COMPLIANT_GRACE - - def __init__(self, script: ComplianceScript): - self.script = script - - def _evaluate(self, ctx) -> Tuple[CriterionStatus, str]: - today = ctx.now.astimezone(timezone.utc).date() - if today in self.script.unknown: - # No validation report for the latest dataset — the real Compliant's UNKNOWN - # case, and the reason a never-validated feed does not sit at a confirmed - # failure. - return CriterionStatus.UNKNOWN, f"scripted: no report on {today}" - if today in self.script.failing: - return CriterionStatus.FAIL, f"scripted: errors on {today}" - return CriterionStatus.PASS, f"scripted: clean on {today}" - - -@contextmanager -def registry(script: ComplianceScript): - """Run with the scripted stand-in as the only criterion. - - Sole occupant on purpose: `update_seals` treats a shorter evaluator list as a partial run - and skips the has_seal roll-up, so patching the registry itself rather than filtering it - is what keeps the seal in play. - """ - with patch( - "tasks.seal_of_reliability.seal_updater.EVALUATORS", - [ScriptedCompliantEvaluator(script)], - ): - yield - - -def _cleanup(db_session): - db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{PREFIX}%"))) - db_session.commit() - - -class ScriptedCompliantTestCase(unittest.TestCase): - """Seeds one eligible feed and replays scripted days against it.""" - - @with_db_session(db_url=default_db_url) - def setUp(self, db_session): - _cleanup(db_session) - db_session.add( - Gtfsfeed( - id=FEED, - stable_id=FEED, - data_type="gtfs", - status="active", - operational_status="published", - official=True, - created_at=DAY_ZERO - timedelta(days=400), - producer_url=f"https://example.com/{FEED}.zip", - ) - ) - db_session.commit() - - @with_db_session(db_url=default_db_url) - def tearDown(self, db_session): - _cleanup(db_session) - - @staticmethod - def run_days(script: ComplianceScript, offsets) -> None: - """Evaluate the feed once per day, in order, writing each day's state. - - This is a march done the slow way — through the database, one `update_seals` call per - day — which is exactly what #1763's in-memory march has to reproduce. - """ - with registry(script): - for offset in offsets: - update_seals(stable_feed_ids=[FEED], dry_run=False, now=day(offset)) - - @staticmethod - @with_db_session(db_url=default_db_url) - def state(db_session=None): - return db_session.execute( - select(SealCriterion.__table__).where( - SealCriterion.__table__.c.feed_id == FEED, - SealCriterion.__table__.c.criterion - == SealCriterionName.COMPLIANT.value, - ) - ).one() - - -class TestScriptDrivesTheEvaluator(unittest.TestCase): - """The fixture itself, with no database in the way.""" - - class _Ctx: - def __init__(self, now): - self.now = now - self.feed_id = FEED - self.stable_id = FEED - - def observed(self, script: ComplianceScript, offset: int) -> CriterionStatus: - evaluator = ScriptedCompliantEvaluator(script) - return evaluator.evaluate(self._Ctx(day(offset))).observed_status - - def test_unnamed_days_pass(self): - self.assertIs( - self.observed(ComplianceScript.failing_on(3), 0), CriterionStatus.PASS - ) - - def test_named_days_fail(self): - self.assertIs( - self.observed(ComplianceScript.failing_on(3), 3), CriterionStatus.FAIL - ) - - def test_a_range_is_inclusive_of_both_ends(self): - script = ComplianceScript.failing_between(5, 7) - for offset, expected in ( - (4, CriterionStatus.PASS), - (5, CriterionStatus.FAIL), - (7, CriterionStatus.FAIL), - (8, CriterionStatus.PASS), - ): - with self.subTest(offset=offset): - self.assertIs(self.observed(script, offset), expected) - - def test_unknown_days_win_over_failing_days(self): - """An input we could not read is not a failure, whatever else the script says.""" - script = ComplianceScript.failing_on(3).with_unknown_on(3) - self.assertIs(self.observed(script, 3), CriterionStatus.UNKNOWN) - - def test_it_carries_compliant_policy(self): - self.assertEqual(ScriptedCompliantEvaluator.grace_period, timedelta(days=30)) - self.assertEqual( - ScriptedCompliantEvaluator.probation_period, - PROBATION_PERIOD, - "the default 180 days, not an override", - ) - - -class TestGracePeriod(ScriptedCompliantTestCase): - def test_a_short_streak_is_absorbed(self): - """29 failing days is inside the 30-day grace period, so the status holds.""" - script = ComplianceScript.failing_between(1, 29) - self.run_days(script, range(0, 30)) - - row = self.state() - self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) - self.assertEqual( - row.confirmed_status, - CriterionStatus.PASS.value, - "still inside the grace period on day 29", - ) - self.assertIsNone(row.probation_start, "an absorbed failure opens no probation") - - def test_a_streak_past_the_grace_period_confirms(self): - script = ComplianceScript.failing_between(1, 40) - self.run_days(script, range(0, 41)) - - row = self.state() - self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) - self.assertIsNotNone(row.last_confirmed_failure_at) - - def test_the_first_evaluation_gets_no_grace(self): - """A criterion that has never passed has no track record to hold.""" - self.run_days(ComplianceScript.failing_on(0), [0]) - self.assertEqual(self.state().confirmed_status, CriterionStatus.FAIL.value) - - -class TestProbationFollowsRecovery(ScriptedCompliantTestCase): - def test_recovery_from_a_confirmed_failure_opens_probation(self): - script = ComplianceScript.failing_between(1, 40) - self.run_days(script, list(range(0, 42))) # day 41 is the repair - - row = self.state() - self.assertEqual( - row.confirmed_status, - CriterionStatus.PASS.value, - "the check passes again", - ) - self.assertIsNotNone( - row.probation_start, "but it is serving probation for the failure" - ) - - def test_probation_suspends_the_grace_period(self): - """One bad day during probation confirms at once, and restarts the count. - - Off probation, a single failing day sits well inside the 30-day grace period and - would confirm nothing. This is the ratchet that makes a cold start's error persist. - """ - script = ComplianceScript( - failing=frozenset( - [day(offset).date() for offset in range(1, 41)] + [day(60).date()] - ) - ) - self.run_days(script, list(range(0, 62))) - - row = self.state() - self.assertEqual( - row.last_confirmed_failure_at.astimezone(timezone.utc).date(), - day(60).date(), - "the single day confirmed because probation had suspended the grace period", - ) - self.assertEqual( - row.probation_start.astimezone(timezone.utc).date(), - day(61).date(), - "and probation restarted from the day after it", - ) - - def test_probation_clears_once_served(self): - script = ComplianceScript.failing_between(1, 40) - served = 41 + PROBATION_PERIOD.days - self.run_days(script, [*range(0, 42), served]) - - self.assertIsNone( - self.state().probation_start, "the full stretch has been served" - ) - - -class TestUnknownFreezesTheState(ScriptedCompliantTestCase): - def test_an_unknown_day_leaves_the_verdict_standing(self): - script = ComplianceScript().with_unknown_on(1) - self.run_days(script, [0, 1]) - - row = self.state() - self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) - self.assertEqual( - row.confirmed_status, - CriterionStatus.PASS.value, - "a missing input must never read as a failure", - ) - - def test_an_unknown_day_does_not_advance_a_failure_streak(self): - """The streak keeps its start, so the grace period is not quietly extended.""" - script = ComplianceScript.failing_between(1, 5).with_unknown_on(3) - self.run_days(script, range(0, 6)) - - row = self.state() - self.assertEqual( - row.first_observed_failure_at.astimezone(timezone.utc).date(), - day(1).date(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py new file mode 100644 index 000000000..5cbdd29f5 --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_scripted_evaluator.py @@ -0,0 +1,420 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A day-scripted stand-in evaluator, and the tests proving it drives. + +Only Official is implemented, and it has neither a grace period nor probation. So nothing +otherwise exercises the path-dependent behaviour a backfill (#1763) exists to reconstruct: a +failure streak debounced by a grace period, a confirmed failure, the probation that follows +recovery, and an UNKNOWN day that freezes the lot. + +**It borrows the `official` enum value on purpose.** Official already has an evaluator, so no +future criterion implementation can collide with this fixture — unlike borrowing `compliant` +or `available`, whose real evaluators are still to be written (#1782, #1784). The grace and +probation values below are the harness's, chosen to exercise the state machine; the real +Official has neither, and nothing here should be read as its policy. + +Why scripted by day rather than driven through the database, as the stand-ins in +`test_seal_updater_db.py` are: those read `ctx.official` and a test moves them by issuing an +UPDATE between runs. A backfill marches its days in memory with no writes in between, so a +criterion it can drive has to answer from `ctx.now` alone. `Script` is that — sets of failing, +unknown and not-applicable days fixed up front, replayed by advancing the clock. +""" + +import unittest +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta, timezone +from typing import FrozenSet, Iterable, Optional, Tuple +from unittest.mock import patch + +from sqlalchemy import delete, select + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import Feed, Gtfsfeed, SealCriterion +from tasks.seal_of_reliability.criteria import ( + PROBATION_PERIOD, + CriterionStatus, + SealCriterionName, +) +from tasks.seal_of_reliability.evaluators import CriterionEvaluator, OfficialEvaluator +from tasks.seal_of_reliability.seal_updater import update_seals +from test_shared.test_utils.database_utils import default_db_url + +# The harness's debouncing values, not any criterion's published policy. 30 days is long +# enough that a streak has to be deliberate to outlast it, and short enough that a test can +# step over the boundary without marching a year. +TEST_GRACE = timedelta(days=30) + +DAY_ZERO = datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc) + +PREFIX = "seal_sc_" +FEED = f"{PREFIX}scripted" + +# Distinguishes "not given" from an explicit None, which means "this criterion has no grace +# period" and is a value a test may legitimately want to pass. +_UNSET = object() + + +def day(offset: int) -> datetime: + """The run timestamp `offset` days after day zero.""" + return DAY_ZERO + timedelta(days=offset) + + +@dataclass(frozen=True) +class Script: + """What the stand-in answers on each day, fixed before the run starts. + + Days named in none of the three sets pass. Built from offsets against an anchor — a + feed's march start, usually — so a scenario reads in the same relative terms whatever + day the window actually begins on. + """ + + failing: FrozenSet[date] = field(default_factory=frozenset) + unknown: FrozenSet[date] = field(default_factory=frozenset) + not_applicable: FrozenSet[date] = field(default_factory=frozenset) + + @classmethod + def from_offsets( + cls, + anchor: date, + failing: Iterable[int] = (), + unknown: Iterable[int] = (), + not_applicable: Iterable[int] = (), + ) -> "Script": + def days(offsets): + return frozenset(anchor + timedelta(days=int(o)) for o in offsets) + + return cls(days(failing), days(unknown), days(not_applicable)) + + def status_on(self, today: date) -> CriterionStatus: + """Precedence matters: an input we could not read is never a failure.""" + if today in self.unknown: + return CriterionStatus.UNKNOWN + if today in self.not_applicable: + return CriterionStatus.NOT_APPLICABLE + if today in self.failing: + return CriterionStatus.FAIL + return CriterionStatus.PASS + + +class ScriptedEvaluator(CriterionEvaluator): + """A stand-in whose verdict is a pure function of the day being evaluated. + + It files its rows under the `official` criterion — see the module docstring for why that + name and not one of the unimplemented ones. + + **Its debouncing is not Official's.** The real `OfficialEvaluator` has `grace_period` and + `probation_period` both `None`: it is a point-in-time check that flips the same day the + flag moves, in either direction. This stand-in gives that name a 30-day grace period and + the standard 180-day probation *for the duration of the test only*, because those are the + mechanisms a backfill has to reconstruct and no implemented criterion has them yet. + + Nothing here changes the real evaluator. `registry()` patches the whole `EVALUATORS` + list for the length of a `with` block, so the substitution cannot leak past it. + """ + + name = SealCriterionName.OFFICIAL + grace_period = TEST_GRACE + + def __init__( + self, + script: Script, + criterion: Optional[SealCriterionName] = None, + grace_period=_UNSET, + probation_period=_UNSET, + ): + self.script = script + # Instance attributes shadow the class ones the job reads, so a test can vary the + # policy per case without subclassing. + if criterion is not None: + self.name = criterion + if grace_period is not _UNSET: + self.grace_period = grace_period + if probation_period is not _UNSET: + self.probation_period = probation_period + + def _evaluate(self, ctx) -> Tuple[CriterionStatus, str]: + today = ctx.now.astimezone(timezone.utc).date() + status = self.script.status_on(today) + return status, f"scripted: {status.value} on {today}" + + +@contextmanager +def registry(evaluator: ScriptedEvaluator): + """Run with the stand-in as the only criterion. + + Sole occupant on purpose: `update_seals` treats a shorter evaluator list as a partial run + and skips the has_seal roll-up, so patching the registry itself rather than filtering it + is what keeps the seal in play. Patched in `seal_updater`, which is where both the nightly + job and the backfill resolve the registry from. + """ + with patch( + "tasks.seal_of_reliability.seal_updater.EVALUATORS", + [evaluator], + ): + yield + + +def cleanup(db_session, prefix: str = PREFIX): + """Delete from `feed`, not `gtfsfeed`. + + Gtfsfeed is a joined-table subclass, so deleting the subclass leaves the parent row and + the next insert collides on feed_pkey. The seal tables are ON DELETE CASCADE. + """ + db_session.execute(delete(Feed).where(Feed.stable_id.like(f"{prefix}%"))) + db_session.commit() + + +def seed_feed(db_session, stable_id: str, created_at: datetime, official=True): + db_session.add( + Gtfsfeed( + id=stable_id, + stable_id=stable_id, + data_type="gtfs", + status="active", + operational_status="published", + official=official, + created_at=created_at, + producer_url=f"https://example.com/{stable_id}.zip", + ) + ) + db_session.flush() + + +class ScriptedEvaluatorTestCase(unittest.TestCase): + """Seeds one eligible feed and replays scripted days against it.""" + + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + cleanup(db_session) + seed_feed(db_session, FEED, DAY_ZERO - timedelta(days=400)) + db_session.commit() + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + cleanup(db_session) + + @staticmethod + def run_days(script: Script, offsets) -> None: + """Evaluate the feed once per day, in order, writing each day's state. + + A march done the slow way — through the database, one `update_seals` call per day — + which is exactly what #1763's in-memory march has to reproduce. + """ + with registry(ScriptedEvaluator(script)): + for offset in offsets: + update_seals(stable_feed_ids=[FEED], dry_run=False, now=day(offset)) + + @staticmethod + @with_db_session(db_url=default_db_url) + def state(db_session=None): + return db_session.execute( + select(SealCriterion.__table__).where( + SealCriterion.__table__.c.feed_id == FEED, + SealCriterion.__table__.c.criterion == SealCriterionName.OFFICIAL.value, + ) + ).one() + + +class TestScriptDrivesTheEvaluator(unittest.TestCase): + """The fixture itself, with no database in the way.""" + + class _Ctx: + def __init__(self, now): + self.now = now + self.feed_id = FEED + self.stable_id = FEED + + def observed(self, script: Script, offset: int) -> CriterionStatus: + return ( + ScriptedEvaluator(script).evaluate(self._Ctx(day(offset))).observed_status + ) + + def test_unnamed_days_pass(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3]) + self.assertIs(self.observed(script, 0), CriterionStatus.PASS) + + def test_named_days_fail(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.FAIL) + + def test_a_range_is_inclusive_of_both_ends(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(5, 8)) + for offset, expected in ( + (4, CriterionStatus.PASS), + (5, CriterionStatus.FAIL), + (7, CriterionStatus.FAIL), + (8, CriterionStatus.PASS), + ): + with self.subTest(offset=offset): + self.assertIs(self.observed(script, offset), expected) + + def test_unknown_wins_over_failing(self): + """An input we could not read is not a failure, whatever else the script says.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=[3], unknown=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.UNKNOWN) + + def test_not_applicable_wins_over_failing(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=[3], not_applicable=[3]) + self.assertIs(self.observed(script, 3), CriterionStatus.NOT_APPLICABLE) + + def test_it_borrows_official_so_nothing_future_can_collide(self): + self.assertIs(ScriptedEvaluator(Script()).name, SealCriterionName.OFFICIAL) + + def test_policy_is_the_harness_own_and_overridable(self): + default = ScriptedEvaluator(Script()) + self.assertEqual(default.grace_period, TEST_GRACE) + self.assertEqual(default.probation_period, PROBATION_PERIOD) + + custom = ScriptedEvaluator(Script(), grace_period=None, probation_period=None) + self.assertIsNone(custom.grace_period) + self.assertIsNone(custom.probation_period) + + def test_the_stand_in_debounces_where_the_real_official_does_not(self): + """Pins the divergence, so it is a fact rather than a comment. + + The real Official is a point-in-time check with neither mechanism. Every scenario + built on this fixture depends on the stand-in having both — so if Official were ever + given a grace period for real, this fails and says which assumption moved. + """ + self.assertIsNone(OfficialEvaluator.grace_period) + self.assertIsNone(OfficialEvaluator.probation_period) + + stand_in = ScriptedEvaluator(Script()) + self.assertIs(stand_in.name, OfficialEvaluator.name) + self.assertIsNotNone(stand_in.grace_period) + self.assertIsNotNone(stand_in.probation_period) + + def test_the_substitution_does_not_outlive_the_context(self): + """`registry()` swaps the whole list, so nothing leaks into a later test.""" + from tasks.seal_of_reliability import seal_updater + + before = list(seal_updater.EVALUATORS) + with registry(ScriptedEvaluator(Script())): + self.assertEqual(len(seal_updater.EVALUATORS), 1) + self.assertEqual(list(seal_updater.EVALUATORS), before) + + +class TestGracePeriod(ScriptedEvaluatorTestCase): + def test_a_short_streak_is_absorbed(self): + """29 failing days is inside the 30-day grace period, so the status holds.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 30)) + self.run_days(script, range(0, 30)) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.FAIL.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "still inside the grace period on day 29", + ) + self.assertIsNone(row.probation_start, "an absorbed failure opens no probation") + + def test_a_streak_past_the_grace_period_confirms(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + self.run_days(script, range(0, 41)) + + row = self.state() + self.assertEqual(row.confirmed_status, CriterionStatus.FAIL.value) + self.assertIsNotNone(row.last_confirmed_failure_at) + + def test_the_first_evaluation_gets_no_grace(self): + """A criterion that has never passed has no track record to hold.""" + self.run_days(Script.from_offsets(DAY_ZERO.date(), failing=[0]), [0]) + self.assertEqual(self.state().confirmed_status, CriterionStatus.FAIL.value) + + +class TestProbationFollowsRecovery(ScriptedEvaluatorTestCase): + def test_recovery_from_a_confirmed_failure_opens_probation(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + self.run_days(script, list(range(0, 42))) # day 41 is the repair + + row = self.state() + self.assertEqual( + row.confirmed_status, CriterionStatus.PASS.value, "the check passes again" + ) + self.assertIsNotNone( + row.probation_start, "but it is serving probation for the failure" + ) + + def test_probation_suspends_the_grace_period(self): + """One bad day during probation confirms at once, and restarts the count. + + Off probation, a single failing day sits well inside the 30-day grace period and + would confirm nothing. This is the ratchet that makes a cold start's error persist. + """ + script = Script.from_offsets(DAY_ZERO.date(), failing=list(range(1, 41)) + [60]) + self.run_days(script, list(range(0, 62))) + + row = self.state() + self.assertEqual( + row.last_confirmed_failure_at.astimezone(timezone.utc).date(), + day(60).date(), + "the single day confirmed because probation had suspended the grace period", + ) + self.assertEqual( + row.probation_start.astimezone(timezone.utc).date(), + day(61).date(), + "and probation restarted from the day after it", + ) + + def test_probation_clears_once_served(self): + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 41)) + served = 41 + PROBATION_PERIOD.days + self.run_days(script, [*range(0, 42), served]) + + self.assertIsNone( + self.state().probation_start, "the full stretch has been served" + ) + + +class TestNoVerdictDays(ScriptedEvaluatorTestCase): + def test_an_unknown_day_leaves_the_verdict_standing(self): + script = Script.from_offsets(DAY_ZERO.date(), unknown=[1]) + self.run_days(script, [0, 1]) + + row = self.state() + self.assertEqual(row.observed_status, CriterionStatus.UNKNOWN.value) + self.assertEqual( + row.confirmed_status, + CriterionStatus.PASS.value, + "a missing input must never read as a failure", + ) + + def test_an_unknown_day_does_not_advance_a_failure_streak(self): + """The streak keeps its start, so the grace period is not quietly extended.""" + script = Script.from_offsets(DAY_ZERO.date(), failing=range(1, 6), unknown=[3]) + self.run_days(script, range(0, 6)) + + self.assertEqual( + self.state().first_observed_failure_at.astimezone(timezone.utc).date(), + day(1).date(), + ) + + def test_a_not_applicable_day_withdraws_the_criterion(self): + script = Script.from_offsets(DAY_ZERO.date(), not_applicable=[1]) + self.run_days(script, [0, 1]) + + row = self.state() + self.assertEqual( + row.confirmed_status, + CriterionStatus.NOT_APPLICABLE.value, + "it leaves the roll-up rather than being frozen in it", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py index 5c161b48b..9dfe93247 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -52,7 +52,7 @@ ) from tasks.seal_of_reliability.criteria import SealCriterionName from tasks.seal_of_reliability.seal_updater import update_seals -from test_scripted_compliant import ComplianceScript, ScriptedCompliantEvaluator +from test_scripted_evaluator import Script, ScriptedEvaluator from test_shared.test_utils.database_utils import default_db_url PREFIX = "seal_bf_" @@ -327,12 +327,17 @@ def test_handler_threads_the_payload_through(self): MARCHED = f"{PREFIX}marched" REPLAYED = f"{PREFIX}replayed" +# The march tests below run with EVALUATORS patched to a single ScriptedEvaluator. It files +# its rows under the `official` criterion but carries a 30-day grace period and the standard +# 180-day probation, for the duration of each test only — the real OfficialEvaluator has +# neither. Those two mechanisms are what a backfill has to reconstruct, and no implemented +# criterion has them yet; see test_scripted_evaluator.py. +# # A short window, so the equivalence test replays a tractable number of days through the -# database. The failing run is long enough to outlast the stand-in's 30-day grace period, -# so the comparison covers a confirmed failure and the probation that follows it. +# database. The failing run is long enough to outlast that 30-day grace period, so the +# comparison covers a confirmed failure and the probation that follows it. MARCH_START = date(2026, 1, 1) MARCH_END = date(2026, 3, 15) -FAILING = ComplianceScript.failing_between(10, 45) STATE_COLUMNS = ( "observed_status", @@ -347,16 +352,8 @@ def test_handler_threads_the_payload_through(self): def _script_for(offsets_from_march_start): - """A ComplianceScript whose failing days are offsets from MARCH_START. - - `ComplianceScript` counts from its own day zero, so the offsets are rebased here rather - than the fixture being reconfigured. - """ - return ComplianceScript( - failing=frozenset( - MARCH_START + timedelta(days=offset) for offset in offsets_from_march_start - ) - ) + """A Script whose failing days are offsets from MARCH_START.""" + return Script.from_offsets(MARCH_START, failing=offsets_from_march_start) @with_db_session(db_url=default_db_url) @@ -407,7 +404,7 @@ def registry(script): """Patch the registry `_resolve_evaluators` reads, which is the one both paths use.""" return patch( "tasks.seal_of_reliability.seal_updater.EVALUATORS", - [ScriptedCompliantEvaluator(script)], + [ScriptedEvaluator(script)], ) @staticmethod @@ -435,7 +432,7 @@ def replay_through_db(stable_id, script): ) def state_of(self, stable_id): - row = criterion_rows(stable_id)[SealCriterionName.COMPLIANT.value] + row = criterion_rows(stable_id)[SealCriterionName.OFFICIAL.value] return {column: getattr(row, column) for column in STATE_COLUMNS} From a536424056a2df6f2af641f874a3aa582721dcaa Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 26 Aug 2026 13:48:16 -0400 Subject: [PATCH 08/13] Trim comments --- .../backfill/backfill_seal_of_reliability.py | 55 ++--- .../backfill/seal_backfill.py | 217 ++++++------------ .../backfill/seal_backfill_orchestrator.py | 51 ++-- .../backfill/seal_backfill_worker.py | 26 +-- 4 files changed, 111 insertions(+), 238 deletions(-) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py index 636e5a0f2..8abddd17b 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -30,12 +30,10 @@ def _parse_day(value: Optional[str], field: str) -> Optional[date]: - """Parse a payload date string to a `date`, or None when it is absent. + """Parse a payload date string to a `date`, or None when absent. - A plain date is what the window is expressed in, but an operator copying a value from a - log or from the nightly task's `now` will paste a full timestamp. Accept both rather than - failing on a value whose meaning is unambiguous; the time of day is dropped either way, - since the march is day-granular. + Accepts a full timestamp too — an operator pasting the nightly task's `now` should not + hit a parse error. The march is day-granular, so the time is dropped either way. """ if value is None: return None @@ -71,38 +69,23 @@ def get_parameters(payload: dict): def backfill_seal_of_reliability_handler(payload: dict) -> dict: - """ - Handler for the Seal of Reliability backfill. - - A dry run returns the resolved plan without marching or writing. + """Handler for the Seal of Reliability backfill. A dry run returns the plan only. - Payload parameters: - stable_feed_ids (list[str]): Required and non-empty. The feeds to backfill; there is - no run-the-whole-catalogue mode. Ineligible ids are skipped with a - logged warning, and it raises if none can be used. - start_date (str | None): First day of the window, ISO date. Clamped up to each feed's - own created_at. Default: end_date minus days_back. - end_date (str | None): Last day simulated, and the day the written state belongs to. - Resolved once for the whole run. Default: yesterday UTC. - days_back (int): Window length used when start_date is absent. Default: 365 — the - "12 months" of #1763, a cost/coverage default rather than a - correctness threshold. - dry_run (bool): Resolve and return the plan without marching or writing. - Default: True. - limit (int | None): Cap the number of feeds, from the list. Default: no limit. - criteria (list[str] | None): Backfill only these criteria. Default: None, meaning - every implemented criterion. - batch_size (int): Feeds loaded and marched per batch. Default: 200. - only_missing (bool): Skip feeds that already have seal state, which is #1763's stated - scope. False re-backfills them, overwriting what is stored. - Default: True. - snapshot_mode (str): "final" (only the last day, per #1763), "all" (every simulated - day — millions of rows over a year, but what would let #1803 resume - inside the backfilled window), or "none". Default: "final". - resume_from_snapshot (bool): Seed each criterion from its snapshot at march_start - 1 - rather than cold-starting empty. The #1803 hook. Default: False. - max_reported_feeds (int): Cap on the `feeds` list in the response; `feeds_omitted` - reports how many entries were left out. Default: 50. + Payload, all optional but `stable_feed_ids`: + stable_feed_ids required, non-empty; there is no run-the-whole-catalogue mode + start_date ISO date, clamped up to each feed's created_at. Default: end_date + minus days_back + end_date ISO date, last day simulated. Default: yesterday UTC + days_back window length when start_date is absent. Default: 365 + dry_run Default: True + limit cap the number of feeds. Default: no limit + criteria restrict to these criteria. Default: every implemented one + batch_size Default: 200 + only_missing skip feeds that already have seal state. Default: True + snapshot_mode final | all | none. Default: final + resume_from_snapshot seed from the snapshot before march_start (#1803). + Default: False + max_reported_feeds cap on the `feeds` list in the response. Default: 50 """ ( stable_feed_ids, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index d8f549715..0330af8d6 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -15,38 +15,19 @@ # """Seal of Reliability backfill (issue #1763). -Establishes a starting seal state for feeds that have none, so the nightly job (#1761) has a -"yesterday" to step from. For each feed it cold-starts at `march_start`, replays the nightly -evaluation forward one day at a time to `end_date`, and writes only the final day. The -intermediate days are held in memory and discarded — marching forward is what builds up the -path-dependent state (grace-period streaks, probation) that makes the final state right. - -A dry run resolves and returns the plan — which feeds, which window per feed, how many days — -without marching or writing anything. - -Per-feed window ---------------- -`march_start = max(start_date, feed.created_at)`. Clamping to the feed's own creation date -does two things: it skips days before the feed existed, and it is the value the Stable -criterion measures its 180 days from. A feed younger than the window therefore gets an exact -cold start rather than a guessed one — there is no history before its creation to be wrong -about. - -`end_date` is resolved once by the caller and passed down, never recomputed per feed. Two -workers of the same run started either side of midnight would otherwise march to different -final days. - -What the backfill cannot know ------------------------------ -Official and Stable have no historical record, so they can only be evaluated against their -current values. Neither has a grace period or probation, so a wrong value on a past day does -not propagate into the days after it (see #1763). - -The cold start assumes an empty prior state — no failure streak, no probation — which may not -match reality for a feed whose history is truncated by `start_date`. Errors from that -assumption are not bounded by the window: a single observed failure inside it can extend the -divergence by another probation period, and repeatedly. The window is therefore a -cost/coverage default, not a correctness guarantee. +Gives feeds with no seal state a starting one, so the nightly job (#1761) has a "yesterday" +to step from: cold-start each feed at `march_start`, replay the nightly evaluation forward a +day at a time to `end_date`, write only the final day. Marching is what builds the +path-dependent state (grace streaks, probation) the final state depends on. A dry run returns +the plan without writing. + +`march_start = max(start_date, feed.created_at)` — skips days before the feed existed, and is +what Stable counts its 180 days from. `end_date` is resolved once by the caller, never per +feed, so every feed of a run ends on the same day. + +Two limits, argued in misc/AI/seal_backfill_algorithm_1763.md: Official and Stable have no +history and are read at today's values; and the cold start's error is not bounded by the +window, so `days_back` is a cost/coverage default rather than a correctness guarantee. """ import logging @@ -86,19 +67,12 @@ logger = logging.getLogger(__name__) -# How far back the window reaches when `start_date` is not given. Expressed in days rather -# than months so the arithmetic is exact and needs no calendar library: 365 is the "12 months" -# of #1763. It is roughly twice the 180-day probation period, which is where the number came -# from — but see the module docstring: that reasoning bounds nothing, so treat this as a -# default for how much history to replay rather than as a correctness threshold. +# Days rather than months, so the arithmetic needs no calendar library: 365 is #1763's +# "12 months". DEFAULT_DAYS_BACK: int = 365 -# What to record in seal_criterion_snapshot. -# final — only the last day's state, per #1763. The intermediate days are discarded. -# all — every simulated day. Costs len(days) x feeds x criteria rows, which is millions -# over a year, but it is what would let #1803 resume inside the backfilled window -# instead of cold-starting again. -# none — write nothing to the snapshot table. +# What to record in seal_criterion_snapshot: only the last day (per #1763), every simulated +# day (millions of rows over a year, but what lets #1803 resume inside the window), or none. SNAPSHOT_MODES: Tuple[str, ...] = ("final", "all", "none") DEFAULT_SNAPSHOT_MODE: str = "final" @@ -115,11 +89,7 @@ def resolve_window( end_date: Optional[date], days_back: int, ) -> Tuple[date, date]: - """Resolve the run-wide window, applying defaults and rejecting a nonsensical one. - - Resolved once for the whole run rather than per feed, so every feed of a run marches to - the same final day whatever time the run started or how long it takes. - """ + """Resolve the run-wide window once, so every feed of a run ends on the same day.""" if days_back <= 0: raise ValueError("days_back must be a positive integer") @@ -135,17 +105,13 @@ def resolve_window( def march_start_for(feed: Gtfsfeed, start_date: date) -> date: - """Where this feed's march begins: the later of the window start and its creation. + """The later of the window start and the feed's creation. - Clamping to `feed.created_at` is not only an optimisation. It is also the value the - Stable criterion counts its 180 days from, and it is what makes the cold start exact for - a feed younger than the window: such a feed has no history before its creation, so the - empty starting state is the truth rather than an assumption. + Also the value Stable counts from, and what makes the cold start exact for a feed younger + than the window: it has no history before its creation to be wrong about. """ created = feed.created_at - if created is None: - # created_at is NOT NULL in the schema, so this is defensive only: a feed with no - # creation date gets the full window rather than being skipped. + if created is None: # NOT NULL in the schema; defensive only return start_date created_day = ( created.astimezone(timezone.utc).date() @@ -156,11 +122,9 @@ def march_start_for(feed: Gtfsfeed, start_date: date) -> date: def _feeds_with_seal_state(db_session: Session, feed_ids: Sequence[str]) -> Set[str]: - """The subset of `feed_ids` that already has at least one seal_criterion row. + """Feeds that already have seal state, which `only_missing` excludes. - `only_missing` filters on this: #1763 backfills feeds that have no stored state to carry - forward, and re-running the march over a feed the nightly job already owns would throw - away real history in favour of a simulation of it. + Re-marching a feed the nightly job owns would write a simulation over real history. """ if not feed_ids: return set() @@ -175,10 +139,8 @@ def _feeds_with_seal_state(db_session: Session, feed_ids: Sequence[str]) -> Set[ def day_start(day: date) -> datetime: """The `now` a simulated day is evaluated at: midnight UTC. - A fixed time of day, so that `snapshot_date_of(now)` is the day itself and - `_next_day_start(now)` — which probation uses — lands on the following midnight with no - rounding to reason about. The nightly job's own `now` is whatever time it ran; the march - only has to be consistent with itself and day-aligned. + Fixed so `snapshot_date_of(now)` is the day itself and probation's `_next_day_start(now)` + lands on the following midnight, with no rounding to reason about. """ return datetime.combine(day, time.min, tzinfo=timezone.utc) @@ -189,12 +151,11 @@ def days_between(first: date, last: date) -> List[date]: def _state_from_snapshot(row) -> SealCriterionState: - """Rebuild a `SealCriterionState` from one seal_criterion_snapshot row. + """Rebuild a `SealCriterionState` from one snapshot row. - The state columns are taken from the table rather than listed, the mirror of - `seal_updater._snapshot_row` which writes them: a column added to the snapshot table is - read back without touching this function, and fails loudly here if `SealCriterionState` - has no field for it rather than being silently dropped. + Columns come from the table rather than a list — the mirror of `_snapshot_row`, which + writes them — so a new column is read back without editing this, and fails loudly if + `SealCriterionState` has no field for it. """ values = {} for column in SNAPSHOT_STATE_COLUMNS: @@ -217,20 +178,16 @@ def _seed_states( ) -> Dict[Tuple[str, str], SealCriterionState]: """The state each (feed, criterion) enters its first simulated day with. - Empty unless `resume_from_snapshot`, in which case each pair is seeded from its latest - snapshot strictly before that feed's march start — a complete state, which is what turns - a cold start into a resume (#1803). - - Pairs with no snapshot are simply absent from the result, and `transition` builds them - from nothing on their first day, exactly as a cold start would. A resume that reaches - further back than the snapshots go therefore degrades to a cold start for those criteria - rather than failing. + Empty unless `resume_from_snapshot`, which seeds each pair from its latest snapshot + before that feed's march start — a complete state, so a cold start becomes a resume + (#1803). Pairs with no snapshot are absent, and cold-start as usual: a resume reaching + further back than the snapshots go degrades rather than fails. """ if not resume_from_snapshot or not feeds: return {} - # One query for the batch. Each feed has its own cut-off, so the conditions are OR-ed - # rather than sharing a single date; DISTINCT ON keeps the latest row per pair. + # One query for the batch. Each feed has its own cut-off, hence the OR; DISTINCT ON + # keeps the latest row per pair. cutoffs = [ and_( SNAPSHOT_TABLE.c.feed_id == feed.id, @@ -262,21 +219,14 @@ def _upsert_seals_from_backfill( ) -> None: """Write feed_reliability_seal for the marched feeds. - Two things differ from the nightly job's `_upsert_seals`, and both are about - `created_at`: - - * It is written explicitly, as the feed's march start rather than the write time. That - column is what the Stable criterion counts its 180 days from, so left at its - `DEFAULT now()` every backfilled feed would fail Stable on every simulated day and the - backfill would grant no seals at all. - * It is **insert-only**, absent from the conflict clause. A re-backfill of a feed the - nightly job already owns must not reset a countdown that has been running for real. - - `seal_earned_at` is stamped with `now` — the run's end_date — for a feed the backfill - grants. The march does know the day the roll-up flipped, but under a cold start that day - is often the very first simulated one, which would claim a feed earned its seal a year - ago on the strength of a single simulated day. `end_date` says only that the seal record - begins here, which is exactly what a backfill establishes. + Differs from the nightly `_upsert_seals` only in `created_at`, which is written as the + feed's march start (left at `DEFAULT now()`, Stable would fail on every simulated day and + the backfill would grant nothing) and is **insert-only**, so a re-backfill cannot reset a + countdown already running. + + `seal_earned_at` gets `end_date`. The march knows the day the roll-up flipped, but under + a cold start that is often day one — which would claim a feed earned its seal a year ago + on one simulated day. """ for outcome in outcomes: row = { @@ -319,21 +269,16 @@ def _march( ) -> dict: """Replay the nightly evaluation day by day for one batch, and write the final day. - The evaluation itself is the nightly job's, unmodified: `transition` is called once per - feed, criterion and day with `now` set to that day. What this adds is that the returned - state is threaded into the next day in memory instead of being written, so a year's march - costs one write per feed rather than three hundred and sixty-five. - - Ascending day order is not a convenience, it is the algorithm — each day's state is the - input to the next. + The evaluation is the nightly job's, unmodified; what this adds is threading each day's + state into the next in memory, so a year costs one write per feed rather than 366. + Ascending order is the algorithm, not a convenience: each day feeds the next. """ if not feeds: return {"feeds": 0, "criterion_rows": 0, "snapshot_rows": 0, "outcomes": []} marched_days = days_between(min(start for start, _ in windows.values()), end_date) - # One load per criterion for the whole batch and the whole range. A criterion querying - # per day would turn this into several thousand queries; see `CriterionEvaluator.load_inputs`. + # One load per criterion for the whole range; per-day queries would be thousands. inputs = collect_inputs(db_session, feeds, marched_days, evaluators) states = _seed_states(db_session, feeds, windows, resume_from_snapshot) @@ -369,9 +314,8 @@ def _march( days_states.append(states[key]) if snapshot_mode == "all": - # The expensive mode, and the only one that writes inside the loop. Flushed per - # day rather than accumulated so a year's march does not hold every day's state - # in memory at once. + # The only mode that writes inside the loop, flushed per day so a year's march + # does not hold every day in memory. _upsert_criterion_snapshot(db_session, days_states, today) snapshot_rows += len(days_states) db_session.commit() @@ -404,9 +348,8 @@ def _final_outcomes( ) -> List[dict]: """Roll `has_seal` up from the final day's state, one entry per marched feed. - Skipped entirely on a partial criteria run, mirroring `update_seals`: criteria that were - not evaluated cannot be judged, so the roll-up would be answering a question it has only - part of the evidence for. + Skipped on a partial criteria run, as in `update_seals`: criteria that were not evaluated + cannot be judged. """ if partial_run: return [] @@ -428,8 +371,7 @@ def _final_outcomes( "tracking_start": windows[feed.id][0], "had_seal": had_seal, "has_seal": has_seal, - # A first evaluation is a grant if it passes, but not a loss if it fails: - # nothing was held, so nothing was lost. + # A first evaluation can grant but never revoke: nothing was held to lose. "granted": has_seal and not had_seal, "revoked": had_seal and not has_seal, } @@ -453,35 +395,14 @@ def backfill_seals( resume_from_snapshot: bool = False, max_reported_feeds: int = DEFAULT_MAX_REPORTED_FEEDS, ) -> dict: - """Plan and run the backfill for the requested feeds. - - Like `update_seals`, this always runs against an explicit list of feeds — enumerating the - catalogue is a producer's job, not this function's. - - Args: - db_session: SQLAlchemy session, injected by @with_db_session. - stable_feed_ids: The feeds to backfill. Required and non-empty. Unknown or ineligible - ids are skipped with a logged warning; it raises only if none can be used. - start_date: First day of the window. Clamped up to each feed's `created_at`. Defaults - to `end_date - days_back`. - end_date: Last day simulated, and the day the written state belongs to. Defaults to - yesterday UTC. Resolved once here so every feed of a run ends on the same day. - days_back: Window length used when `start_date` is absent. Default 365. - dry_run: Resolve and return the plan without marching or writing. Default True. - limit: Cap the number of feeds, applied to the requested list. - criteria: Backfill only these criteria. Same names as the nightly task. - batch_size: Feeds loaded and marched per batch. - only_missing: Skip feeds that already have seal state, which is #1763's stated scope. - Set False to re-backfill a feed and overwrite what is stored. - snapshot_mode: One of `SNAPSHOT_MODES` — how much of the march to record in - seal_criterion_snapshot. Default "final". - resume_from_snapshot: Seed each criterion from its snapshot at `march_start - 1` - rather than cold-starting empty. The #1803 hook; requires snapshots to exist. - max_reported_feeds: Cap on the `feeds` list in the report. - - Returns: - A report. `days` is the longest march in the run; feeds clamped to their own - `created_at` march fewer. + """Plan and run the backfill for an explicit list of feeds. + + Enumerating the catalogue is the producer's job, as with `update_seals`. Unknown or + ineligible ids are skipped with a warning; it raises only if none can be used. See + `backfill_seal_of_reliability` for the parameters as an operator passes them. + + Returns a report; `days` is the longest march in the run, since feeds clamped to their + own `created_at` march fewer. """ started = clock.monotonic() if not stable_feed_ids: @@ -496,9 +417,8 @@ def backfill_seals( window_start, window_end = resolve_window(start_date, end_date, days_back) evaluators = _resolve_evaluators(criteria) - # Plain by-id load, then eligibility in Python on the loaded rows — the same shape as - # `update_seals`, so a feed that does not exist can be told apart from one that exists - # but is not eligible without a second query. + # By-id load then eligibility in Python, as `update_seals` does: tells "not found" from + # "found but ineligible" without a second query. query = db_session.query(Gtfsfeed).filter( Gtfsfeed.stable_id.in_(list(stable_feed_ids)) ) @@ -533,8 +453,7 @@ def backfill_seals( "march_start": windows[feed.id][0].isoformat(), "end_date": window_end.isoformat(), "days": (window_end - windows[feed.id][0]).days + 1, - # The march start doubles as the Stable criterion's anchor, and is what - # feed_reliability_seal.created_at will be set to on insert. + # Also Stable's anchor, and what created_at gets on insert. "tracking_start": windows[feed.id][0].isoformat(), } for feed in selected @@ -587,9 +506,8 @@ def backfill_seals( outcomes.extend(result["outcomes"]) granted = [outcome for outcome in outcomes if outcome["granted"]] - # A backfill can only revoke when only_missing is False, since a feed with no - # stored seal held nothing to lose. Reported anyway so the run-level aggregate in - # `seal_orchestrator_monitor` reads the same keys from both fan-outs. + # Only reachable with only_missing=False, but reported anyway so the monitor's + # aggregate reads the same keys from both fan-outs. revoked = [outcome for outcome in outcomes if outcome["revoked"]] report["seals_granted"] = len(granted) report["seals_revoked"] = len(revoked) @@ -609,8 +527,7 @@ def backfill_seals( report["feeds"] = feed_plans[:max_reported_feeds] report["feeds_omitted"] = max(0, len(feed_plans) - max_reported_feeds) - # Logged without `feeds`: Cloud Logging drops a LogEntry over 256 KB, so a run naming a - # few hundred feeds would lose the whole entry. + # Without `feeds`: Cloud Logging drops a LogEntry over 256 KB. logger.info( "Backfill %s: %s", "plan" if dry_run else "complete", diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py index b90d0086c..5b5a6dc80 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py @@ -15,28 +15,16 @@ # """Cloud Tasks producer: fan the Seal of Reliability backfill out across the catalog (#1763). -`backfill_seal_of_reliability` only ever marches an explicit `stable_feed_ids` list. This -producer is what enumerates the catalog and chunks it, the same shape as the nightly -`seal_orchestrator`: +Enumerates the catalog and chunks it for `backfill_seal_of_reliability`, which only marches +an explicit list — the same shape as the nightly `seal_orchestrator`: resolve the eligible +feeds, batch them, register the run in TaskExecutionTracker, enqueue one worker per batch +plus one `seal_orchestrator_monitor` carrying this run's `task_name`. - 1. resolves every seal-eligible GTFS feed that has no seal state yet; - 2. splits the stable_ids into batches of `batch_size`; - 3. registers the run + one entry per batch in TaskExecutionTracker and enqueues one - `seal_backfill_worker` Cloud Task per batch; - 4. enqueues a single `seal_orchestrator_monitor` barrier task, carrying this run's - `task_name` so the shared monitor settles the right tracker. - -Three things differ from the nightly producer, and all three follow from a march being -long where a nightly evaluation is a single day: - -* **`end_date` is resolved here, once**, and passed to every worker. Left to each worker to - default, two workers of the same run started either side of midnight would march to - different final days and write states that do not correspond to the same moment. -* **Batches are smaller** and the **deadline is longer**. A batch marches a year for each of - its feeds, not one day. -* **`only_missing` is the eligibility predicate, not a filter the worker applies.** A feed - the nightly job already owns has real accumulated history, and must not have a simulation - written over it. +Three differences from the nightly producer, all because a march is long where a nightly +evaluation is one day: `end_date` is resolved here once and passed to every worker (or two +workers either side of midnight would end on different days); batches are smaller and the +deadline longer; and `only_missing` is the eligibility predicate rather than a worker-side +filter, so a feed the nightly job owns never has a simulation written over it. Payload (all optional):: @@ -83,12 +71,10 @@ logger = logging.getLogger(__name__) -# TaskExecutionTracker task_name for a backfill run. Distinct from the nightly run's, so -# the two never share a tracker and the monitor aggregates only its own batches. +# Distinct from the nightly run's, so the two never share a tracker. SEAL_BACKFILL_TASK_NAME = "seal_backfill_run" -# Smaller than the nightly default of 250: a batch marches every one of its feeds across the -# whole window, so the per-batch cost scales with days as well as feeds. +# Smaller than the nightly 250: per-batch cost scales with days as well as feeds. DEFAULT_BATCH_SIZE = 100 DEFAULT_DEADLINE_SECONDS = 2 * 60 * 60 # 2h wall-clock cap for a run DEFAULT_MONITOR_DELAY_SECONDS = 300 @@ -103,8 +89,7 @@ def seal_backfill_orchestrator_handler(payload: dict) -> dict: f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" ) - # Validated and resolved here rather than in the workers, so a bad date fails the run at - # the producer instead of once per batch. + # Resolved here, so a bad date fails once at the producer rather than once per batch. window_start, window_end = resolve_window( _parse_day(payload.get("start_date"), "start_date"), _parse_day(payload.get("end_date"), "end_date"), @@ -218,7 +203,6 @@ def _plan_run( "batch_id": batch_id, "stable_feed_ids": batch_stable_ids, "criteria": criteria, - # Both ends are explicit, so a worker never re-derives the window. "start_date": window_start.isoformat(), "end_date": window_end.isoformat(), "only_missing": only_missing, @@ -233,14 +217,12 @@ def _plan_run( ): enqueued += 1 else: - # Dead on arrival: don't leave this batch as `triggered` for the monitor to - # only notice once the deadline passes. + # Dead on arrival: don't leave it `triggered` until the deadline. _mark_enqueue_failed(run_id, batch_id) if consumed < len(batch_ids): - # The eligible-feed stream yielded fewer chunks than the plan-time count implied — - # eligibility narrowed in the gap between the two queries. Fail the leftovers - # immediately rather than leaving them `triggered` until the deadline. + # Eligibility narrowed between the count and the stream. Fail the leftovers now + # rather than leaving them `triggered` until the deadline. missing = batch_ids[consumed:] logger.error( "seal_backfill_orchestrator: run=%s stream yielded %d batch(es), expected %d " @@ -260,8 +242,7 @@ def _plan_run( else: extra_chunk = next(stable_id_batches, None) if extra_chunk is not None: - # More chunks than planned: feeds became newly eligible in the gap. Log-only — - # a backfill is operator-triggered, so the fix is to run it again. + # Feeds became newly eligible in the gap. Log-only: re-run to pick them up. logger.error( "seal_backfill_orchestrator: run=%s stream had more batches than the " "plan-time count of %d expected (>=%d additional feed(s)) — those feeds " diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py index 2d395e56a..e19f629c3 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_worker.py @@ -15,18 +15,12 @@ # """Cloud Tasks worker: march one batch of the Seal of Reliability backfill (#1763). -One `seal_backfill_worker` task is enqueued per batch by `seal_backfill_orchestrator`. It -calls `backfill_seals` for its slice of stable_ids and reports the outcome to the shared -`TaskExecutionTracker` so the monitor knows when the run has drained. +One task per batch, enqueued by `seal_backfill_orchestrator`. Calls `backfill_seals` for its +slice and reports to the shared `TaskExecutionTracker`. -`backfill_seals` writes via upsert, so a Cloud Tasks redelivery of the same batch — a -timeout on the response after the write already committed, say — is safe to reprocess: the -same window over the same source data produces the same final state. `created_at` on -`feed_reliability_seal` is insert-only, so even a redelivery cannot move a feed's tracking -start. - -`start_date` and `end_date` both arrive explicit. A worker never defaults them: the run's -window belongs to the run, not to the moment a particular batch happened to execute. +Redelivery is safe: `backfill_seals` upserts, the same window over the same sources gives the +same final state, and `created_at` is insert-only. Both dates arrive explicit — the window +belongs to the run, not to when a batch happened to execute. Payload:: @@ -75,8 +69,7 @@ def seal_backfill_worker_handler(payload: dict) -> dict: start_date = _parse_day(payload.get("start_date"), "start_date") end_date = _parse_day(payload.get("end_date"), "end_date") if start_date is None or end_date is None: - # The producer resolves the window for the whole run. A worker that defaulted it - # could march to a different final day than its siblings. + # Defaulting here could march to a different final day than a sibling batch. raise ValueError("start_date and end_date are required") try: @@ -111,11 +104,10 @@ def _mark_entry( error: Optional[str] = None, db_session=None, ) -> None: - """Record this batch's completion in the run's TaskExecutionTracker. + """Record this batch in the run's tracker. - The stored metadata is what `seal_orchestrator_monitor` aggregates into the run-level - report, so the keys it reads — total_feeds, criterion_rows_written, seals_granted / - seals_revoked and their stable_id lists — must survive here. + The stored metadata is what `seal_orchestrator_monitor` aggregates, so its keys must + survive here. """ tracker = TaskExecutionTracker( task_name=SEAL_BACKFILL_TASK_NAME, From 719b4ca0b5c0626a0de0ca26e228c111cb963b83 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 26 Aug 2026 16:15:31 -0400 Subject: [PATCH 09/13] Share the fan-out mechanism between both seal producers --- .../backfill/seal_backfill_orchestrator.py | 213 +++--------- .../src/tasks/seal_of_reliability/fanout.py | 298 ++++++++++++++++ .../orchestrator/seal_orchestrator.py | 319 +++--------------- .../orchestrator/seal_orchestrator_monitor.py | 30 +- .../backfill/test_seal_backfill_fanout.py | 31 +- .../test_seal_monitor_aggregation.py | 262 ++++++++++++++ .../test_seal_orchestrator_fanout.py | 63 ++-- 7 files changed, 729 insertions(+), 487 deletions(-) create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py create mode 100644 functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py index 5b5a6dc80..1806e80a7 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill_orchestrator.py @@ -16,11 +16,9 @@ """Cloud Tasks producer: fan the Seal of Reliability backfill out across the catalog (#1763). Enumerates the catalog and chunks it for `backfill_seal_of_reliability`, which only marches -an explicit list — the same shape as the nightly `seal_orchestrator`: resolve the eligible -feeds, batch them, register the run in TaskExecutionTracker, enqueue one worker per batch -plus one `seal_orchestrator_monitor` carrying this run's `task_name`. +an explicit list. The mechanism is `fanout.plan_fanout`, shared with the nightly producer. -Three differences from the nightly producer, all because a march is long where a nightly +Three differences from the nightly run, all because a march is long where a nightly evaluation is one day: `end_date` is resolved here once and passed to every worker (or two workers either side of midnight would end on different days); batches are smaller and the deadline longer; and `only_missing` is the eligibility predicate rather than a worker-side @@ -46,12 +44,9 @@ """ import logging -import math -from datetime import datetime, timezone from typing import Any, Dict, List, Optional from shared.database.database import with_db_session -from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker from tasks.seal_of_reliability.backfill.backfill_seal_of_reliability import _parse_day from tasks.seal_of_reliability.backfill.seal_backfill import ( @@ -64,10 +59,7 @@ count_eligible_feeds, iter_eligible_stable_ids, ) -from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( - _enqueue, - _safe_task_name, -) +from tasks.seal_of_reliability.fanout import FanoutSpec, plan_fanout logger = logging.getLogger(__name__) @@ -79,6 +71,16 @@ DEFAULT_DEADLINE_SECONDS = 2 * 60 * 60 # 2h wall-clock cap for a run DEFAULT_MONITOR_DELAY_SECONDS = 300 +# The monitor is shared with the nightly run, so it has to be told which tracker to settle. +SPEC = FanoutSpec( + task_name=SEAL_BACKFILL_TASK_NAME, + worker_task="seal_backfill_worker", + run_id_prefix="seal-backfill", + task_prefix="seal-backfill", + log_name="seal_backfill_orchestrator", + monitor_extra={"task_name": SEAL_BACKFILL_TASK_NAME}, +) + def seal_backfill_orchestrator_handler(payload: dict) -> dict: """Entry point for the `seal_backfill_orchestrator` task.""" @@ -131,169 +133,52 @@ def _plan_run( db_session=None, ) -> Dict[str, Any]: """Resolve the feeds to backfill, chunk them, and (unless dry_run) fan the run out.""" - if batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - run_started_at = datetime.now(timezone.utc) - total_feeds = count_eligible_feeds( - db_session, - stable_feed_ids=stable_feed_ids, - limit=limit, - exclude_backfilled=only_missing, - ) - num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 - run_id = f"seal-backfill-{run_started_at.strftime('%Y%m%dT%H%M%S')}" - - logger.info( - "seal_backfill_orchestrator: run=%s total_feeds=%d batch_size=%d batches=%d " - "window=%s..%s dry_run=%s", - run_id, - total_feeds, - batch_size, - num_batches, - window_start.isoformat(), - window_end.isoformat(), - dry_run, - ) - - plan = { - "run_id": run_id, - "total_feeds": total_feeds, - "batch_size": batch_size, - "batches": num_batches, + window = { "start_date": window_start.isoformat(), "end_date": window_end.isoformat(), - "only_missing": only_missing, - "snapshot_mode": snapshot_mode, - "resume_from_snapshot": resume_from_snapshot, - "enqueued": 0, - "dry_run": dry_run, } - if dry_run or not num_batches: - return plan - - run_params = { - "dry_run": False, - "batch_size": batch_size, - "criteria": criteria, - "start_date": window_start.isoformat(), - "end_date": window_end.isoformat(), + settings = { "only_missing": only_missing, "snapshot_mode": snapshot_mode, "resume_from_snapshot": resume_from_snapshot, - "run_started_at": run_started_at.isoformat(), - "deadline_seconds": deadline_seconds, } - batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] - _start_run(run_id, batch_ids, run_params) - enqueued = 0 - consumed = 0 - stable_id_batches = iter_eligible_stable_ids( + plan = plan_fanout( db_session, - batch_size, - stable_feed_ids=stable_feed_ids, - limit=limit, - exclude_backfilled=only_missing, - ) - for batch_id, batch_stable_ids in zip(batch_ids, stable_id_batches): - consumed += 1 - worker_payload = { + SPEC, + count_feeds=lambda session: count_eligible_feeds( + session, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ), + iter_batches=lambda session, size: iter_eligible_stable_ids( + session, + size, + stable_feed_ids=stable_feed_ids, + limit=limit, + exclude_backfilled=only_missing, + ), + build_worker_payload=lambda run_id, batch_id, ids: { "run_id": run_id, "batch_id": batch_id, - "stable_feed_ids": batch_stable_ids, + "stable_feed_ids": ids, "criteria": criteria, - "start_date": window_start.isoformat(), - "end_date": window_end.isoformat(), - "only_missing": only_missing, - "snapshot_mode": snapshot_mode, - "resume_from_snapshot": resume_from_snapshot, - } - if _enqueue( - in_body_task="seal_backfill_worker", - payload=worker_payload, - queue_env="SEAL_ORCHESTRATOR_QUEUE", - task_name=_safe_task_name(f"seal-backfill-{run_id}-{batch_id}"), - ): - enqueued += 1 - else: - # Dead on arrival: don't leave it `triggered` until the deadline. - _mark_enqueue_failed(run_id, batch_id) - - if consumed < len(batch_ids): - # Eligibility narrowed between the count and the stream. Fail the leftovers now - # rather than leaving them `triggered` until the deadline. - missing = batch_ids[consumed:] - logger.error( - "seal_backfill_orchestrator: run=%s stream yielded %d batch(es), expected %d " - "— marking %d failed: %s", - run_id, - consumed, - len(batch_ids), - len(missing), - missing, - ) - for batch_id in missing: - _mark_enqueue_failed( - run_id, - batch_id, - error_message="no eligible-feed data for this batch (count/stream mismatch)", - ) - else: - extra_chunk = next(stable_id_batches, None) - if extra_chunk is not None: - # Feeds became newly eligible in the gap. Log-only: re-run to pick them up. - logger.error( - "seal_backfill_orchestrator: run=%s stream had more batches than the " - "plan-time count of %d expected (>=%d additional feed(s)) — those feeds " - "were not backfilled; re-run to pick them up", - run_id, - len(batch_ids), - len(extra_chunk), - ) - - _enqueue( - in_body_task="seal_orchestrator_monitor", - payload={"run_id": run_id, "task_name": SEAL_BACKFILL_TASK_NAME}, - queue_env="SEAL_ORCHESTRATOR_MONITOR_QUEUE", - task_name=_safe_task_name(f"seal-backfill-monitor-{run_id}"), - schedule_seconds=monitor_delay_seconds, - ) - - plan["enqueued"] = enqueued - return plan - - -@with_db_session -def _start_run( - run_id: str, - batch_ids: List[str], - run_params: dict, - db_session=None, -) -> None: - """Register the run and one tracked entry per batch.""" - tracker = TaskExecutionTracker( - task_name=SEAL_BACKFILL_TASK_NAME, - run_id=run_id, - db_session=db_session, - ) - tracker.start_run(total_count=len(batch_ids), params=run_params) - for batch_id in batch_ids: - tracker.mark_triggered(batch_id) - db_session.commit() - - -@with_db_session -def _mark_enqueue_failed( - run_id: str, - batch_id: str, - error_message: str = "enqueue failed", - db_session=None, -) -> None: - tracker = TaskExecutionTracker( - task_name=SEAL_BACKFILL_TASK_NAME, - run_id=run_id, - db_session=db_session, + **window, + **settings, + }, + run_params=lambda run_started_at: { + "dry_run": False, + "batch_size": batch_size, + "criteria": criteria, + **window, + **settings, + "run_started_at": run_started_at, + "deadline_seconds": deadline_seconds, + }, + batch_size=batch_size, + dry_run=dry_run, + monitor_delay_seconds=monitor_delay_seconds, ) - tracker.mark_failed(batch_id, error_message=error_message) - db_session.commit() + # Echoed back so an operator can see what a dry run resolved to. + return {**plan, **window, **settings} diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py new file mode 100644 index 000000000..d0380a597 --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/fanout.py @@ -0,0 +1,298 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""The Cloud Tasks fan-out both seal producers share. + +The nightly run (#1800) and the backfill (#1763) differ in what they send a worker and which +feeds they select, but the mechanism between those two points is identical: count, chunk, +register the run, enqueue a worker per batch, reconcile the count against what the stream +actually yielded, enqueue one monitor. That reconciliation is the subtle part — it is what +stops a batch sitting `triggered` until the deadline — and having it in one place is the +reason this module exists. + +A producer supplies a `FanoutSpec` (the names and queues it uses) plus three callables: how +to count its feeds, how to stream them, and how to build a worker payload. Everything else is +here. +""" + +import json +import logging +import math +import os +import re +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Sequence + +from shared.database.database import with_db_session +from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class FanoutSpec: + """What distinguishes one producer's fan-out from the other's. + + `monitor_extra` is merged into the monitor's payload. The nightly run needs nothing + there; the backfill passes its tracker `task_name`, since the two share one monitor. + """ + + task_name: str # TaskExecutionTracker task_name for the run + worker_task: str # in-body task name of the worker to enqueue + run_id_prefix: str # run ids are "-" + task_prefix: str # Cloud Tasks names are "--" + log_name: str # what this producer calls itself in logs + queue_env: str = "SEAL_ORCHESTRATOR_QUEUE" + monitor_queue_env: str = "SEAL_ORCHESTRATOR_MONITOR_QUEUE" + monitor_task: str = "seal_orchestrator_monitor" + monitor_extra: Mapping[str, Any] = field(default_factory=dict) + + +def safe_task_name(name: str) -> str: + return re.sub(r"[^a-zA-Z0-9_-]", "-", name)[:500] + + +def enqueue_task( + *, + in_body_task: str, + payload: dict, + queue_env: str, + task_name: str, + schedule_seconds: int = 0, +) -> bool: + """Enqueue a Cloud Task targeting the tasks_executor function. + + Returns True on enqueue (or already-exists), False when misconfigured. + """ + project = os.getenv("PROJECT_ID") + queue = os.getenv(queue_env) + gcp_region = os.getenv("GCP_REGION") + environment = os.getenv("ENVIRONMENT") + if not all([project, queue, gcp_region, environment]): + logger.warning( + "enqueue_task: missing env (PROJECT_ID/GCP_REGION/ENVIRONMENT/%s) — " + "skipping enqueue of %s", + queue_env, + task_name, + ) + return False + + try: + from google.cloud import tasks_v2 + from google.protobuf import timestamp_pb2 + from shared.common.gcp_utils import create_http_task_with_name + + url = ( + f"https://{gcp_region}-{project}.cloudfunctions.net/" + f"tasks_executor-{environment}" + ) + body = json.dumps({"task": in_body_task, "payload": payload}).encode() + + schedule_time: Optional[Any] = None + if schedule_seconds > 0: + run_at = datetime.now(timezone.utc) + timedelta(seconds=schedule_seconds) + schedule_time = timestamp_pb2.Timestamp() + schedule_time.FromDatetime(run_at.replace(tzinfo=None)) + + create_http_task_with_name( + client=tasks_v2.CloudTasksClient(), + body=body, + url=url, + project_id=project, + gcp_region=gcp_region, + queue_name=queue, + task_name=task_name, + task_time=schedule_time, + http_method=tasks_v2.HttpMethod.POST, + ) + return True + except Exception as e: # pragma: no cover - network/env dependent + if "already exists" in str(e).lower() or "ALREADY_EXISTS" in str(e): + logger.info("enqueue_task: task %s already exists — skipping", task_name) + return True + logger.warning("enqueue_task: could not enqueue %s: %s", task_name, e) + return False + + +@with_db_session +def start_run( + task_name: str, + run_id: str, + batch_ids: List[str], + run_params: dict, + db_session=None, +) -> None: + """Register the run and one tracked entry per batch.""" + tracker = TaskExecutionTracker( + task_name=task_name, run_id=run_id, db_session=db_session + ) + tracker.start_run(total_count=len(batch_ids), params=run_params) + for batch_id in batch_ids: + tracker.mark_triggered(batch_id) + db_session.commit() + + +@with_db_session +def mark_enqueue_failed( + task_name: str, + run_id: str, + batch_id: str, + error_message: str = "enqueue failed", + db_session=None, +) -> None: + tracker = TaskExecutionTracker( + task_name=task_name, run_id=run_id, db_session=db_session + ) + tracker.mark_failed(batch_id, error_message=error_message) + db_session.commit() + + +def new_run_id(spec: FanoutSpec, started_at: datetime) -> str: + return f"{spec.run_id_prefix}-{started_at.strftime('%Y%m%dT%H%M%S')}" + + +def _reconcile( + spec: FanoutSpec, + run_id: str, + batch_ids: List[str], + consumed: int, + stream: Iterator[List[str]], +) -> None: + """Settle the difference between the plan-time count and what the stream yielded. + + Both directions come from the same cause: the count and the stream are separate queries, + so eligibility can move in the gap between them. + """ + if consumed < len(batch_ids): + # Fewer chunks than planned. The leftovers are already `triggered` from start_run + # and would otherwise sit there until the deadline failed the whole run. + missing = batch_ids[consumed:] + logger.error( + "%s: run=%s stream yielded %d batch(es), expected %d — marking %d failed: %s", + spec.log_name, + run_id, + consumed, + len(batch_ids), + len(missing), + missing, + ) + for batch_id in missing: + mark_enqueue_failed( + spec.task_name, + run_id, + batch_id, + error_message="no eligible-feed data for this batch (count/stream mismatch)", + ) + return + + # zip() with batch_ids (a list) first never calls next() on the stream for a final + # round once batch_ids is exhausted, so this reflects what is genuinely left over. + extra_chunk = next(stream, None) + if extra_chunk is not None: + # More chunks than planned: feeds became newly eligible in the gap. Log-only — + # self-healing would mean mutating total_count after start_run fixed it, for a + # race whose only consequence is a feed waiting for the next run. + logger.error( + "%s: run=%s stream had more batches than the plan-time count of %d expected " + "(>=%d additional feed(s) seen) — those feeds were not processed this run", + spec.log_name, + run_id, + len(batch_ids), + len(extra_chunk), + ) + + +def plan_fanout( + db_session, + spec: FanoutSpec, + *, + count_feeds: Callable[[Any], int], + iter_batches: Callable[[Any, int], Iterator[List[str]]], + build_worker_payload: Callable[[str, str, Sequence[str]], dict], + run_params: Callable[[str], dict], + batch_size: int, + dry_run: bool, + monitor_delay_seconds: int, +) -> Dict[str, Any]: + """Count, chunk, register, enqueue and reconcile. Returns the plan. + + `run_params` is a callable rather than a dict so a producer can fold in `run_started_at`, + which is decided here. + + On a dry run — or when nothing is eligible — nothing is registered and nothing is + enqueued, so the returned plan is purely informational. + """ + if batch_size <= 0: + raise ValueError("batch_size must be a positive integer") + + run_started_at = datetime.now(timezone.utc) + total_feeds = count_feeds(db_session) + num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 + run_id = new_run_id(spec, run_started_at) + + logger.info( + "%s: run=%s total_feeds=%d batch_size=%d batches=%d dry_run=%s", + spec.log_name, + run_id, + total_feeds, + batch_size, + num_batches, + dry_run, + ) + + plan = { + "run_id": run_id, + "total_feeds": total_feeds, + "batch_size": batch_size, + "batches": num_batches, + "enqueued": 0, + "dry_run": dry_run, + } + if dry_run or not num_batches: + return plan + + batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] + start_run(spec.task_name, run_id, batch_ids, run_params(run_started_at.isoformat())) + + enqueued = 0 + consumed = 0 + stream = iter_batches(db_session, batch_size) + for batch_id, batch_stable_ids in zip(batch_ids, stream): + consumed += 1 + if enqueue_task( + in_body_task=spec.worker_task, + payload=build_worker_payload(run_id, batch_id, batch_stable_ids), + queue_env=spec.queue_env, + task_name=safe_task_name(f"{spec.task_prefix}-{run_id}-{batch_id}"), + ): + enqueued += 1 + else: + # Dead on arrival: don't leave it `triggered` until the deadline. + mark_enqueue_failed(spec.task_name, run_id, batch_id) + + _reconcile(spec, run_id, batch_ids, consumed, stream) + + # Single barrier task, delayed so it does not fire before any worker has run. + enqueue_task( + in_body_task=spec.monitor_task, + payload={"run_id": run_id, **spec.monitor_extra}, + queue_env=spec.monitor_queue_env, + task_name=safe_task_name(f"{spec.task_prefix}-monitor-{run_id}"), + schedule_seconds=monitor_delay_seconds, + ) + + plan["enqueued"] = enqueued + return plan diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py index a73aca639..085b3d47b 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator.py @@ -17,23 +17,14 @@ """Cloud Tasks producer: fan the nightly Seal of Reliability evaluation out to per-batch workers (issue #1800). -`update_seal_of_reliability` only ever evaluates an explicit `stable_feed_ids` list — -there is no run-the-whole-catalogue mode, and pure-SQL evaluation for the whole catalog -in one invocation would eventually hit `tasks_executor`'s own timeout as the catalog -grows. This producer is what enumerates the catalog and chunks it: +`update_seal_of_reliability` only evaluates an explicit `stable_feed_ids` list, and one +invocation over the whole catalog would eventually hit `tasks_executor`'s timeout. This +enumerates the catalog and chunks it; the mechanism itself lives in `fanout.plan_fanout`, +shared with the backfill producer (#1763). - 1. resolves every seal-eligible GTFS feed (same eligibility predicate `update_seals` - itself applies via `context.is_seal_eligible` — see `context.iter_eligible_stable_ids`); - 2. splits the stable_ids into batches of `batch_size`; - 3. registers the run + one entry per batch in TaskExecutionTracker and enqueues one - `seal_orchestrator_worker` Cloud Task per batch; - 4. enqueues a single `seal_orchestrator_monitor` barrier task. - -Batches, not feeds, are the tracked unit: seal evaluation is pure DB work with no -per-feed side effect requiring isolation (unlike notification dispatch, where each -subscription needs an independent send + claim), so one Cloud Task per ~250 feeds -keeps the daily invocation count low while still removing the single-invocation -timeout ceiling entirely. +Batches, not feeds, are the tracked unit: seal evaluation is pure DB work with no per-feed +side effect requiring isolation, so one Cloud Task per ~250 feeds keeps the daily invocation +count low while removing the single-invocation timeout ceiling. Payload (all optional):: @@ -49,56 +40,50 @@ } """ -import json import logging -import math -import os -import re -from datetime import datetime, timezone from typing import Any, Dict, List, Optional from shared.database.database import with_db_session -from shared.helpers.task_execution.task_execution_tracker import TaskExecutionTracker from tasks.seal_of_reliability.context import ( count_eligible_feeds, iter_eligible_stable_ids, ) +from tasks.seal_of_reliability.fanout import FanoutSpec, plan_fanout logger = logging.getLogger(__name__) -# TaskExecutionTracker task_name for a seal orchestrator run (fan-out workers + -# monitor all key off this plus a per-run run_id). +# TaskExecutionTracker task_name for a seal orchestrator run (fan-out workers + monitor +# all key off this plus a per-run run_id). SEAL_ORCHESTRATOR_TASK_NAME = "seal_orchestrator_run" DEFAULT_BATCH_SIZE = 250 DEFAULT_DEADLINE_SECONDS = 60 * 60 # 1h wall-clock cap for a run DEFAULT_MONITOR_DELAY_SECONDS = 60 +SPEC = FanoutSpec( + task_name=SEAL_ORCHESTRATOR_TASK_NAME, + worker_task="seal_orchestrator_worker", + run_id_prefix="seal", + task_prefix="seal-orchestrator", + log_name="seal_orchestrator", +) + def seal_orchestrator_handler(payload: dict) -> dict: """Entry point for the `seal_orchestrator` task.""" payload = payload or {} - dry_run = bool(payload.get("dry_run", True)) - batch_size = int(payload.get("batch_size", DEFAULT_BATCH_SIZE)) - criteria = payload.get("criteria") - now = payload.get("now") - limit = payload.get("limit") - stable_feed_ids = payload.get("stable_feed_ids") - deadline_seconds = int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)) - monitor_delay_seconds = int( - payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) - ) - return _plan_run( - dry_run=dry_run, - batch_size=batch_size, - criteria=criteria, - now=now, - limit=limit, - stable_feed_ids=stable_feed_ids, - deadline_seconds=deadline_seconds, - monitor_delay_seconds=monitor_delay_seconds, + dry_run=bool(payload.get("dry_run", True)), + batch_size=int(payload.get("batch_size", DEFAULT_BATCH_SIZE)), + criteria=payload.get("criteria"), + now=payload.get("now"), + limit=payload.get("limit"), + stable_feed_ids=payload.get("stable_feed_ids"), + deadline_seconds=int(payload.get("deadline_seconds", DEFAULT_DEADLINE_SECONDS)), + monitor_delay_seconds=int( + payload.get("monitor_delay_seconds", DEFAULT_MONITOR_DELAY_SECONDS) + ), ) @@ -115,233 +100,31 @@ def _plan_run( db_session=None, ) -> Dict[str, Any]: """Resolve eligible feeds, chunk them, and (unless dry_run) fan the run out.""" - if batch_size <= 0: - raise ValueError("batch_size must be a positive integer") - - run_started_at = datetime.now(timezone.utc) - total_feeds = count_eligible_feeds( - db_session, stable_feed_ids=stable_feed_ids, limit=limit - ) - num_batches = math.ceil(total_feeds / batch_size) if total_feeds else 0 - run_id = f"seal-{run_started_at.strftime('%Y%m%dT%H%M%S')}" - - logger.info( - "seal_orchestrator: run=%s total_feeds=%d batch_size=%d batches=%d dry_run=%s", - run_id, - total_feeds, - batch_size, - num_batches, - dry_run, - ) - - if dry_run or not num_batches: - return { - "run_id": run_id, - "total_feeds": total_feeds, - "batch_size": batch_size, - "batches": num_batches, - "enqueued": 0, - "dry_run": dry_run, - } - - run_params = { - "dry_run": False, - "batch_size": batch_size, - "criteria": criteria, - "now": now, - "run_started_at": run_started_at.isoformat(), - "deadline_seconds": deadline_seconds, - } - batch_ids = [f"batch-{index:04d}" for index in range(num_batches)] - _start_run(run_id, batch_ids, run_params) - - enqueued = 0 - consumed = 0 - stable_id_batches = iter_eligible_stable_ids( - db_session, batch_size, stable_feed_ids=stable_feed_ids, limit=limit - ) - for batch_id, batch_stable_ids in zip(batch_ids, stable_id_batches): - consumed += 1 - worker_payload = { + return plan_fanout( + db_session, + SPEC, + count_feeds=lambda session: count_eligible_feeds( + session, stable_feed_ids=stable_feed_ids, limit=limit + ), + iter_batches=lambda session, size: iter_eligible_stable_ids( + session, size, stable_feed_ids=stable_feed_ids, limit=limit + ), + build_worker_payload=lambda run_id, batch_id, ids: { "run_id": run_id, "batch_id": batch_id, - "stable_feed_ids": batch_stable_ids, + "stable_feed_ids": ids, "criteria": criteria, "now": now, - } - if _enqueue( - in_body_task="seal_orchestrator_worker", - payload=worker_payload, - queue_env="SEAL_ORCHESTRATOR_QUEUE", - task_name=_safe_task_name(f"seal-orchestrator-{run_id}-{batch_id}"), - ): - enqueued += 1 - else: - # Dead on arrival: don't leave this batch as `triggered` for the monitor - # to only notice once the deadline passes. - _mark_enqueue_failed(run_id, batch_id) - - if consumed < len(batch_ids): - # The eligible-feed stream (a separately executed query) yielded fewer chunks - # than count_eligible_feeds implied at plan time — eligibility narrowed in the - # gap between the two queries. The leftover batch_ids are already `triggered` - # (via _start_run) but would otherwise never be enqueued or reported, sitting - # stuck until deadline_seconds forces the whole run to `failed`. Fail them - # immediately and visibly instead. - missing = batch_ids[consumed:] - logger.error( - "seal_orchestrator: run=%s eligible-feed stream yielded %d batch(es), " - "expected %d from the plan-time count — marking %d batch(es) failed: %s", - run_id, - consumed, - len(batch_ids), - len(missing), - missing, - ) - for batch_id in missing: - _mark_enqueue_failed( - run_id, - batch_id, - error_message="no eligible-feed data for this batch (count/stream mismatch)", - ) - else: - # zip() with batch_ids (a plain list) first never calls next() on - # stable_id_batches for a final round once batch_ids is exhausted, so this - # reliably reflects whatever the stream still has left, with no off-by-one. - extra_chunk = next(stable_id_batches, None) - if extra_chunk is not None: - # Opposite direction: more chunks than the plan-time count implied (feeds - # became newly eligible in the gap). Log-only: making this self-healing - # would mean mutating TaskExecutionTracker's total_count after _start_run - # already fixed it, for a narrow race window whose only consequence is one - # feed waiting until the next nightly run. - logger.error( - "seal_orchestrator: run=%s eligible-feed stream had more batches than " - "the plan-time count of %d expected (>=%d additional feed(s) seen) — " - "some newly-eligible feeds were not evaluated this run; the next " - "scheduled run will pick them up", - run_id, - len(batch_ids), - len(extra_chunk), - ) - - # Single barrier/summary task; polls until the run drains, then reports. - # Delayed slightly so it doesn't fire before any worker has had a chance to run. - _enqueue( - in_body_task="seal_orchestrator_monitor", - payload={"run_id": run_id}, - queue_env="SEAL_ORCHESTRATOR_MONITOR_QUEUE", - task_name=_safe_task_name(f"seal-orchestrator-monitor-{run_id}"), - schedule_seconds=monitor_delay_seconds, - ) - - return { - "run_id": run_id, - "total_feeds": total_feeds, - "batch_size": batch_size, - "batches": num_batches, - "enqueued": enqueued, - "dry_run": False, - } - - -@with_db_session -def _start_run( - run_id: str, - batch_ids: List[str], - run_params: dict, - db_session=None, -) -> None: - """Register the run and one tracked entry per batch.""" - tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, - run_id=run_id, - db_session=db_session, - ) - tracker.start_run(total_count=len(batch_ids), params=run_params) - for batch_id in batch_ids: - tracker.mark_triggered(batch_id) - db_session.commit() - - -@with_db_session -def _mark_enqueue_failed( - run_id: str, - batch_id: str, - error_message: str = "enqueue failed", - db_session=None, -) -> None: - tracker = TaskExecutionTracker( - task_name=SEAL_ORCHESTRATOR_TASK_NAME, - run_id=run_id, - db_session=db_session, + }, + run_params=lambda run_started_at: { + "dry_run": False, + "batch_size": batch_size, + "criteria": criteria, + "now": now, + "run_started_at": run_started_at, + "deadline_seconds": deadline_seconds, + }, + batch_size=batch_size, + dry_run=dry_run, + monitor_delay_seconds=monitor_delay_seconds, ) - tracker.mark_failed(batch_id, error_message=error_message) - db_session.commit() - - -def _safe_task_name(name: str) -> str: - return re.sub(r"[^a-zA-Z0-9_-]", "-", name)[:500] - - -def _enqueue( - *, - in_body_task: str, - payload: dict, - queue_env: str, - task_name: str, - schedule_seconds: int = 0, -) -> bool: - """Enqueue a Cloud Task targeting the tasks_executor function. - - Returns True on enqueue (or already-exists), False when misconfigured. - """ - project = os.getenv("PROJECT_ID") - queue = os.getenv(queue_env) - gcp_region = os.getenv("GCP_REGION") - environment = os.getenv("ENVIRONMENT") - if not all([project, queue, gcp_region, environment]): - logger.warning( - "_enqueue: missing env (PROJECT_ID/GCP_REGION/ENVIRONMENT/%s) — " - "skipping enqueue of %s", - queue_env, - task_name, - ) - return False - - try: - from google.cloud import tasks_v2 - from google.protobuf import timestamp_pb2 - from datetime import timedelta - from shared.common.gcp_utils import create_http_task_with_name - - url = ( - f"https://{gcp_region}-{project}.cloudfunctions.net/" - f"tasks_executor-{environment}" - ) - body = json.dumps({"task": in_body_task, "payload": payload}).encode() - - schedule_time: Optional[Any] = None - if schedule_seconds > 0: - run_at = datetime.now(timezone.utc) + timedelta(seconds=schedule_seconds) - schedule_time = timestamp_pb2.Timestamp() - schedule_time.FromDatetime(run_at.replace(tzinfo=None)) - - create_http_task_with_name( - client=tasks_v2.CloudTasksClient(), - body=body, - url=url, - project_id=project, - gcp_region=gcp_region, - queue_name=queue, - task_name=task_name, - task_time=schedule_time, - http_method=tasks_v2.HttpMethod.POST, - ) - return True - except Exception as e: # pragma: no cover - network/env dependent - if "already exists" in str(e).lower() or "ALREADY_EXISTS" in str(e): - logger.info("_enqueue: task %s already exists — skipping", task_name) - return True - logger.warning("_enqueue: could not enqueue %s: %s", task_name, e) - return False diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py index b3eae033a..12bebf504 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/orchestrator/seal_orchestrator_monitor.py @@ -67,6 +67,16 @@ _SETTLED_STATUSES = (STATUS_COMPLETED, STATUS_FAILED) +# Numeric keys summed across a run's batches. A batch that does not report one contributes +# zero, which is what lets the nightly and backfill fan-outs share this aggregation. +_SUMMED_KEYS = ( + "total_feeds", + "criterion_rows_written", + "snapshot_rows_written", + "seals_granted", + "seals_revoked", +) + def seal_orchestrator_monitor_handler(payload: dict) -> dict: """Entry point for the `seal_orchestrator_monitor` task.""" @@ -173,20 +183,17 @@ def _aggregate_batches(db_session, run_id: str, task_name: str) -> Dict[str, Any .all() ) - total_feeds = 0 - criterion_rows_written = 0 - seals_granted = 0 - seals_revoked = 0 + # snapshot_rows_written is only ever reported by a backfill batch; a nightly batch + # simply has no such key and contributes zero. + totals = dict.fromkeys(_SUMMED_KEYS, 0) granted_stable_ids: list = [] revoked_stable_ids: list = [] for (metadata,) in rows: if not metadata: continue - total_feeds += metadata.get("total_feeds", 0) or 0 - criterion_rows_written += metadata.get("criterion_rows_written", 0) or 0 - seals_granted += metadata.get("seals_granted", 0) or 0 - seals_revoked += metadata.get("seals_revoked", 0) or 0 + for key in _SUMMED_KEYS: + totals[key] += metadata.get(key, 0) or 0 granted_stable_ids.extend(metadata.get("granted_stable_ids") or []) revoked_stable_ids.extend(metadata.get("revoked_stable_ids") or []) @@ -195,10 +202,9 @@ def _aggregate_batches(db_session, run_id: str, task_name: str) -> Dict[str, Any ) return { - "total_feeds_evaluated": total_feeds, - "criterion_rows_written": criterion_rows_written, - "seals_granted": seals_granted, - "seals_revoked": seals_revoked, + # Kept under its historical name; the others carry the key the batch reported. + "total_feeds_evaluated": totals["total_feeds"], + **{key: totals[key] for key in _SUMMED_KEYS if key != "total_feeds"}, "granted_stable_ids": granted_stable_ids[:MAX_REPORTED_IDS], "revoked_stable_ids": revoked_stable_ids[:MAX_REPORTED_IDS], "ids_omitted": ids_omitted, diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py index 5aed77384..855c6bdea 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill_fanout.py @@ -25,6 +25,7 @@ from datetime import date from unittest.mock import patch +_FANOUT = "tasks.seal_of_reliability.fanout" _PLAN = "tasks.seal_of_reliability.backfill.seal_backfill_orchestrator" _WORKER = "tasks.seal_of_reliability.backfill.seal_backfill_worker" @@ -33,8 +34,8 @@ class TestBackfillOrchestrator(unittest.TestCase): - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=5) def test_enqueues_worker_per_batch_plus_monitor( @@ -65,8 +66,8 @@ def test_enqueues_worker_per_batch_plus_monitor( self.assertEqual(result["batches"], 3) self.assertEqual(result["enqueued"], 3) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_every_worker_gets_the_same_explicit_window( @@ -98,8 +99,8 @@ def test_every_worker_gets_the_same_explicit_window( ] self.assertEqual(windows, [(START.isoformat(), END.isoformat())] * 2) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) def test_the_monitor_is_told_which_tracker_to_settle( @@ -125,8 +126,8 @@ def test_the_monitor_is_told_which_tracker_to_settle( monitor.kwargs["payload"]["task_name"], SEAL_BACKFILL_TASK_NAME ) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) def test_only_missing_narrows_the_candidate_set( @@ -145,8 +146,8 @@ def test_only_missing_narrows_the_candidate_set( self.assertTrue(count_mock.call_args.kwargs["exclude_backfilled"]) self.assertTrue(iter_mock.call_args.kwargs["exclude_backfilled"]) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=3) def test_only_missing_false_widens_it( @@ -163,8 +164,8 @@ def test_only_missing_false_widens_it( self.assertFalse(count_mock.call_args.kwargs["exclude_backfilled"]) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=4) def test_dry_run_enqueues_nothing( @@ -182,9 +183,9 @@ def test_dry_run_enqueues_nothing( self.assertEqual(result["enqueued"], 0) self.assertEqual(result["total_feeds"], 4) - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=False) + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=False) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_a_failed_enqueue_fails_its_batch_immediately( diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py new file mode 100644 index 000000000..4705999dd --- /dev/null +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_monitor_aggregation.py @@ -0,0 +1,262 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""`_aggregate_batches` against real TaskExecutionLog rows. + +Every test in test_seal_orchestrator_fanout.py patches this function out — it needs a real +session, and those tests drive the monitor with a MagicMock. Mocking it hides both whether it +sums correctly and whether it sums the right keys, so it is covered here instead. + +It is what turns each batch's stored report into the run-level one, which after a manual +backfill is the only thing an operator sees. +""" + +import unittest + +from sqlalchemy import delete + +from shared.database.database import with_db_session +from shared.database_gen.sqlacodegen_models import TaskExecutionLog +from tasks.seal_of_reliability.backfill.seal_backfill_orchestrator import ( + SEAL_BACKFILL_TASK_NAME, +) +from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, +) +from tasks.seal_of_reliability.orchestrator.seal_orchestrator_monitor import ( + MAX_REPORTED_IDS, + _aggregate_batches, + _parse_iso, +) +from test_shared.test_utils.database_utils import default_db_url + +RUN = "seal-agg-test-run" +OTHER_RUN = "seal-agg-test-other" + + +def _batch(task_name, run_id, entity_id, metadata): + return TaskExecutionLog( + task_name=task_name, + run_id=run_id, + entity_id=entity_id, + status="completed", + metadata_=metadata, + ) + + +class AggregationTestCase(unittest.TestCase): + @with_db_session(db_url=default_db_url) + def setUp(self, db_session): + self._cleanup(db_session) + + @with_db_session(db_url=default_db_url) + def tearDown(self, db_session): + self._cleanup(db_session) + + @staticmethod + def _cleanup(db_session): + db_session.execute( + delete(TaskExecutionLog).where( + TaskExecutionLog.run_id.in_([RUN, OTHER_RUN]) + ) + ) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _seed(entries, db_session=None): + for entry in entries: + db_session.add(entry) + db_session.commit() + + @staticmethod + @with_db_session(db_url=default_db_url) + def _aggregate(task_name=SEAL_ORCHESTRATOR_TASK_NAME, run_id=RUN, db_session=None): + return _aggregate_batches(db_session, run_id, task_name) + + +class TestAggregateBatches(AggregationTestCase): + def test_sums_across_batches(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + { + "total_feeds": 10, + "criterion_rows_written": 20, + "seals_granted": 2, + "seals_revoked": 1, + "granted_stable_ids": ["a", "b"], + "revoked_stable_ids": ["c"], + }, + ), + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0001", + { + "total_feeds": 5, + "criterion_rows_written": 7, + "seals_granted": 1, + "seals_revoked": 0, + "granted_stable_ids": ["d"], + }, + ), + ] + ) + + result = self._aggregate() + self.assertEqual(result["total_feeds_evaluated"], 15) + self.assertEqual(result["criterion_rows_written"], 27) + self.assertEqual(result["seals_granted"], 3) + self.assertEqual(result["seals_revoked"], 1) + self.assertEqual(sorted(result["granted_stable_ids"]), ["a", "b", "d"]) + self.assertEqual(result["revoked_stable_ids"], ["c"]) + self.assertEqual(result["ids_omitted"], 0) + + def test_a_backfill_snapshot_count_survives_to_the_run_report(self): + """The key a backfill batch reports and a nightly one does not. + + It was being dropped at aggregation, so a backfill's snapshot count never reached + the operator who triggered the run. + """ + self._seed( + [ + _batch( + SEAL_BACKFILL_TASK_NAME, + RUN, + "batch-0000", + {"total_feeds": 3, "snapshot_rows_written": 18}, + ), + _batch( + SEAL_BACKFILL_TASK_NAME, + RUN, + "batch-0001", + {"total_feeds": 2, "snapshot_rows_written": 12}, + ), + ] + ) + + result = self._aggregate(task_name=SEAL_BACKFILL_TASK_NAME) + self.assertEqual(result["snapshot_rows_written"], 30) + self.assertEqual(result["total_feeds_evaluated"], 5) + + def test_a_nightly_batch_contributes_zero_for_keys_it_never_reports(self): + """One aggregation serves both fan-outs, so a missing key must not raise.""" + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + {"total_feeds": 4, "criterion_rows_written": 4}, + ) + ] + ) + + result = self._aggregate() + self.assertEqual(result["snapshot_rows_written"], 0) + + def test_it_only_sums_its_own_run(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + OTHER_RUN, + "batch-0000", + {"total_feeds": 99}, + ), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + + def test_it_only_sums_its_own_task_name(self): + """The two fan-outs share this function; a backfill run must not absorb a nightly one.""" + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch(SEAL_BACKFILL_TASK_NAME, RUN, "batch-0001", {"total_feeds": 50}), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + self.assertEqual( + self._aggregate(task_name=SEAL_BACKFILL_TASK_NAME)["total_feeds_evaluated"], + 50, + ) + + def test_a_batch_with_no_metadata_is_skipped(self): + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0000", {"total_feeds": 4} + ), + _batch(SEAL_ORCHESTRATOR_TASK_NAME, RUN, "batch-0001", {}), + ] + ) + self.assertEqual(self._aggregate()["total_feeds_evaluated"], 4) + + def test_a_run_with_no_batches_aggregates_to_zero(self): + result = self._aggregate() + self.assertEqual(result["total_feeds_evaluated"], 0) + self.assertEqual(result["granted_stable_ids"], []) + self.assertEqual(result["ids_omitted"], 0) + + def test_the_id_lists_are_capped_and_the_overflow_counted(self): + """The seal tables hold every transition; this only bounds the response size.""" + granted = [f"mdb-{n}" for n in range(MAX_REPORTED_IDS + 5)] + self._seed( + [ + _batch( + SEAL_ORCHESTRATOR_TASK_NAME, + RUN, + "batch-0000", + {"granted_stable_ids": granted}, + ) + ] + ) + + result = self._aggregate() + self.assertEqual(len(result["granted_stable_ids"]), MAX_REPORTED_IDS) + self.assertEqual(result["ids_omitted"], 5) + + +class TestParseIso(unittest.TestCase): + """The deadline check silently loses its guard if this returns None, so pin the branches.""" + + def test_absent_is_none(self): + self.assertIsNone(_parse_iso(None)) + self.assertIsNone(_parse_iso("")) + + def test_unparseable_is_none_rather_than_raising(self): + self.assertIsNone(_parse_iso("not a timestamp")) + + def test_naive_is_read_as_utc(self): + parsed = _parse_iso("2026-06-01T12:00:00") + self.assertEqual(parsed.tzinfo, __import__("datetime").timezone.utc) + + def test_offset_is_preserved(self): + self.assertIsNotNone(_parse_iso("2026-06-01T12:00:00+02:00").tzinfo) + + +if __name__ == "__main__": + unittest.main() diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py index 843175556..4184afc4c 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/orchestrator/test_seal_orchestrator_fanout.py @@ -32,12 +32,13 @@ # seal_orchestrator (producer) # --------------------------------------------------------------------------- +_FANOUT = "tasks.seal_of_reliability.fanout" _PLAN = "tasks.seal_of_reliability.orchestrator.seal_orchestrator" class TestSealOrchestratorHandler(unittest.TestCase): - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=5) def test_enqueues_worker_per_batch_plus_monitor( @@ -64,8 +65,8 @@ def test_enqueues_worker_per_batch_plus_monitor( self.assertEqual(result["enqueued"], 3) self.assertFalse(result["dry_run"]) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) def test_dynamic_task_names_use_prefix( @@ -82,8 +83,8 @@ def test_dynamic_task_names_use_prefix( self.assertTrue(all(n.startswith("seal-orchestrator-") for n in names)) self.assertTrue(any(n.startswith("seal-orchestrator-monitor-") for n in names)) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_dry_run_enqueues_nothing( @@ -101,8 +102,8 @@ def test_dry_run_enqueues_nothing( self.assertEqual(result["enqueued"], 0) self.assertEqual(result["batches"], 2) - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=0) def test_no_eligible_feeds_enqueues_nothing( @@ -120,15 +121,16 @@ def test_no_eligible_feeds_enqueues_nothing( self.assertEqual(result["enqueued"], 0) self.assertEqual(result["batches"], 0) - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue") + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=1) def test_failed_enqueue_marks_batch_failed_immediately( self, count_mock, iter_mock, enqueue_mock, start_run_mock, mark_failed_mock ): from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, seal_orchestrator_handler, ) @@ -138,12 +140,13 @@ def test_failed_enqueue_marks_batch_failed_immediately( seal_orchestrator_handler({"dry_run": False, "batch_size": 1}) mark_failed_mock.assert_called_once() - call_args = mark_failed_mock.call_args[0] - self.assertTrue(call_args[0].startswith("seal-")) - self.assertEqual(call_args[1], "batch-0000") + task_name, run_id, batch_id = mark_failed_mock.call_args[0] + self.assertEqual(task_name, SEAL_ORCHESTRATOR_TASK_NAME) + self.assertTrue(run_id.startswith("seal-")) + self.assertEqual(batch_id, "batch-0000") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task") @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds") def test_non_positive_batch_size_raises( @@ -163,9 +166,9 @@ def test_non_positive_batch_size_raises( enqueue_mock.assert_not_called() start_run_mock.assert_not_called() - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=6) def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( @@ -176,6 +179,7 @@ def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( leftover pre-registered batch_id must be failed immediately, not left `triggered` for the monitor's deadline to eventually notice.""" from tasks.seal_of_reliability.orchestrator.seal_orchestrator import ( + SEAL_ORCHESTRATOR_TASK_NAME, seal_orchestrator_handler, ) @@ -187,16 +191,18 @@ def test_stream_yields_fewer_batches_than_planned_marks_leftover_failed( mark_failed_mock.assert_called_once() call_args, call_kwargs = mark_failed_mock.call_args - self.assertTrue(call_args[0].startswith("seal-")) - self.assertEqual(call_args[1], "batch-0002") + task_name, run_id, batch_id = call_args + self.assertEqual(task_name, SEAL_ORCHESTRATOR_TASK_NAME) + self.assertTrue(run_id.startswith("seal-")) + self.assertEqual(batch_id, "batch-0002") self.assertEqual( call_kwargs["error_message"], "no eligible-feed data for this batch (count/stream mismatch)", ) - @patch(f"{_PLAN}._mark_enqueue_failed") - @patch(f"{_PLAN}._start_run") - @patch(f"{_PLAN}._enqueue", return_value=True) + @patch(f"{_FANOUT}.mark_enqueue_failed") + @patch(f"{_FANOUT}.start_run") + @patch(f"{_FANOUT}.enqueue_task", return_value=True) @patch(f"{_PLAN}.iter_eligible_stable_ids") @patch(f"{_PLAN}.count_eligible_feeds", return_value=2) def test_stream_yields_more_batches_than_planned_logs_and_does_not_mark_failed( @@ -213,13 +219,14 @@ def test_stream_yields_more_batches_than_planned_logs_and_does_not_mark_failed( # chunk. iter_mock.return_value = iter([["mdb-1"], ["mdb-2"], ["mdb-3"]]) - with self.assertLogs( - "tasks.seal_of_reliability.orchestrator.seal_orchestrator", level="ERROR" - ) as log_ctx: + with self.assertLogs(_FANOUT, level="ERROR") as log_ctx: result = seal_orchestrator_handler({"dry_run": False, "batch_size": 1}) mark_failed_mock.assert_not_called() - self.assertTrue(any("newly-eligible" in msg for msg in log_ctx.output)) + self.assertTrue( + any("more batches than the plan-time count" in m for m in log_ctx.output) + ) + self.assertTrue(any("seal_orchestrator" in m for m in log_ctx.output)) self.assertEqual(result["enqueued"], 2) From 976f5823b4f3f7cd2c038011f018175973240e0a Mon Sep 17 00:00:00 2001 From: jcpitre Date: Wed, 26 Aug 2026 21:20:28 -0400 Subject: [PATCH 10/13] Added simulation mode --- .../backfill/backfill_seal_of_reliability.py | 12 + .../backfill/seal_backfill.py | 114 ++++++-- .../backfill/simulation.py | 260 ++++++++++++++++++ .../backfill/test_seal_backfill.py | 253 +++++++++++++++++ 4 files changed, 619 insertions(+), 20 deletions(-) create mode 100644 functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py index 8abddd17b..160a73c60 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -65,6 +65,8 @@ def get_parameters(payload: dict): payload.get("snapshot_mode", DEFAULT_SNAPSHOT_MODE), payload.get("resume_from_snapshot", False), payload.get("max_reported_feeds", DEFAULT_MAX_REPORTED_FEEDS), + payload.get("simulate", None), + payload.get("trace", False), ) @@ -86,6 +88,12 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: resume_from_snapshot seed from the snapshot before march_start (#1803). Default: False max_reported_feeds cap on the `feeds` list in the response. Default: 50 + simulate force observed statuses on given days, counted from each feed's + march start: {"official": {"fail": [3, 4], "unknown": [8]}}. + Requires dry_run — a forced verdict must never be written, since + the stored row carries no mark saying it was simulated + trace return one row per feed, criterion and day: observed, confirmed, + phase and probation. Marches without writing when dry_run """ ( stable_feed_ids, @@ -100,6 +108,8 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: snapshot_mode, resume_from_snapshot, max_reported_feeds, + simulate, + trace, ) = get_parameters(payload) return backfill_seals( stable_feed_ids=stable_feed_ids, @@ -114,4 +124,6 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: snapshot_mode=snapshot_mode, resume_from_snapshot=resume_from_snapshot, max_reported_feeds=max_reported_feeds, + simulate=simulate, + trace=trace, ) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index 0330af8d6..c2ad3f4cb 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -48,6 +48,14 @@ collect_inputs, is_seal_eligible, ) +from tasks.seal_of_reliability.backfill.simulation import ( + MAX_TRACE_ROWS, + check_simulation_fits, + observe, + parse_simulation, + policy_for, + trace_row, +) from tasks.seal_of_reliability.criteria import CriterionStatus, SealCriterionName from tasks.seal_of_reliability.seal_updater import ( DEFAULT_BATCH_SIZE, @@ -257,6 +265,16 @@ def _upsert_seals_from_backfill( ) +def _longest_march(windows: Dict[str, Tuple[date, date]]) -> int: + """Days in the longest window of the run, both ends included. + + Feeds clamped to their own `created_at` march fewer, so this is an upper bound rather + than a length they all share. It is the report's `days`, and the range a simulated day + offset has to fall inside. Zero when nothing was selected. + """ + return max(((end - start).days + 1 for start, end in windows.values()), default=0) + + def _march( db_session: Session, feeds: Sequence[Gtfsfeed], @@ -266,6 +284,9 @@ def _march( snapshot_mode: str, resume_from_snapshot: bool, partial_run: bool, + simulation: Optional[Dict[str, Dict[int, CriterionStatus]]] = None, + trace: bool = False, + write: bool = True, ) -> dict: """Replay the nightly evaluation day by day for one batch, and write the final day. @@ -274,7 +295,13 @@ def _march( Ascending order is the algorithm, not a convenience: each day feeds the next. """ if not feeds: - return {"feeds": 0, "criterion_rows": 0, "snapshot_rows": 0, "outcomes": []} + return { + "feeds": 0, + "criterion_rows": 0, + "snapshot_rows": 0, + "outcomes": [], + "trace": [], + } marched_days = days_between(min(start for start, _ in windows.values()), end_date) @@ -282,7 +309,9 @@ def _march( inputs = collect_inputs(db_session, feeds, marched_days, evaluators) states = _seed_states(db_session, feeds, windows, resume_from_snapshot) + simulation = simulation or {} snapshot_rows = 0 + trace_rows: List[dict] = [] for today in marched_days: now = day_start(today) @@ -301,19 +330,30 @@ def _march( official=feed.official, inputs=inputs, ) + offset = (today - windows[feed.id][0]).days for evaluator in evaluators: key = (feed.id, evaluator.name.value) + observation = observe(evaluator, ctx, simulation, offset) + # A simulation may lend a criterion a grace period or probation it does not + # have, which is the only way Official shows any debouncing at all. + grace, probation = policy_for(evaluator, simulation) states[key] = transition( prev=states.get(key), - observation=evaluator.evaluate(ctx), - grace_period=evaluator.grace_period, - probation_period=evaluator.probation_period, + observation=observation, + grace_period=grace, + probation_period=probation, now=now, feed_id=feed.id, ) days_states.append(states[key]) - - if snapshot_mode == "all": + if trace and len(trace_rows) < MAX_TRACE_ROWS: + trace_rows.append( + trace_row( + feed, evaluator, today, offset, observation, states[key] + ) + ) + + if write and snapshot_mode == "all": # The only mode that writes inside the loop, flushed per day so a year's march # does not hold every day in memory. _upsert_criterion_snapshot(db_session, days_states, today) @@ -323,19 +363,21 @@ def _march( final_states = list(states.values()) outcomes = _final_outcomes(db_session, feeds, windows, states, partial_run) - _upsert_criteria(db_session, final_states, day_start(end_date)) - if snapshot_mode == "final": - _upsert_criterion_snapshot(db_session, final_states, end_date) - snapshot_rows += len(final_states) - if outcomes: - _upsert_seals_from_backfill(db_session, outcomes, day_start(end_date)) - db_session.commit() + if write: + _upsert_criteria(db_session, final_states, day_start(end_date)) + if snapshot_mode == "final": + _upsert_criterion_snapshot(db_session, final_states, end_date) + snapshot_rows += len(final_states) + if outcomes: + _upsert_seals_from_backfill(db_session, outcomes, day_start(end_date)) + db_session.commit() return { "feeds": len(feeds), "criterion_rows": len(final_states), "snapshot_rows": snapshot_rows, "outcomes": outcomes, + "trace": trace_rows, } @@ -394,6 +436,8 @@ def backfill_seals( snapshot_mode: str = DEFAULT_SNAPSHOT_MODE, resume_from_snapshot: bool = False, max_reported_feeds: int = DEFAULT_MAX_REPORTED_FEEDS, + simulate: Optional[dict] = None, + trace: bool = False, ) -> dict: """Plan and run the backfill for an explicit list of feeds. @@ -401,12 +445,24 @@ def backfill_seals( ineligible ids are skipped with a warning; it raises only if none can be used. See `backfill_seal_of_reliability` for the parameters as an operator passes them. + `simulate` forces observed statuses on named days, and `trace` returns the state each + day left behind. Both are inspection tools and neither may write: see the dry_run check + below. + Returns a report; `days` is the longest march in the run, since feeds clamped to their own `created_at` march fewer. """ started = clock.monotonic() if not stable_feed_ids: raise ValueError("stable_feed_ids is required and must be non-empty") + if simulate and not dry_run: + # A simulated verdict written to seal_criterion is indistinguishable from an earned + # one — the row carries no provenance. Refuse rather than quietly downgrade, so an + # operator cannot believe a real run happened. + raise ValueError( + "simulate requires dry_run: forced verdicts must never be written to the seal " + "tables, where nothing would mark them as simulated" + ) if snapshot_mode not in SNAPSHOT_MODES: raise ValueError( f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" @@ -416,6 +472,7 @@ def backfill_seals( window_start, window_end = resolve_window(start_date, end_date, days_back) evaluators = _resolve_evaluators(criteria) + simulation = parse_simulation(simulate, evaluators) # By-id load then eligibility in Python, as `update_seals` does: tells "not found" from # "found but ineligible" without a second query. @@ -443,9 +500,9 @@ def backfill_seals( windows = { feed.id: (march_start_for(feed, window_start), window_end) for feed in selected } - longest_march = ( - max((end - start).days + 1 for start, end in windows.values()) if windows else 0 - ) + longest_march = _longest_march(windows) + if simulation: + check_simulation_fits(simulation, longest_march) feed_plans = [ { @@ -488,8 +545,12 @@ def backfill_seals( "revoked_stable_ids": [], } - if not dry_run: + # A plain dry run stops at the plan. One asked to simulate or trace has to march — + # that is the whole point — so it marches with writing suppressed. + inspecting = bool(simulation) or trace + if not dry_run or inspecting: outcomes: List[dict] = [] + trace_rows: List[dict] = [] for batch in batched(selected, batch_size): result = _march( db_session, @@ -497,13 +558,26 @@ def backfill_seals( windows, evaluators, window_end, - snapshot_mode, + "none" if dry_run else snapshot_mode, resume_from_snapshot, partial_run, + simulation=simulation, + trace=trace, + write=not dry_run, ) - report["criterion_rows_written"] += result["criterion_rows"] - report["snapshot_rows_written"] += result["snapshot_rows"] + if not dry_run: + report["criterion_rows_written"] += result["criterion_rows"] + report["snapshot_rows_written"] += result["snapshot_rows"] outcomes.extend(result["outcomes"]) + trace_rows.extend(result["trace"]) + if trace: + report["trace"] = trace_rows + report["trace_truncated"] = len(trace_rows) >= MAX_TRACE_ROWS + if simulation: + report["simulated"] = { + criterion: forced.as_reported() + for criterion, forced in simulation.items() + } granted = [outcome for outcome in outcomes if outcome["granted"]] # Only reachable with only_missing=False, but reported anyway so the monitor's diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py new file mode 100644 index 000000000..47c12135b --- /dev/null +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py @@ -0,0 +1,260 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Forced per-day statuses and the day-by-day trace, for inspecting a backfill march. + +Neither may write. A simulated verdict in `seal_criterion` would be indistinguishable from +an earned one — the row carries no provenance — so `backfill_seals` refuses to combine +`simulate` with a real run, and a traced dry run marches with writing suppressed. + +Day offsets are counted from each feed's own march start, so day 0 is that feed's first +evaluated day: the one denied a grace period. Anchoring to the run's `start_date` instead +would point at days a younger feed never marched. +""" + +from dataclasses import dataclass, field +from datetime import date, timedelta +from typing import Any, Dict, Mapping, Optional, Sequence + +from tasks.seal_of_reliability.criteria import CriterionStatus +from tasks.seal_of_reliability.evaluators.base import CriterionObservation +from tasks.seal_of_reliability.state_machine import phase + +# Cap on the trace a single call returns. A year x a batch of feeds x six criteria would be +# a response no one can read and Cloud Logging would drop; a simulation is a small thing. +MAX_TRACE_ROWS: int = 2000 + +# Payload keys inside a criterion that set policy rather than name days. Status names are a +# closed set, so there is no collision. +GRACE_KEY = "grace_days" +PROBATION_KEY = "probation_days" +_POLICY_KEYS = (GRACE_KEY, PROBATION_KEY) + +# Distinguishes "not given" — use the evaluator's own value — from an explicit null, which +# means the criterion has no such period. +_UNSET = object() + + +@dataclass(frozen=True) +class CriterionSimulation: + """What a payload forces for one criterion: its days, and optionally its policy. + + Overriding the periods is what makes a simulation informative for a criterion that has + none. Official is a point-in-time check with no grace and no probation, so a forced + failure confirms the same day and the trace shows nothing about debouncing. Given + `grace_days: 14` it behaves like the criteria still to be written, and the march can be + watched doing the thing a backfill exists to reconstruct. + + Only ever applied to a dry run, so no fabricated policy can reach the seal tables. + """ + + days: Mapping[int, CriterionStatus] = field(default_factory=dict) + grace_period: Optional[timedelta] = None + probation_period: Optional[timedelta] = None + grace_overridden: bool = False + probation_overridden: bool = False + + def grace_for(self, evaluator) -> Optional[timedelta]: + return self.grace_period if self.grace_overridden else evaluator.grace_period + + def probation_for(self, evaluator) -> Optional[timedelta]: + return ( + self.probation_period + if self.probation_overridden + else evaluator.probation_period + ) + + def as_reported(self) -> dict: + """The echo returned in the report, so a run states what it was told to pretend.""" + echo: Dict[str, Any] = { + offset: status.value for offset, status in sorted(self.days.items()) + } + if self.grace_overridden: + echo[GRACE_KEY] = ( + self.grace_period.days if self.grace_period is not None else None + ) + if self.probation_overridden: + echo[PROBATION_KEY] = ( + self.probation_period.days + if self.probation_period is not None + else None + ) + return echo + + +def policy_for(evaluator, simulation) -> tuple: + """The (grace, probation) this run applies to `evaluator` — its own unless overridden.""" + forced = (simulation or {}).get(evaluator.name.value) + if forced is None: + return evaluator.grace_period, evaluator.probation_period + return forced.grace_for(evaluator), forced.probation_for(evaluator) + + +def _parse_period(value, criterion: str, key: str) -> Optional[timedelta]: + """A whole number of days, or null meaning the criterion has no such period.""" + if value is None: + return None + try: + days = int(value) + except (TypeError, ValueError): + raise ValueError( + f"{key} for {criterion!r} must be a whole number of days or null, got {value!r}" + ) + if days < 0: + raise ValueError( + f"{key} for {criterion!r} cannot be negative; got {days}. Use null to mean the " + f"criterion has no such period." + ) + return timedelta(days=days) + + +def parse_simulation( + simulate: Optional[dict], evaluators: Sequence +) -> Dict[str, CriterionSimulation]: + """Turn the `simulate` payload into criterion -> CriterionSimulation. + + Shape, offsets counted from each feed's own march start:: + + {"official": {"grace_days": 14, "probation_days": 180, "fail": [3, 4], "unknown": [8]}} + + Offsets rather than dates because a scenario is about the shape of a history — "it fails + on day 3" — not about a calendar. Anchoring per feed rather than to the window start is + what makes day 0 the feed's first evaluation, the one denied a grace period. + + `grace_days` and `probation_days` are optional and override the criterion's own values + for the run. Omit either to keep the evaluator's; pass null to mean it has none. + """ + if not simulate: + return {} + + known = {evaluator.name.value for evaluator in evaluators} + forced_statuses = { + status.value for status in CriterionStatus if status.is_verdict + } | { + CriterionStatus.UNKNOWN.value, + CriterionStatus.NOT_APPLICABLE.value, + } + + parsed: Dict[str, CriterionSimulation] = {} + for criterion, by_status in simulate.items(): + if criterion not in known: + raise ValueError( + f"Cannot simulate unknown criterion {criterion!r}. This run evaluates: " + f"{sorted(known)}" + ) + by_status = dict(by_status or {}) + grace = by_status.pop(GRACE_KEY, _UNSET) + probation = by_status.pop(PROBATION_KEY, _UNSET) + + days: Dict[int, CriterionStatus] = {} + for status, offsets in by_status.items(): + if status not in forced_statuses: + raise ValueError( + f"Cannot simulate status {status!r} for {criterion!r}. Valid: " + f"{sorted(forced_statuses)} — plus {list(_POLICY_KEYS)}" + ) + for offset in offsets or []: + offset = int(offset) + if offset < 0: + raise ValueError( + f"Simulated day offsets are counted from the march start and cannot " + f"be negative; got {offset} for {criterion!r}" + ) + if offset in days and days[offset].value != status: + raise ValueError( + f"Day {offset} of {criterion!r} is simulated twice, as " + f"{days[offset].value!r} and {status!r}" + ) + days[offset] = CriterionStatus(status) + + parsed[criterion] = CriterionSimulation( + days=days, + grace_period=( + None if grace is _UNSET else _parse_period(grace, criterion, GRACE_KEY) + ), + probation_period=( + None + if probation is _UNSET + else _parse_period(probation, criterion, PROBATION_KEY) + ), + grace_overridden=grace is not _UNSET, + probation_overridden=probation is not _UNSET, + ) + return parsed + + +def check_simulation_fits( + simulation: Dict[str, Dict[int, CriterionStatus]], + longest_march: int, +) -> None: + """Reject offsets no march reaches, rather than letting them silently do nothing. + + A typo like day 400 in an eight-day window would otherwise look like it worked. + """ + if not longest_march: + # Nothing was selected, so blaming the offsets would send the reader to the wrong + # parameter entirely. `only_missing` excluding an already-backfilled feed is the + # usual cause. + raise ValueError( + "Nothing to simulate: no feed was selected for this run. If the feeds already " + "have seal state, only_missing (default true) excludes them — pass " + "only_missing=false to march them again." + ) + for criterion, forced in simulation.items(): + beyond = sorted(offset for offset in forced.days if offset >= longest_march) + if beyond: + raise ValueError( + f"Simulated day(s) {beyond} for {criterion!r} are past the end of every " + f"feed's march; the longest here is {longest_march} day(s), so valid " + f"offsets are 0..{max(longest_march - 1, 0)}" + ) + + +def observe(evaluator, ctx, simulation, offset: int) -> CriterionObservation: + """The criterion's own verdict, unless this day is simulated. + + Days the payload does not name fall through to the real evaluator, so a simulation is + real data with per-day overrides rather than a wholly synthetic run. + """ + entry = simulation.get(evaluator.name.value) + forced = entry.days.get(offset) if entry else None + if forced is None: + return evaluator.evaluate(ctx) + return CriterionObservation( + criterion=evaluator.name, + observed_status=forced, + reason=f"simulated: {forced.value} on day {offset}", + ) + + +def trace_row(feed, evaluator, today: date, offset: int, observation, state) -> dict: + """One day of one criterion, as the state machine left it. + + Flask's jsonify sorts keys, so this order is for reading the source, not the response. + """ + return { + "stable_id": feed.stable_id, + "criterion": evaluator.name.value, + "day": offset, + "date": today.isoformat(), + "observed_status": observation.observed_status.value, + "confirmed_status": state.confirmed_status.value, + "phase": phase(state).value, + "probation_start": ( + state.probation_start.date().isoformat() if state.probation_start else None + ), + "simulated": observation.reason.startswith("simulated:"), + "reason": observation.reason, + } diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py index 9dfe93247..7a476b39a 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -113,6 +113,8 @@ def test_defaults(self): snapshot_mode, resume_from_snapshot, max_reported_feeds, + simulate, + trace, ) = get_parameters({"stable_feed_ids": ["a"]}) self.assertEqual(stable_feed_ids, ["a"]) @@ -127,6 +129,8 @@ def test_defaults(self): self.assertEqual(snapshot_mode, "final") self.assertFalse(resume_from_snapshot) self.assertEqual(max_reported_feeds, 50) + self.assertIsNone(simulate) + self.assertFalse(trace, "a run must not pay for a trace unless asked") def test_empty_payload_does_not_raise_here(self): """Validation belongs to the engine, so the parser stays a plain reader.""" @@ -567,5 +571,254 @@ def test_without_the_flag_the_same_window_cold_starts(self): ) +class TestSimulateAndTrace(MarchTestCase): + """Forced per-day statuses, and the day-by-day trace they are there to make visible.""" + + def simulate(self, stable_id, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[stable_id], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + **kwargs, + ) + + def test_a_simulated_run_never_writes(self): + """The reason simulate forces dry_run: a forced verdict in seal_criterion would be + indistinguishable from an earned one.""" + report = self.simulate(MARCHED, simulate={"official": {"fail": [0, 1]}}) + self.assertEqual(criterion_rows(MARCHED), {}) + self.assertIsNone(seal_row(MARCHED)) + self.assertEqual(report["criterion_rows_written"], 0) + + def test_writing_with_a_simulation_is_refused(self): + with self.assertRaises(ValueError) as caught: + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + simulate={"official": {"fail": [0]}}, + ) + self.assertIn("dry_run", str(caught.exception)) + + def test_a_forced_failure_reaches_the_state_machine(self): + """Day 0 is the first evaluation, so it gets no grace and confirms immediately.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [0]}}, trace=True + ) + day_zero = next(row for row in report["trace"] if row["day"] == 0) + self.assertEqual(day_zero["observed_status"], "fail") + self.assertEqual(day_zero["confirmed_status"], "fail") + self.assertTrue(day_zero["simulated"]) + + def test_unnamed_days_fall_through_to_the_real_evaluator(self): + """A simulation is real data with overrides, not a synthetic run.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [0]}}, trace=True + ) + day_one = next(row for row in report["trace"] if row["day"] == 1) + self.assertEqual(day_one["observed_status"], "pass") + self.assertFalse(day_one["simulated"]) + + def test_a_streak_past_the_grace_period_confirms_then_serves_probation(self): + """Days 1-39 fail, then the feed recovers — the whole arc in one trace. + + The stand-in's grace period is 30 days and the streak starts on day 1, so day 31 is + the first confirmed failure. Recovery on day 40 clears the status but opens + probation, which 34 remaining days cannot serve. + """ + report = self.simulate( + MARCHED, + simulate={"official": {"fail": list(range(1, 40))}}, + trace=True, + ) + by_day = {row["day"]: row for row in report["trace"]} + + self.assertEqual(by_day[30]["confirmed_status"], "pass", "last day of grace") + self.assertEqual(by_day[31]["confirmed_status"], "fail", "grace outlasted") + + last = report["trace"][-1] + self.assertEqual(last["observed_status"], "pass") + self.assertEqual(last["confirmed_status"], "pass", "recovered") + self.assertEqual(last["phase"], "on_probation") + self.assertIsNotNone(last["probation_start"]) + + def test_the_trace_covers_every_marched_day(self): + report = self.simulate(MARCHED, trace=True) + days = [row["day"] for row in report["trace"]] + self.assertEqual(days, list(range(len(days)))) + self.assertEqual( + len(days), (MARCH_END - MARCH_START).days + 1, "one row per day" + ) + + def test_the_trace_is_offset_from_the_feed_own_march_start(self): + """Day 0 is the feed's first evaluated day, not the window start.""" + report = self.simulate(MARCHED, trace=True) + first = report["trace"][0] + self.assertEqual(first["day"], 0) + self.assertEqual(first["date"], MARCH_START.isoformat()) + + def test_an_unknown_criterion_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"punctual": {"fail": [0]}}) + self.assertIn("punctual", str(caught.exception)) + + def test_an_unknown_status_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"broken": [0]}}) + self.assertIn("broken", str(caught.exception)) + + def test_a_negative_offset_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"fail": [-1]}}) + self.assertIn("negative", str(caught.exception)) + + def test_an_offset_past_the_march_is_rejected(self): + """A typo like day 400 in a short window would otherwise silently do nothing.""" + with self.assertRaises(ValueError) as caught: + self.simulate(MARCHED, simulate={"official": {"fail": [9999]}}) + self.assertIn("9999", str(caught.exception)) + + def test_the_report_echoes_what_was_simulated(self): + report = self.simulate( + MARCHED, simulate={"official": {"fail": [2], "unknown": [4]}} + ) + self.assertEqual(report["simulated"]["official"], {2: "fail", 4: "unknown"}) + + def test_a_plain_dry_run_still_stops_at_the_plan(self): + """Only simulate or trace makes a dry run pay for the march.""" + report = self.simulate(MARCHED) + self.assertNotIn("trace", report) + + +class TestSimulatedPolicy(MarchTestCase): + """Lending a criterion a grace period and probation it does not have. + + The stand-in already has both, so these use `grace_days`/`probation_days` to *remove* + and to *change* them — the same mechanism that lets Official, which has neither, show + debouncing in a simulation. + """ + + def simulate(self, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + trace=True, + **kwargs, + ) + + @staticmethod + def _day(report, day): + return next(row for row in report["trace"] if row["day"] == day) + + def test_a_lent_grace_period_absorbs_a_failure(self): + """Without an override this criterion confirms on day 1; with 14 days it holds.""" + report = self.simulate(simulate={"official": {"grace_days": 14, "fail": [1]}}) + day_one = self._day(report, 1) + self.assertEqual(day_one["observed_status"], "fail") + self.assertEqual(day_one["confirmed_status"], "pass", "held by the lent grace") + self.assertEqual(day_one["phase"], "in_grace_period") + + def test_a_removed_grace_period_confirms_immediately(self): + """null means the criterion has none, which is Official's real behaviour.""" + report = self.simulate(simulate={"official": {"grace_days": None, "fail": [1]}}) + self.assertEqual(self._day(report, 1)["confirmed_status"], "fail") + + def test_the_lent_grace_period_expires_on_schedule(self): + report = self.simulate( + simulate={"official": {"grace_days": 14, "fail": list(range(1, 20))}} + ) + self.assertEqual( + self._day(report, 14)["confirmed_status"], "pass", "last day inside grace" + ) + self.assertEqual( + self._day(report, 15)["confirmed_status"], "fail", "grace outlasted" + ) + + def test_a_lent_probation_opens_on_recovery(self): + report = self.simulate( + simulate={ + "official": { + "grace_days": None, + "probation_days": 180, + "fail": [1], + } + } + ) + recovered = self._day(report, 2) + self.assertEqual(recovered["confirmed_status"], "pass") + self.assertEqual(recovered["phase"], "on_probation") + + def test_a_removed_probation_never_opens_one(self): + report = self.simulate( + simulate={ + "official": { + "grace_days": None, + "probation_days": None, + "fail": [1], + } + } + ) + recovered = self._day(report, 2) + self.assertEqual(recovered["confirmed_status"], "pass") + self.assertEqual(recovered["phase"], "steady") + self.assertIsNone(recovered["probation_start"]) + + def test_a_shorter_probation_is_served_sooner(self): + report = self.simulate( + simulate={ + "official": {"grace_days": None, "probation_days": 5, "fail": [1]} + } + ) + # Probation opens on day 2 and clears once five days have passed. + self.assertEqual(self._day(report, 6)["phase"], "on_probation") + self.assertEqual(self._day(report, 7)["phase"], "steady") + + def test_omitting_the_keys_keeps_the_criterion_own_policy(self): + """The stand-in's own 30-day grace, untouched — a streak from day 1 confirms on 31.""" + report = self.simulate(simulate={"official": {"fail": list(range(1, 40))}}) + self.assertEqual(self._day(report, 30)["confirmed_status"], "pass") + self.assertEqual(self._day(report, 31)["confirmed_status"], "fail") + + def test_the_report_echoes_the_lent_policy(self): + report = self.simulate( + simulate={ + "official": {"grace_days": 14, "probation_days": 180, "fail": [1]} + } + ) + echoed = report["simulated"]["official"] + self.assertEqual(echoed["grace_days"], 14) + self.assertEqual(echoed["probation_days"], 180) + self.assertEqual(echoed[1], "fail") + + def test_a_negative_period_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"grace_days": -1}}) + self.assertIn("negative", str(caught.exception)) + + def test_a_non_numeric_period_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"probation_days": "a fortnight"}}) + self.assertIn("probation_days", str(caught.exception)) + + def test_a_lent_policy_never_writes(self): + """Same rule as forced verdicts: a fabricated policy must not reach the tables.""" + with self.assertRaises(ValueError): + with self.registry(_script_for([])): + backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + simulate={"official": {"grace_days": 14}}, + ) + + if __name__ == "__main__": unittest.main() From c97df6cee572240c0362ee3ae8e0fa4e57414741 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Thu, 27 Aug 2026 15:40:28 -0400 Subject: [PATCH 11/13] Reduce the size of the trace. --- .../backfill/backfill_seal_of_reliability.py | 13 +- .../backfill/seal_backfill.py | 14 +- .../backfill/simulation.py | 199 +++++++++++++++--- .../backfill/test_seal_backfill.py | 175 +++++++++++++-- 4 files changed, 339 insertions(+), 62 deletions(-) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py index 160a73c60..c46d391d7 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -88,12 +88,13 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: resume_from_snapshot seed from the snapshot before march_start (#1803). Default: False max_reported_feeds cap on the `feeds` list in the response. Default: 50 - simulate force observed statuses on given days, counted from each feed's - march start: {"official": {"fail": [3, 4], "unknown": [8]}}. - Requires dry_run — a forced verdict must never be written, since - the stored row carries no mark saying it was simulated - trace return one row per feed, criterion and day: observed, confirmed, - phase and probation. Marches without writing when dry_run + simulate force statuses per criterion, on days counted from each feed's + march start: {"fresh_coverage": {"default": "pass", "fail": [3]}}. + Requires dry_run; see `parse_simulation` for the full shape + trace return the march day by day: every seal_criterion field, plus where + it came from. Marches without writing when dry_run. Consecutive days + in which nothing changed are always collapsed into one entry — its + first day, its last, and the count between """ ( stable_feed_ids, diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index e61ac8846..354d1057c 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -51,6 +51,7 @@ from tasks.seal_of_reliability.backfill.simulation import ( MAX_TRACE_ROWS, check_simulation_fits, + collapse_runs, observe, parse_simulation, policy_for, @@ -356,9 +357,7 @@ def _march( days_states.append(states[key]) if trace and len(trace_rows) < MAX_TRACE_ROWS: trace_rows.append( - trace_row( - feed, evaluator, today, offset, observation, states[key] - ) + trace_row(feed, evaluator, offset, observation, states[key]) ) if write and snapshot_mode == "all": @@ -454,8 +453,9 @@ def backfill_seals( `backfill_seal_of_reliability` for the parameters as an operator passes them. `simulate` forces observed statuses on named days, and `trace` returns the state each - day left behind. Both are inspection tools and neither may write: see the dry_run check - below. + day left behind, always collapsed to one entry per unchanged stretch — a year of days is + mostly repetition, and no caller wanted it row by row. Both are inspection tools and + neither may write: see the dry_run check below. Returns a report; `days` is the longest march in the run, since feeds clamped to their own `created_at` march fewer. @@ -579,7 +579,9 @@ def backfill_seals( outcomes.extend(result["outcomes"]) trace_rows.extend(result["trace"]) if trace: - report["trace"] = trace_rows + report["trace"] = collapse_runs(trace_rows) + # Counted on the marched days, not on the collapsed entries: the cap is what the + # march stopped recording, and collapsing happens after. report["trace_truncated"] = len(trace_rows) >= MAX_TRACE_ROWS if simulation: report["simulated"] = { diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py index b50b3c847..21ed19d73 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py @@ -24,23 +24,55 @@ would point at days a younger feed never marched. """ -from dataclasses import dataclass, field -from datetime import date, timedelta -from typing import Any, Dict, Mapping, Optional, Sequence +from dataclasses import dataclass, field, fields as dataclass_fields +from datetime import timedelta +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple from shared.common.seal_criteria import CriterionStatus from tasks.seal_of_reliability.evaluators.base import CriterionObservation -from tasks.seal_of_reliability.state_machine import phase +from tasks.seal_of_reliability.state_machine import SealCriterionState, phase + +# Every field of the state a day leaves behind, minus the two that identify it — those are +# already on the row as `stable_id` and `criterion`. Taken from the dataclass rather than +# listed, so a field added to SealCriterionState shows up in the trace without touching +# this file, which is the same trick `seal_updater._snapshot_row` uses for the snapshots. +TRACED_STATE_FIELDS: Tuple[str, ...] = tuple( + f.name + for f in dataclass_fields(SealCriterionState) + if f.name not in ("feed_id", "criterion") +) + +# Fields that advance on their own inside a stretch where nothing actually happened, and so +# must not break a run when collapsing. `day` and `evaluated_at` move every day; +# `last_verdict_at` moves on every verdict; and during a confirmed failure streak +# `last_observed_failure_at`, `last_confirmed_failure_at` and `probation_start` are all +# re-stamped daily — probation_start to tomorrow, which is why the longest runs would never +# collapse if it counted. `reason` is prose and differs per simulated day. +# +# Everything else is part of the signature, so a field added to SealCriterionState breaks +# runs by default rather than being silently ignored: too eager beats invisible. +TICKING_FIELDS: frozenset = frozenset( + { + "day", + "evaluated_at", + "last_verdict_at", + "last_observed_failure_at", + "last_confirmed_failure_at", + "probation_start", + "reason", + } +) # Cap on the trace a single call returns. A year x a batch of feeds x six criteria would be # a response no one can read and Cloud Logging would drop; a simulation is a small thing. MAX_TRACE_ROWS: int = 2000 -# Payload keys inside a criterion that set policy rather than name days. Status names are a +# Payload keys inside a criterion that do something other than name days. Status names are a # closed set, so there is no collision. GRACE_KEY = "grace_days" PROBATION_KEY = "probation_days" -_POLICY_KEYS = (GRACE_KEY, PROBATION_KEY) +DEFAULT_KEY = "default" +_RESERVED_KEYS = (DEFAULT_KEY, GRACE_KEY, PROBATION_KEY) # Distinguishes "not given" — use the evaluator's own value — from an explicit null, which # means the criterion has no such period. @@ -49,23 +81,36 @@ @dataclass(frozen=True) class CriterionSimulation: - """What a payload forces for one criterion: its days, and optionally its policy. - - Overriding the periods is what makes a simulation informative for a criterion that has - none. Official is a point-in-time check with no grace and no probation, so a forced - failure confirms the same day and the trace shows nothing about debouncing. Given - `grace_days: 14` it behaves like the criteria still to be written, and the march can be - watched doing the thing a backfill exists to reconstruct. - - Only ever applied to a dry run, so no fabricated policy can reach the seal tables. + """What a payload sets for one criterion: a baseline, named days, and optionally policy. + + A scenario is usually "this criterion holds one status, except on these days". `default` + is that baseline, and the named days are the exceptions on top of it. Without a baseline + the unnamed days fall through to the real evaluator, which is only useful where the + source data says something: locally `fresh_coverage` has no dataset history to read and + every unnamed day comes back UNKNOWN, so a scenario built purely from exceptions never + leaves `never_evaluated`. + + Overriding the periods matters for a criterion that has none — `official` and `stable` + are point-in-time checks, so a forced failure confirms the same day and the trace shows + nothing about debouncing. `fresh_coverage` ships with 14 days of grace and 180 of + probation, so it needs no lending to exercise either. + + Only ever applied to a dry run, so no fabricated status or policy can reach the seal + tables. """ days: Mapping[int, CriterionStatus] = field(default_factory=dict) + baseline: Optional[CriterionStatus] = None grace_period: Optional[timedelta] = None probation_period: Optional[timedelta] = None grace_overridden: bool = False probation_overridden: bool = False + def status_on(self, offset: int) -> Optional[CriterionStatus]: + """The status this payload forces on `offset`, or None to ask the evaluator.""" + forced = self.days.get(offset) + return forced if forced is not None else self.baseline + def grace_for(self, evaluator) -> Optional[timedelta]: return self.grace_period if self.grace_overridden else evaluator.grace_period @@ -77,10 +122,18 @@ def probation_for(self, evaluator) -> Optional[timedelta]: ) def as_reported(self) -> dict: - """The echo returned in the report, so a run states what it was told to pretend.""" + """The echo returned in the report, so a run states what it was told to pretend. + + Offsets are stringified rather than left as ints: they share the dict with + `grace_days` and `probation_days`, and Flask's jsonify sorts keys, which raises on a + dict mixing str and int. JSON object keys are strings anyway, so the response shape + is unchanged. + """ echo: Dict[str, Any] = { - offset: status.value for offset, status in sorted(self.days.items()) + str(offset): status.value for offset, status in sorted(self.days.items()) } + if self.baseline is not None: + echo[DEFAULT_KEY] = self.baseline.value if self.grace_overridden: echo[GRACE_KEY] = ( self.grace_period.days if self.grace_period is not None else None @@ -127,12 +180,16 @@ def parse_simulation( Shape, offsets counted from each feed's own march start:: - {"official": {"grace_days": 14, "probation_days": 180, "fail": [3, 4], "unknown": [8]}} + {"fresh_coverage": {"default": "pass", "fail": [3, 4], "unknown": [8]}} Offsets rather than dates because a scenario is about the shape of a history — "it fails on day 3" — not about a calendar. Anchoring per feed rather than to the window start is what makes day 0 the feed's first evaluation, the one denied a grace period. + `default` is the status every unnamed day takes. Omit it and unnamed days fall through to + the real evaluator instead, which is what you want when the source data has something to + say and useless when it does not. + `grace_days` and `probation_days` are optional and override the criterion's own values for the run. Omit either to keep the evaluator's; pass null to mean it has none. """ @@ -157,13 +214,19 @@ def parse_simulation( by_status = dict(by_status or {}) grace = by_status.pop(GRACE_KEY, _UNSET) probation = by_status.pop(PROBATION_KEY, _UNSET) + baseline = by_status.pop(DEFAULT_KEY, None) + if baseline is not None and baseline not in forced_statuses: + raise ValueError( + f"Cannot simulate {DEFAULT_KEY} status {baseline!r} for {criterion!r}. " + f"Valid: {sorted(forced_statuses)}" + ) days: Dict[int, CriterionStatus] = {} for status, offsets in by_status.items(): if status not in forced_statuses: raise ValueError( f"Cannot simulate status {status!r} for {criterion!r}. Valid: " - f"{sorted(forced_statuses)} — plus {list(_POLICY_KEYS)}" + f"{sorted(forced_statuses)} — plus {list(_RESERVED_KEYS)}" ) for offset in offsets or []: offset = int(offset) @@ -181,6 +244,7 @@ def parse_simulation( parsed[criterion] = CriterionSimulation( days=days, + baseline=(CriterionStatus(baseline) if baseline is not None else None), grace_period=( None if grace is _UNSET else _parse_period(grace, criterion, GRACE_KEY) ), @@ -225,36 +289,105 @@ def check_simulation_fits( def observe(evaluator, ctx, simulation, offset: int) -> CriterionObservation: """The criterion's own verdict, unless this day is simulated. - Days the payload does not name fall through to the real evaluator, so a simulation is - real data with per-day overrides rather than a wholly synthetic run. + A named day wins over the baseline, and with neither the day falls through to the real + evaluator — so a simulation ranges from real data with a couple of overrides to a wholly + synthetic history, depending on whether `default` was given. """ entry = simulation.get(evaluator.name.value) - forced = entry.days.get(offset) if entry else None + forced = entry.status_on(offset) if entry else None if forced is None: return evaluator.evaluate(ctx) + named = entry.days.get(offset) is not None return CriterionObservation( criterion=evaluator.name, observed_status=forced, - reason=f"simulated: {forced.value} on day {offset}", + reason=( + f"simulated: {forced.value} on day {offset}" + if named + else f"simulated: {forced.value} by default" + ), ) -def trace_row(feed, evaluator, today: date, offset: int, observation, state) -> dict: - """One day of one criterion, as the state machine left it. +def _as_day(value): + """Render a state value for the trace: statuses as their name, timestamps as their day. + + The march evaluates at midnight UTC, so a date loses nothing and reads better than a + full timestamp repeated down a year of rows. + """ + if isinstance(value, CriterionStatus): + return value.value + if hasattr(value, "date"): + return value.date().isoformat() + return value + + +def trace_row(feed, evaluator, offset: int, observation, state) -> dict: + """One day of one criterion: every seal_criterion field, plus where it came from. + + Carries the whole state rather than a summary, so a trace answers the same questions the + stored row would — when the current streak began, when a verdict was last reached — and + a reader never has to run the march again to see a field that was left out. - Flask's jsonify sorts keys, so this order is for reading the source, not the response. + Flask's jsonify sorts keys, so the order here is for reading the source, not the + response. """ - return { + row = { "stable_id": feed.stable_id, "criterion": evaluator.name.value, "day": offset, - "date": today.isoformat(), - "observed_status": observation.observed_status.value, - "confirmed_status": state.confirmed_status.value, + # No `date`: `evaluated_at` is the same day by construction, since the march + # evaluates every criterion once per day and `transition` stamps it every time. "phase": phase(state).value, - "probation_start": ( - state.probation_start.date().isoformat() if state.probation_start else None - ), "simulated": observation.reason.startswith("simulated:"), "reason": observation.reason, } + for name in TRACED_STATE_FIELDS: + row[name] = _as_day(getattr(state, name)) + return row + + +def _signature(row: dict) -> tuple: + """What makes a day different from the one before it, ignoring the ticking fields.""" + return tuple( + sorted((key, value) for key, value in row.items() if key not in TICKING_FIELDS) + ) + + +def collapse_runs(rows: Sequence[dict]) -> List[dict]: + """Collapse consecutive days in which nothing changed into one entry per run. + + A year of trace is mostly repetition — a criterion sits in one situation for weeks. Each + run is reported as its first day, its last day, and how many days sat between them, so + the boundaries stay exact while the middle collapses. + + Rows are grouped by feed and criterion first: the march emits them day-major, so + consecutive entries in the flat list are different feeds, not consecutive days. + """ + grouped: Dict[Tuple[str, str], List[dict]] = {} + for row in rows: + grouped.setdefault((row["stable_id"], row["criterion"]), []).append(row) + + collapsed: List[dict] = [] + for series in grouped.values(): + series.sort(key=lambda row: row["day"]) + run: List[dict] = [] + for row in series: + if run and _signature(run[-1]) == _signature(row): + run.append(row) + continue + if run: + collapsed.append(_as_run(run)) + run = [row] + if run: + collapsed.append(_as_run(run)) + return collapsed + + +def _as_run(run: Sequence[dict]) -> dict: + """One unchanged stretch: its first day, its last, and the count between them.""" + entry = {"days": len(run), "first": run[0]} + if len(run) > 1: + entry["last"] = run[-1] + entry["in_between"] = len(run) - 2 + return entry diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py index 1d0db434c..581add788 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -20,6 +20,7 @@ a success that wrote nothing. """ +import json import unittest from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone @@ -389,6 +390,28 @@ def snapshot_days(stable_id, db_session=None): return sorted({row.snapshot_date for row in rows}) +def trace_day(report, day): + """The marched day `day`, found inside the collapsed trace. + + A trace reports one entry per unchanged stretch, so a day in the middle of one is + represented by that stretch's `first` row — every field asserted on here is part of the + signature the stretch collapsed on, and so is identical across it. The closing day carries + its own row, returned as `last`. + """ + for entry in report["trace"]: + first = entry["first"] + last = entry.get("last", first) + if first["day"] <= day <= last["day"]: + return last if day == last["day"] else first + raise AssertionError(f"day {day} is not in the trace") + + +def trace_last(report): + """The final marched day of the trace.""" + entry = report["trace"][-1] + return entry.get("last", entry["first"]) + + class MarchTestCase(unittest.TestCase): """Two identically-aged feeds: one marched in memory, one replayed through the database.""" @@ -609,7 +632,7 @@ def test_a_forced_failure_reaches_the_state_machine(self): report = self.simulate( MARCHED, simulate={"official": {"fail": [0]}}, trace=True ) - day_zero = next(row for row in report["trace"] if row["day"] == 0) + day_zero = trace_day(report, 0) self.assertEqual(day_zero["observed_status"], "fail") self.assertEqual(day_zero["confirmed_status"], "fail") self.assertTrue(day_zero["simulated"]) @@ -619,7 +642,7 @@ def test_unnamed_days_fall_through_to_the_real_evaluator(self): report = self.simulate( MARCHED, simulate={"official": {"fail": [0]}}, trace=True ) - day_one = next(row for row in report["trace"] if row["day"] == 1) + day_one = trace_day(report, 1) self.assertEqual(day_one["observed_status"], "pass") self.assertFalse(day_one["simulated"]) @@ -635,31 +658,89 @@ def test_a_streak_past_the_grace_period_confirms_then_serves_probation(self): simulate={"official": {"fail": list(range(1, 40))}}, trace=True, ) - by_day = {row["day"]: row for row in report["trace"]} - - self.assertEqual(by_day[30]["confirmed_status"], "pass", "last day of grace") - self.assertEqual(by_day[31]["confirmed_status"], "fail", "grace outlasted") + self.assertEqual( + trace_day(report, 30)["confirmed_status"], "pass", "last day of grace" + ) + self.assertEqual( + trace_day(report, 31)["confirmed_status"], "fail", "grace outlasted" + ) - last = report["trace"][-1] + last = trace_last(report) self.assertEqual(last["observed_status"], "pass") self.assertEqual(last["confirmed_status"], "pass", "recovered") self.assertEqual(last["phase"], "on_probation") self.assertIsNotNone(last["probation_start"]) - def test_the_trace_covers_every_marched_day(self): - report = self.simulate(MARCHED, trace=True) - days = [row["day"] for row in report["trace"]] - self.assertEqual(days, list(range(len(days)))) + def test_a_trace_row_carries_every_seal_criterion_field(self): + """The trace is the stored row plus provenance, not a summary of it. + + Derived from SealCriterionState, so a field added there appears here without anyone + remembering to widen the trace — and this fails if it ever stops being derived. + """ + from dataclasses import fields as dataclass_fields + + from tasks.seal_of_reliability.state_machine import SealCriterionState + + report = self.simulate( + MARCHED, simulate={"official": {"fail": [1]}}, trace=True + ) + row = trace_day(report, 0) + + expected = {f.name for f in dataclass_fields(SealCriterionState)} - { + "feed_id", + "criterion", + } + self.assertTrue( + expected.issubset(row), + f"trace is missing state fields: {sorted(expected - set(row))}", + ) + for name in ("day", "phase", "simulated", "reason", "criterion"): + self.assertIn(name, row) + self.assertNotIn( + "date", row, "dropped: evaluated_at is the same day by construction" + ) + + def test_the_carried_state_tracks_the_streak(self): + """first_observed_failure_at is set while failing and cleared on recovery.""" + report = self.simulate( + MARCHED, simulate={"official": {"fail": [1, 2]}}, trace=True + ) + days = {day: trace_day(report, day) for day in (0, 1, 2, 3)} + + self.assertIsNone(days[0]["first_observed_failure_at"]) + self.assertEqual(days[1]["first_observed_failure_at"], days[1]["evaluated_at"]) self.assertEqual( - len(days), (MARCH_END - MARCH_START).days + 1, "one row per day" + days[2]["first_observed_failure_at"], + days[1]["evaluated_at"], + "the streak keeps its start", ) + self.assertIsNone(days[3]["first_observed_failure_at"], "cleared on recovery") + self.assertEqual( + days[3]["last_observed_failure_at"], + days[2]["evaluated_at"], + "but the history is never cleared", + ) + + def test_the_trace_accounts_for_every_marched_day(self): + """Collapsed stretches tile the march: contiguous, in order, none missing.""" + report = self.simulate(MARCHED, trace=True) + marched = (MARCH_END - MARCH_START).days + 1 + + next_expected = 0 + for entry in report["trace"]: + first = entry["first"] + last = entry.get("last", first) + self.assertEqual(first["day"], next_expected, "stretches are contiguous") + self.assertEqual(entry["days"], last["day"] - first["day"] + 1) + next_expected = last["day"] + 1 + self.assertEqual(next_expected, marched, "every marched day is accounted for") def test_the_trace_is_offset_from_the_feed_own_march_start(self): """Day 0 is the feed's first evaluated day, not the window start.""" report = self.simulate(MARCHED, trace=True) - first = report["trace"][0] + first = trace_day(report, 0) self.assertEqual(first["day"], 0) - self.assertEqual(first["date"], MARCH_START.isoformat()) + self.assertEqual(first["evaluated_at"], MARCH_START.isoformat()) def test_an_unknown_criterion_is_rejected(self): with self.assertRaises(ValueError) as caught: @@ -686,7 +767,9 @@ def test_the_report_echoes_what_was_simulated(self): report = self.simulate( MARCHED, simulate={"official": {"fail": [2], "unknown": [4]}} ) - self.assertEqual(report["simulated"]["official"], {2: "fail", 4: "unknown"}) + # String keys: the echo shares its dict with grace_days/probation_days, and jsonify + # sorts keys, which raises on a dict mixing str and int. + self.assertEqual(report["simulated"]["official"], {"2": "fail", "4": "unknown"}) def test_a_plain_dry_run_still_stops_at_the_plan(self): """Only simulate or trace makes a dry run pay for the march.""" @@ -715,7 +798,7 @@ def simulate(self, **kwargs): @staticmethod def _day(report, day): - return next(row for row in report["trace"] if row["day"] == day) + return trace_day(report, day) def test_a_lent_grace_period_absorbs_a_failure(self): """Without an override this criterion confirms on day 1; with 14 days it holds.""" @@ -795,7 +878,10 @@ def test_the_report_echoes_the_lent_policy(self): echoed = report["simulated"]["official"] self.assertEqual(echoed["grace_days"], 14) self.assertEqual(echoed["probation_days"], 180) - self.assertEqual(echoed[1], "fail") + self.assertEqual(echoed["1"], "fail") + # This echo is the shape that used to 500: offsets and policy keys in one dict, which + # only fails once Flask serializes it, so assert it survives that too. + self.assertEqual(json.loads(json.dumps(echoed, sort_keys=True)), echoed) def test_a_negative_period_is_rejected(self): with self.assertRaises(ValueError) as caught: @@ -820,5 +906,60 @@ def test_a_lent_policy_never_writes(self): ) +class TestSimulatedBaseline(MarchTestCase): + """`default` is what every unnamed day observes; named days are exceptions on top. + + Without it, unnamed days fall through to the evaluator — which says nothing useful for a + criterion whose source data is absent, as `fresh_coverage` is on a local database. + """ + + def simulate(self, **kwargs): + # The stand-in passes every day, so a `fail` baseline can only come from the payload. + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=True, + trace=True, + **kwargs, + ) + + def test_a_baseline_replaces_the_evaluator_on_every_unnamed_day(self): + report = self.simulate(simulate={"official": {"default": "fail"}}) + for day in (0, 1, 5): + row = trace_day(report, day) + self.assertEqual(row["observed_status"], "fail", f"day {day}") + self.assertTrue(row["simulated"]) + self.assertIn("by default", row["reason"]) + + def test_a_named_day_overrides_the_baseline(self): + report = self.simulate(simulate={"official": {"default": "fail", "pass": [2]}}) + self.assertEqual(trace_day(report, 1)["observed_status"], "fail") + day_two = trace_day(report, 2) + self.assertEqual(day_two["observed_status"], "pass", "the exception wins") + self.assertIn("on day 2", day_two["reason"]) + + def test_without_a_baseline_unnamed_days_still_fall_through(self): + report = self.simulate(simulate={"official": {"fail": [2]}}) + self.assertFalse(trace_day(report, 1)["simulated"], "the evaluator answered") + + def test_the_report_echoes_the_baseline(self): + report = self.simulate(simulate={"official": {"default": "fail", "pass": [2]}}) + echoed = report["simulated"]["official"] + self.assertEqual(echoed["default"], "fail") + self.assertEqual(echoed["2"], "pass") + + def test_an_unknown_baseline_status_is_rejected(self): + with self.assertRaises(ValueError) as caught: + self.simulate(simulate={"official": {"default": "excellent"}}) + self.assertIn("default", str(caught.exception)) + + def test_a_baseline_alone_is_enough_to_march(self): + """No named day, so nothing to range-check — the baseline still forces the march.""" + report = self.simulate(simulate={"official": {"default": "unknown"}}) + self.assertEqual(trace_day(report, 0)["observed_status"], "unknown") + + if __name__ == "__main__": unittest.main() From f689569a97ee55d1b60968cb4e0d2ce915edb918 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 28 Aug 2026 11:34:52 -0400 Subject: [PATCH 12/13] Corrected a problem with UNKNOWN and the grace period. --- .../seal_of_reliability/state_machine.py | 49 ++++++++--- .../test_seal_state_machine.py | 81 +++++++++++++++++++ 2 files changed, 121 insertions(+), 9 deletions(-) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py index 8a3a1405f..a39f81e1d 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/state_machine.py @@ -30,6 +30,10 @@ privilege, and a criterion serving a penalty for an earlier confirmed failure has forfeited it. That coupling is what makes IN_GRACE_PERIOD and ON_PROBATION mutually exclusive. +The grace period runs from the day the streak began, not in days the criterion was seen +failing, so it can expire on a day with no reading. It is confirmed then regardless: the +evidence is the failures already observed. + The seal (step 5, in seal_updater) requires every criterion in service to be a confirmed pass and not on probation. """ @@ -205,19 +209,46 @@ def transition( observed_status = observation.observed_status if not observed_status.is_verdict: - # UNKNOWN keeps the stored confirmed_status so the criterion stays in the roll-up - # with its last verdict; NOT_APPLICABLE overwrites it so the criterion leaves the - # roll-up. Neither touches probation or the failure timestamps, and neither moves - # last_verdict_at — no verdict was produced. - confirmed_status = ( - CriterionStatus.NOT_APPLICABLE - if observed_status is CriterionStatus.NOT_APPLICABLE - else base.confirmed_status + # NOT_APPLICABLE leaves the roll-up. Its penalty survives, should it come back. + if observed_status is CriterionStatus.NOT_APPLICABLE: + return replace( + base, + observed_status=observed_status, + confirmed_status=CriterionStatus.NOT_APPLICABLE, + evaluated_at=now, + ) + + # UNKNOWN keeps the stored verdict, unless the streak has already outlived its + # grace: today's missing reading does not undo the failures behind it. Only the first + # such day confirms, or probation would advance on days nobody measured. + outlived_grace = ( + grace_period is not None + and base.confirmed_status is not CriterionStatus.FAIL + and base.first_observed_failure_at is not None + and base.last_verdict_at is not None + and now - base.first_observed_failure_at >= grace_period ) + if outlived_grace: + # Stamped at the last observed failure, not today: today has nothing to point at. + last_seen = base.last_observed_failure_at + return replace( + base, + observed_status=observed_status, + confirmed_status=CriterionStatus.FAIL, + evaluated_at=now, + last_confirmed_failure_at=last_seen, + probation_start=( + _next_day_start(last_seen) + if probation_period is not None + else base.probation_start + ), + ) + + # No branch here moves last_verdict_at: the check ran, but returned no verdict. return replace( base, observed_status=observed_status, - confirmed_status=confirmed_status, + confirmed_status=base.confirmed_status, evaluated_at=now, ) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py index e70838347..08204ce93 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/test_seal_state_machine.py @@ -450,5 +450,86 @@ def test_is_verdict_only_covers_pass_and_fail(self): self.assertFalse(status.is_verdict, status) +class TestGraceExpiresOnAnUnknownDay(unittest.TestCase): + """A streak whose grace runs out on a day that produced no reading. + + Found by marching a feed locally: fourteen failures, UNKNOWN across the day grace + expired, then a pass — and nothing was confirmed, so the outage left no trace. + """ + + def _grace_expired_under_unknowns(self): + # Grace runs 14 days from the streak start on day 1, so day 15 both expires it and + # is the first day with no reading. + return _run( + [(0, PASS)] + + _failing(1, 15) + + [ + (15, CriterionStatus.UNKNOWN), + (16, CriterionStatus.UNKNOWN), + (17, CriterionStatus.UNKNOWN), + ] + ) + + def test_the_streak_start_survives_the_unknown_days(self): + """The unknowns neither reset nor forget the streak.""" + state = self._grace_expired_under_unknowns() + self.assertEqual(state.first_observed_failure_at, _day(1)) + self.assertEqual( + state.last_verdict_at, + _day(14), + "no verdict since the last observed failure", + ) + + def test_a_streak_past_its_grace_confirms_without_a_fresh_verdict(self): + state = self._grace_expired_under_unknowns() + self.assertIs( + state.confirmed_status, + FAIL, + "17 days into a streak whose grace expired on day 15", + ) + + def test_a_pass_cannot_forgive_a_streak_that_outlived_its_grace(self): + """Otherwise one pass after the unknowns erases the whole outage.""" + recovered = _run([(18, PASS)], state=self._grace_expired_under_unknowns()) + self.assertIsNotNone( + recovered.last_confirmed_failure_at, + "a fortnight of failure left no record at all", + ) + + +class TestProbationAcrossAnUnknownDay(unittest.TestCase): + """`probation_start` when a confirmed failure is followed by no reading. + + A confirmed streak re-stamps it to the following day. An UNKNOWN day leaves it alone, + which makes that day the first of probation and counts it toward the term. + """ + + def _confirmed_then_unknown(self): + streaking = _run([(0, PASS)] + _failing(1, 17)) + return streaking, _run([(17, CriterionStatus.UNKNOWN)], state=streaking) + + def test_the_streak_leaves_probation_stamped_for_the_following_day(self): + streaking, _ = self._confirmed_then_unknown() + self.assertIs(streaking.confirmed_status, FAIL) + self.assertEqual(streaking.probation_start, _day(17)) + + def test_an_unknown_day_does_not_push_probation_forward(self): + streaking, after = self._confirmed_then_unknown() + self.assertEqual( + after.probation_start, + streaking.probation_start, + "no verdict re-stamps it, so probation begins on the unknown day", + ) + self.assertIs(after.confirmed_status, FAIL, "the last verdict still stands") + self.assertIs(phase(after), CriterionPhase.ON_PROBATION) + + def test_the_unknown_day_counts_toward_serving_probation(self): + """A day with no reading still serves the term.""" + _, after = self._confirmed_then_unknown() + served = _run([(17 + PROBATION_PERIOD.days, PASS)], state=after) + self.assertIsNone(served.probation_start, "the term was served") + self.assertIs(phase(served), CriterionPhase.STEADY) + + if __name__ == "__main__": unittest.main() From 803efc5c4039ece4c03e15da2797545e80f739d4 Mon Sep 17 00:00:00 2001 From: jcpitre Date: Fri, 28 Aug 2026 11:41:19 -0400 Subject: [PATCH 13/13] Allow simulated writes outside production --- .../backfill/backfill_seal_of_reliability.py | 4 +- .../backfill/seal_backfill.py | 13 ++-- .../backfill/simulation.py | 76 ++++++++++++++++++- .../backfill/test_seal_backfill.py | 65 ++++++++++++---- 4 files changed, 133 insertions(+), 25 deletions(-) diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py index c46d391d7..2fa7c8d82 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/backfill_seal_of_reliability.py @@ -90,7 +90,9 @@ def backfill_seal_of_reliability_handler(payload: dict) -> dict: max_reported_feeds cap on the `feeds` list in the response. Default: 50 simulate force statuses per criterion, on days counted from each feed's march start: {"fresh_coverage": {"default": "pass", "fail": [3]}}. - Requires dry_run; see `parse_simulation` for the full shape + see `parse_simulation` for the full shape. Combining it with + dry_run=false writes fabricated verdicts, which is refused in + production and on production's tunnel port trace return the march day by day: every seal_criterion field, plus where it came from. Marches without writing when dry_run. Consecutive days in which nothing changed are always collapsed into one entry — its diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py index 354d1057c..722f0f2a5 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/seal_backfill.py @@ -50,6 +50,7 @@ ) from tasks.seal_of_reliability.backfill.simulation import ( MAX_TRACE_ROWS, + check_simulated_write_allowed, check_simulation_fits, collapse_runs, observe, @@ -464,13 +465,7 @@ def backfill_seals( if not stable_feed_ids: raise ValueError("stable_feed_ids is required and must be non-empty") if simulate and not dry_run: - # A simulated verdict written to seal_criterion is indistinguishable from an earned - # one — the row carries no provenance. Refuse rather than quietly downgrade, so an - # operator cannot believe a real run happened. - raise ValueError( - "simulate requires dry_run: forced verdicts must never be written to the seal " - "tables, where nothing would mark them as simulated" - ) + check_simulated_write_allowed(simulate, db_session) if snapshot_mode not in SNAPSHOT_MODES: raise ValueError( f"Unknown snapshot_mode {snapshot_mode!r}. Known modes: {list(SNAPSHOT_MODES)}" @@ -588,6 +583,10 @@ def backfill_seals( criterion: forced.as_reported() for criterion, forced in simulation.items() } + if not dry_run: + # The only provenance that exists: the response says the rows are fabricated, + # because the rows themselves cannot. + report["simulated_write"] = True granted = [outcome for outcome in outcomes if outcome["granted"]] # Only reachable with only_missing=False, but reported anyway so the monitor's diff --git a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py index 21ed19d73..4fdcbede8 100644 --- a/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py +++ b/functions-python/tasks_executor/src/tasks/seal_of_reliability/backfill/simulation.py @@ -15,23 +15,32 @@ # """Forced per-day statuses and the day-by-day trace, for inspecting a backfill march. -Neither may write. A simulated verdict in `seal_criterion` would be indistinguishable from -an earned one — the row carries no provenance — so `backfill_seals` refuses to combine -`simulate` with a real run, and a traced dry run marches with writing suppressed. +A simulated verdict in `seal_criterion` would be indistinguishable from an earned one — the +row carries no provenance — so writing one is refused by default, and a traced dry run +marches with writing suppressed. `check_simulated_write_allowed` holds the exception and its +conditions: an explicit payload flag, an environment on the allowlist, and not the production +tunnel port. Everything that decides whether a fabricated status may reach the tables lives in +this module, so the rule can be read in one place. Day offsets are counted from each feed's own march start, so day 0 is that feed's first evaluated day: the one denied a grace period. Anchoring to the run's `start_date` instead would point at days a younger feed never marched. """ +import logging +import os from dataclasses import dataclass, field, fields as dataclass_fields from datetime import timedelta from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple +from sqlalchemy.orm import Session + from shared.common.seal_criteria import CriterionStatus from tasks.seal_of_reliability.evaluators.base import CriterionObservation from tasks.seal_of_reliability.state_machine import SealCriterionState, phase +logger = logging.getLogger(__name__) + # Every field of the state a day leaves behind, minus the two that identify it — those are # already on the row as `stable_id` and `criterion`. Taken from the dataclass rather than # listed, so a field added to SealCriterionState shows up in the trace without touching @@ -391,3 +400,64 @@ def _as_run(run: Sequence[dict]) -> dict: entry["last"] = run[-1] entry["in_between"] = len(run) - 2 return entry + + +# Environments in which a forced verdict may be written to the seal tables. Deliberately a +# closed list rather than "anything but prod": an ENVIRONMENT that is unset or misspelled +# refuses, so a deployment that forgets to set it cannot fabricate seal history. dev and qa are +# both in, by decision — they are where scenarios get exercised. Note dev and qa share one +# Cloud SQL instance, so a simulated write in either is one database name away from the other's +# data; the response flag and the warning log are the only marks a fabricated row leaves. +SIMULATED_WRITE_ENVIRONMENTS: Tuple[str, ...] = ("local", "dev", "qa", "test") + +# The local port production is reached on when a tunnel is up, by team convention. Refusing it +# is a guard rail, not a guarantee: the port is a property of how the tunnel was started rather +# than of the database, so a tunnel opened on another port walks straight past this. It is here +# because ENVIRONMENT describes the process, not the write target — a local run labelled `local` +# can be pointed at production through a tunnel and would otherwise pass every other check. The +# actual control is connecting to production as a read-only user. +PROD_TUNNEL_PORT: int = 9901 + + +def _connection_port(db_session: Session) -> Optional[int]: + """The port this session is connected on, or None if it cannot be determined.""" + try: + return db_session.get_bind().url.port + except Exception: # defensive: never let the guard rail itself break a run + logger.warning( + "Could not determine the database port for the simulated-write check" + ) + return None + + +def check_simulated_write_allowed(simulate: dict, db_session: Session) -> None: + """Refuse to write forced verdicts anywhere they could be mistaken for real history. + + A simulated verdict in `seal_criterion` is indistinguishable from an earned one — the row + carries no provenance — so where it may be written is decided here, by the deployment + rather than by the payload. Two conditions, both about the target: the environment has to + be one where fabricated data is expected, and the connection must not be production's + tunnel port. + """ + environment = os.getenv("ENVIRONMENT", "").strip().lower() + if environment not in SIMULATED_WRITE_ENVIRONMENTS: + raise ValueError( + f"a simulated write is refused with ENVIRONMENT={environment or 'unset'!r}: " + f"forced verdicts may only be written in {list(SIMULATED_WRITE_ENVIRONMENTS)}, " + f"and an unset environment is treated as production." + ) + + port = _connection_port(db_session) + if port == PROD_TUNNEL_PORT: + raise ValueError( + f"a simulated write is refused on port {PROD_TUNNEL_PORT}: that is the " + f"production database's tunnel port, whatever ENVIRONMENT={environment!r} claims." + ) + + logger.warning( + "SIMULATED WRITE in ENVIRONMENT=%s on port %s: forced statuses for %s are being " + "written to the seal tables. These rows are indistinguishable from earned ones.", + environment, + port, + sorted(simulate), + ) diff --git a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py index 581add788..b5a7bb15d 100644 --- a/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py +++ b/functions-python/tasks_executor/tests/tasks/seal_of_reliability/backfill/test_seal_backfill.py @@ -21,6 +21,7 @@ """ import json +import os import unittest from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone @@ -615,17 +616,51 @@ def test_a_simulated_run_never_writes(self): self.assertIsNone(seal_row(MARCHED)) self.assertEqual(report["criterion_rows_written"], 0) - def test_writing_with_a_simulation_is_refused(self): - with self.assertRaises(ValueError) as caught: - with self.registry(_script_for([])): - backfill_seals( - stable_feed_ids=[MARCHED], - start_date=MARCH_START, - end_date=MARCH_END, - dry_run=False, - simulate={"official": {"fail": [0]}}, - ) - self.assertIn("dry_run", str(caught.exception)) + def _simulated_write(self, **kwargs): + with self.registry(_script_for([])): + return backfill_seals( + stable_feed_ids=[MARCHED], + start_date=MARCH_START, + end_date=MARCH_END, + dry_run=False, + only_missing=False, + simulate={"official": {"fail": [0]}}, + **kwargs, + ) + + def test_a_simulated_write_is_allowed_in_dev_with_the_flag(self): + """The override exists so a debounced state can be written and read back locally.""" + with patch.dict(os.environ, {"ENVIRONMENT": "dev"}): + report = self._simulated_write() + self.assertFalse(report["dry_run"]) + self.assertTrue( + report["simulated_write"], "the response is the only provenance there is" + ) + + def test_a_simulated_write_is_refused_in_production(self): + with patch.dict(os.environ, {"ENVIRONMENT": "prod"}): + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("prod", str(caught.exception)) + + def test_a_simulated_write_is_refused_on_the_prod_tunnel_port(self): + """ENVIRONMENT describes the process; the port is the only hint about the target.""" + with patch.dict(os.environ, {"ENVIRONMENT": "local"}): + with patch( + "tasks.seal_of_reliability.backfill.simulation._connection_port", + return_value=9901, + ): + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("9901", str(caught.exception)) + + def test_an_unset_environment_is_treated_as_production(self): + """Fail closed: a deployment that forgets ENVIRONMENT must not fabricate history.""" + with patch.dict(os.environ): + os.environ.pop("ENVIRONMENT", None) + with self.assertRaises(ValueError) as caught: + self._simulated_write() + self.assertIn("unset", str(caught.exception)) def test_a_forced_failure_reaches_the_state_machine(self): """Day 0 is the first evaluation, so it gets no grace and confirms immediately.""" @@ -893,9 +928,11 @@ def test_a_non_numeric_period_is_rejected(self): self.simulate(simulate={"official": {"probation_days": "a fortnight"}}) self.assertIn("probation_days", str(caught.exception)) - def test_a_lent_policy_never_writes(self): - """Same rule as forced verdicts: a fabricated policy must not reach the tables.""" - with self.assertRaises(ValueError): + def test_a_lent_policy_never_writes_in_production(self): + """Same rule as forced verdicts: a fabricated policy must not reach real tables.""" + with patch.dict(os.environ, {"ENVIRONMENT": "prod"}), self.assertRaises( + ValueError + ): with self.registry(_script_for([])): backfill_seals( stable_feed_ids=[MARCHED],