diff --git a/api/.openapi-generator/FILES b/api/.openapi-generator/FILES index 74129a6b5..58295cb25 100644 --- a/api/.openapi-generator/FILES +++ b/api/.openapi-generator/FILES @@ -34,6 +34,9 @@ src/feeds_gen/models/gtfs_dataset.py src/feeds_gen/models/gtfs_feed.py src/feeds_gen/models/gtfs_feed_availability_check.py src/feeds_gen/models/gtfs_feed_availability_response.py +src/feeds_gen/models/gtfs_feed_continuous_coverage.py +src/feeds_gen/models/gtfs_feed_continuous_coverage_file.py +src/feeds_gen/models/gtfs_feed_continuous_coverage_response.py src/feeds_gen/models/gtfs_rt_feed.py src/feeds_gen/models/latest_dataset.py src/feeds_gen/models/latest_dataset_validation_report.py @@ -49,6 +52,7 @@ src/feeds_gen/models/redirect.py src/feeds_gen/models/reliability_criterion.py src/feeds_gen/models/search_feed_item_result.py src/feeds_gen/models/search_feeds200_response.py +src/feeds_gen/models/service_date_window.py src/feeds_gen/models/source_info.py src/feeds_gen/models/validation_report.py src/feeds_gen/security_api.py diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py index 62c76ad52..65341158e 100644 --- a/api/src/feeds/impl/feeds_api_impl.py +++ b/api/src/feeds/impl/feeds_api_impl.py @@ -1,7 +1,6 @@ -from datetime import datetime from typing import List, Union, TypeVar, Optional -from sqlalchemy import or_ +from sqlalchemy import or_, desc, nullslast from sqlalchemy.orm import contains_eager, selectinload, Session from sqlalchemy.orm.query import Query @@ -11,6 +10,7 @@ from shared.db_models.feed_reliability_report_impl import FeedReliabilityReportImpl from shared.db_models.gbfs_feed_impl import GbfsFeedImpl from shared.db_models.gtfs_feed_availability_check_impl import GtfsFeedAvailabilityCheckImpl +from shared.db_models.gtfs_feed_continuous_coverage_impl import GtfsFeedContinuousCoverageImpl from shared.db_models.gtfs_feed_impl import GtfsFeedImpl from shared.db_models.gtfs_rt_feed_impl import GtfsRTFeedImpl from feeds_gen.apis.feeds_api_base import BaseFeedsApi @@ -20,8 +20,12 @@ from feeds_gen.models.gtfs_dataset import GtfsDataset from feeds_gen.models.gtfs_feed import GtfsFeed from feeds_gen.models.gtfs_feed_availability_response import GtfsFeedAvailabilityResponse +from feeds_gen.models.gtfs_feed_continuous_coverage import GtfsFeedContinuousCoverage +from feeds_gen.models.gtfs_feed_continuous_coverage_file import GtfsFeedContinuousCoverageFile +from feeds_gen.models.gtfs_feed_continuous_coverage_response import GtfsFeedContinuousCoverageResponse from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed from middleware.request_context import is_user_email_restricted +from shared.common.continuous_coverage import COVERAGE_FILES from shared.common.db_utils import ( get_gtfs_feeds_query, get_gtfs_rt_feeds_query, @@ -31,6 +35,7 @@ ) from shared.common.error_handling import ( availability_from_after_to, + continuous_coverage_downloaded_after_before, invalid_date_message, feed_not_found, gtfs_feed_not_found, @@ -49,7 +54,7 @@ from shared.feed_filters.feed_filter import FeedFilter from shared.feed_filters.gtfs_dataset_filter import GtfsDatasetFilter from shared.feed_filters.gtfs_rt_feed_filter import GtfsRtFeedFilter -from utils.date_utils import valid_iso_date +from utils.date_utils import parse_iso_datetime, valid_iso_date from utils.logger import get_logger T = TypeVar("T", bound="Feed") @@ -177,15 +182,9 @@ def get_gtfs_feed_datasets( if not feed: raise_http_error(404, f"FeedOrm with id {gtfs_feed_id} not found") - # Replace Z with +00:00 to make the datetime object timezone aware - # Due to https://github.com/python/cpython/issues/80010, once migrate to Python 3.11, we can use fromisoformat query = GtfsDatasetFilter( - downloaded_at__lte=( - datetime.fromisoformat(downloaded_before.replace("Z", "+00:00")) if downloaded_before else None - ), - downloaded_at__gte=( - datetime.fromisoformat(downloaded_after.replace("Z", "+00:00")) if downloaded_after else None - ), + downloaded_at__lte=parse_iso_datetime(downloaded_before), + downloaded_at__gte=parse_iso_datetime(downloaded_after), ).filter(DatasetsApiImpl.create_dataset_query().filter(FeedOrm.stable_id == gtfs_feed_id)) if latest: @@ -348,8 +347,8 @@ def get_gtfs_feed_availability( if to and not valid_iso_date(to): raise_http_validation_error(invalid_date_message.format("to")) - from_dt = datetime.fromisoformat(_from.replace("Z", "+00:00")) if _from else None - to_dt = datetime.fromisoformat(to.replace("Z", "+00:00")) if to else None + from_dt = parse_iso_datetime(_from) + to_dt = parse_iso_datetime(to) if from_dt and to_dt and from_dt > to_dt: raise_http_validation_error(availability_from_after_to) @@ -378,6 +377,154 @@ def get_gtfs_feed_availability( checks=[GtfsFeedAvailabilityCheckImpl.from_orm(c) for c in checks], ) + @with_db_session + def get_gtfs_feed_continuous_coverage( + self, + id: str, + downloaded_after: str, + downloaded_before: str, + limit: int, + offset: int, + db_session: Session, + ) -> GtfsFeedContinuousCoverageResponse: + """Returns the continuous coverage history for a GTFS feed.""" + if downloaded_after and not valid_iso_date(downloaded_after): + raise_http_validation_error(invalid_date_message.format("downloaded_after")) + if downloaded_before and not valid_iso_date(downloaded_before): + raise_http_validation_error(invalid_date_message.format("downloaded_before")) + + after_dt = parse_iso_datetime(downloaded_after) + before_dt = parse_iso_datetime(downloaded_before) + + if after_dt and before_dt and after_dt > before_dt: + raise_http_validation_error(continuous_coverage_downloaded_after_before) + + feed = self._get_gtfs_feed(id, db_session, include_options_for_joinedload=False) + if not feed: + raise_http_error(404, gtfs_feed_not_found.format(id)) + + # Kept separate from the filtered query: the dataset immediately older than the page is + # looked up here, and it may well be one the date filters excluded. + feed_datasets = db_session.query(Gtfsdataset).filter(Gtfsdataset.feed_id == feed.id) + + query = feed_datasets + if after_dt: + query = query.filter(Gtfsdataset.downloaded_at >= after_dt) + if before_dt: + query = query.filter(Gtfsdataset.downloaded_at <= before_dt) + + total = query.count() + page = ( + query.order_by(*self._continuous_coverage_order()) + .offset(offset) + .limit(limit) + .options(selectinload(Gtfsdataset.feed_info), selectinload(Gtfsdataset.gtfsfiles)) + .all() + ) + + # Each item's overlap is measured against the dataset downloaded just before it. Within the + # page that is usually the next item, but the oldest item's neighbour lies outside the page, + # and any item bordering a null-`downloaded_at` run can't trust its positional neighbour + # either - `_predecessor` falls back to a real lookup in both cases. + predecessors = [ + self._predecessor(feed_datasets, dataset, next_dataset) + for dataset, next_dataset in zip(page, page[1:] + [None]) + ] + + latest_coverage = self._latest_continuous_coverage(feed, feed_datasets) + + return GtfsFeedContinuousCoverageResponse( + feed_id=id, + total=total, + offset=offset, + limit=limit, + latest_files=( + latest_coverage.files + if latest_coverage + else [GtfsFeedContinuousCoverageFile(name=name, present=False) for name in COVERAGE_FILES] + ), + latest_coverage_window=latest_coverage.coverage_window if latest_coverage else None, + latest_coverage_window_source=latest_coverage.coverage_window_source if latest_coverage else None, + latest_within_max_coverage_window=(latest_coverage.within_max_coverage_window if latest_coverage else None), + latest_service_window=latest_coverage.service_window if latest_coverage else None, + latest_feed_info_window=latest_coverage.feed_info_window if latest_coverage else None, + latest_feed_info_matches=latest_coverage.feed_info_matches if latest_coverage else None, + latest_overlap_days=latest_coverage.overlap_days if latest_coverage else None, + latest_gap_days=latest_coverage.gap_days if latest_coverage else None, + items=[ + GtfsFeedContinuousCoverageImpl.from_orm( + dataset, + previous_dataset=previous, + is_latest=dataset.id == feed.latest_dataset_id, + ) + for dataset, previous in zip(page, predecessors) + ], + ) + + @staticmethod + def _latest_continuous_coverage(feed: Gtfsfeed, feed_datasets: Query) -> Optional[GtfsFeedContinuousCoverage]: + """The coverage snapshot for the feed's latest dataset, independent of the requested page or + date filters - the root `latest_*` response fields always describe this dataset, even when it + falls outside the current page or date range. + """ + if feed.latest_dataset_id is None: + return None + latest_dataset = ( + feed_datasets.filter(Gtfsdataset.id == feed.latest_dataset_id) + .options(selectinload(Gtfsdataset.feed_info), selectinload(Gtfsdataset.gtfsfiles)) + .first() + ) + if latest_dataset is None: + return None + previous = FeedsApiImpl._previous_dataset(feed_datasets, latest_dataset) + return GtfsFeedContinuousCoverageImpl.from_orm(latest_dataset, previous_dataset=previous, is_latest=True) + + @staticmethod + def _continuous_coverage_order() -> tuple: + """Newest dataset first, with a deterministic tiebreak. + + `downloaded_at` is nullable, and a dataset with no download timestamp cannot be placed in a + chronological chain at all, so those sort last rather than ahead of everything. `stable_id` + breaks ties so that paging over datasets sharing a timestamp cannot repeat or skip a row. + """ + return nullslast(desc(Gtfsdataset.downloaded_at)), desc(Gtfsdataset.stable_id) + + @staticmethod + def _previous_dataset(feed_datasets: Query, dataset: Gtfsdataset) -> Optional[Gtfsdataset]: + """The feed's dataset downloaded immediately before `dataset`, ignoring any date filter. + + Returns None for a dataset with no download timestamp: there is no "before" to look for, and + ordering it against timestamped datasets would invent a neighbour. + """ + if dataset.downloaded_at is None: + return None + return ( + feed_datasets.filter(Gtfsdataset.downloaded_at < dataset.downloaded_at) + .order_by(*FeedsApiImpl._continuous_coverage_order()) + .options(selectinload(Gtfsdataset.feed_info)) + .first() + ) + + @staticmethod + def _predecessor( + feed_datasets: Query, dataset: Gtfsdataset, positional_next: Optional[Gtfsdataset] + ) -> Optional[Gtfsdataset]: + """The predecessor to use for one page row. + + The next row in the page is a valid predecessor only when both it and `dataset` have a + real `downloaded_at` - `_continuous_coverage_order`'s `nullslast` sorts undated datasets to + the end of the page, so a positional neighbour next to one of them is not necessarily who + was downloaded immediately before it. Falling back to `_previous_dataset` covers that case + and the page's actual last row (`positional_next is None`) alike. + """ + if ( + dataset.downloaded_at is not None + and positional_next is not None + and positional_next.downloaded_at is not None + ): + return positional_next + return FeedsApiImpl._previous_dataset(feed_datasets, dataset) + @with_db_session def get_gbfs_feed( self, diff --git a/api/src/shared/common/continuous_coverage.py b/api/src/shared/common/continuous_coverage.py new file mode 100644 index 000000000..c7c6b4317 --- /dev/null +++ b/api/src/shared/common/continuous_coverage.py @@ -0,0 +1,96 @@ +"""Continuous coverage policy values and the arithmetic that applies them. + +The maximum coverage window and the list of files the calculation reads are the published +definition of continuous coverage, so - like the rest of the seal policy in +`shared.common.seal_criteria` - they live in code rather than in DB config: changing either +changes which feeds qualify, and that should go through code review, tests and a deploy. + +This module is under `shared/` so that the read API and the nightly `fresh_continuous` +evaluator (`functions-python/tasks_executor`, which symlinks `api/src/shared/*`) can agree on +what "successive datasets overlap" means. The API computes these values on the fly from the +datasets it already stores; nothing here is persisted. + +See #1761 for the seal algorithm and MobilityData/product-tasks#215 for the endpoint. +""" + +import datetime +from datetime import timedelta +from typing import Final, Optional, Tuple + +# The longest service window the seal accepts. A dataset declaring service further out than +# this is not evidence of continuous coverage - it is more likely a placeholder calendar. +MAX_COVERAGE_WINDOW: Final[timedelta] = timedelta(days=730) + +# The files the calculation reads, in the order they are reported. `feed_info.txt` supplies the +# declared window; the two calendar files are what the validator derives the service window +# from. Order is fixed so a client can render one row of chips per dataset without sorting. +COVERAGE_FILES: Final[Tuple[str, ...]] = ("feed_info.txt", "calendar.txt", "calendar_dates.txt") + +# Which input a coverage window was taken from. Values match the `coverage_window_source` enum +# in the API spec. +SOURCE_SERVICE_DATES: Final[str] = "service_dates" +SOURCE_FEED_INFO: Final[str] = "feed_info" + + +def as_date(value: Optional[datetime.date | datetime.datetime]) -> Optional[datetime.date]: + """Reduce a stored value to a plain date. + + The validated service dates are stored as timestamps and the `feed_info.txt` dates as dates, + but both describe whole service days. Comparing them as timestamps would make a window that + starts at midnight look like it starts before one stored as a date, so both are narrowed here + before any arithmetic. + """ + if value is None: + return None + return value.date() if isinstance(value, datetime.datetime) else value + + +def window_days(start: Optional[datetime.date], end: Optional[datetime.date]) -> Optional[int]: + """Length of a closed date window, counting both bounds. + + A window that starts and ends on the same day covers one day, not zero - the bounds are + service dates, not instants. Returns None for an incomplete window and for an inverted one: + a producer whose end date precedes its start date has no window to measure, and reporting a + negative length would read as a real measurement. + """ + if start is None or end is None or end < start: + return None + return (end - start).days + 1 + + +def within_max_coverage_window(start: Optional[datetime.date], end: Optional[datetime.date]) -> Optional[bool]: + """Whether a window stays inside `MAX_COVERAGE_WINDOW`. + + Returns None when there is no window to measure, which a client must not read as passing. + """ + days = window_days(start, end) + if days is None: + return None + return end - start <= MAX_COVERAGE_WINDOW + + +def overlap_and_gap( + older_end: Optional[datetime.date], newer_start: Optional[datetime.date] +) -> Tuple[Optional[int], Optional[int]]: + """How a dataset's window meets the window of the dataset downloaded just before it. + + Returns `(overlap_days, gap_days)`, of which at most one is ever set: + + * The windows overlap - the newer one starts on or before the older one ends - so the shared + days are returned as `overlap_days`, counting both bounds. An older window ending Sep 30 + and a newer one starting Sep 16 share 15 days. + * The windows meet exactly, the newer starting the day after the older ends. That is + continuous with nothing to spare, reported as `overlap_days` of 0 rather than as a gap. + * Service is uncovered in between, returned as `gap_days` - the count of uncovered days, so + one missing day reads as 1. + + Both are None when either bound is missing: an absent window is not a gap. + """ + if older_end is None or newer_start is None: + return None, None + # Days from the end of the older window to the start of the newer one. <= 0 means they + # overlap, exactly 1 means they meet, more than 1 leaves days uncovered. + delta = (newer_start - older_end).days + if delta <= 1: + return 1 - delta, None + return None, delta - 1 diff --git a/api/src/shared/common/error_handling.py b/api/src/shared/common/error_handling.py index bc7e8d6eb..ac9103747 100644 --- a/api/src/shared/common/error_handling.py +++ b/api/src/shared/common/error_handling.py @@ -4,6 +4,9 @@ "Invalid date format for '{}'. Expected ISO 8601 format, example: '2021-01-01T00:00:00Z'" ) availability_from_after_to: Final[str] = "'from' timestamp must be before 'to' timestamp" +continuous_coverage_downloaded_after_before: Final[str] = ( + "'downloaded_after' timestamp must be before 'downloaded_before' timestamp" +) invalid_bounding_coordinates: Final[str] = "Invalid bounding coordinates {} {}" invalid_bounding_method: Final[str] = "Invalid bounding_filter_method {}" feed_not_found: Final[str] = "Feed '{}' not found" diff --git a/api/src/shared/db_models/gtfs_feed_continuous_coverage_impl.py b/api/src/shared/db_models/gtfs_feed_continuous_coverage_impl.py new file mode 100644 index 000000000..7bc8e25cc --- /dev/null +++ b/api/src/shared/db_models/gtfs_feed_continuous_coverage_impl.py @@ -0,0 +1,123 @@ +from typing import Optional + +from feeds_gen.models.gtfs_feed_continuous_coverage import GtfsFeedContinuousCoverage +from feeds_gen.models.gtfs_feed_continuous_coverage_file import GtfsFeedContinuousCoverageFile +from shared.common.continuous_coverage import ( + COVERAGE_FILES, + SOURCE_FEED_INFO, + SOURCE_SERVICE_DATES, + as_date, + overlap_and_gap, + within_max_coverage_window, +) +from shared.database_gen.sqlacodegen_models import Gtfsdataset as GtfsdatasetOrm +from shared.db_models.service_date_window_impl import ServiceDateWindowImpl + + +class GtfsFeedContinuousCoverageImpl(GtfsFeedContinuousCoverage): + """Implementation of the `GtfsFeedContinuousCoverage` model. + + Converts one `gtfsdataset` row - plus the row downloaded just before it - to a Pydantic model, + deriving the windows and the overlap the nightly job does not store. All the policy deciding + what counts as continuous lives in `shared.common.continuous_coverage`, so this class only + applies it. + """ + + class Config: + """Pydantic configuration. + Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object.""" + + from_attributes = True + + @classmethod + def _coverage_window(cls, dataset: GtfsdatasetOrm) -> tuple[Optional[ServiceDateWindowImpl], Optional[str]]: + """The window the calculation uses for a dataset, and which input it came from. + + The validated service dates win: they are what the validator derived from the calendar + files, so they describe the service the dataset actually encodes. `feed_info.txt` is only a + producer's declaration and is used as a fallback, which is also why the two are reported + separately and compared - a mismatch is worth showing rather than resolving silently. + """ + service_window = ServiceDateWindowImpl.from_dates( + dataset.service_date_range_start, dataset.service_date_range_end + ) + if service_window is not None: + return service_window, SOURCE_SERVICE_DATES + + feed_info = dataset.feed_info + if feed_info is None: + return None, None + feed_info_window = ServiceDateWindowImpl.from_dates(feed_info.feed_start_date, feed_info.feed_end_date) + return (feed_info_window, SOURCE_FEED_INFO) if feed_info_window is not None else (None, None) + + @classmethod + def _files(cls, dataset: GtfsdatasetOrm) -> list[GtfsFeedContinuousCoverageFile]: + """Which of the files the calculation reads were present in the dataset. + + Every file in `COVERAGE_FILES` gets an entry, present or not, so a client can render a + fixed row of chips without checking which keys came back. + """ + present = {file.file_name for file in dataset.gtfsfiles} + return [GtfsFeedContinuousCoverageFile(name=name, present=name in present) for name in COVERAGE_FILES] + + @classmethod + def from_orm( + cls, + dataset: GtfsdatasetOrm | None, + previous_dataset: GtfsdatasetOrm | None = None, + is_latest: bool = False, + ) -> GtfsFeedContinuousCoverage | None: + """Create a model instance from a SQLAlchemy Gtfsdataset row object. + + `previous_dataset` is the dataset downloaded immediately before this one, which is what the + overlap is measured against. It is passed in rather than looked up here because it may sit + outside the requested page - the caller is the only one that knows the unpaged neighbour. + """ + if not dataset: + return None + + coverage_window, coverage_window_source = cls._coverage_window(dataset) + service_window = ServiceDateWindowImpl.from_dates( + dataset.service_date_range_start, dataset.service_date_range_end + ) + feed_info = dataset.feed_info + feed_info_window = ( + ServiceDateWindowImpl.from_dates(feed_info.feed_start_date, feed_info.feed_end_date) + if feed_info is not None + else None + ) + + # Only a comparison of two windows that both exist is a verdict. Missing either one leaves + # this None: a dataset with no `feed_info.txt` has not contradicted its calendars. + feed_info_matches = ( + (feed_info_window.start == service_window.start and feed_info_window.end == service_window.end) + if feed_info_window is not None and service_window is not None + else None + ) + + # The previous dataset's own coverage window is what this one has to meet, so it is resolved + # the same way rather than read straight off the service date columns. + previous_window = cls._coverage_window(previous_dataset)[0] if previous_dataset else None + overlap_days, gap_days = overlap_and_gap( + previous_window.end if previous_window else None, + coverage_window.start if coverage_window else None, + ) + + return cls( + dataset_id=dataset.stable_id, + is_latest=is_latest, + downloaded_at=dataset.downloaded_at, + coverage_window=coverage_window, + coverage_window_source=coverage_window_source, + within_max_coverage_window=within_max_coverage_window( + as_date(coverage_window.start) if coverage_window else None, + as_date(coverage_window.end) if coverage_window else None, + ), + service_window=service_window, + feed_info_window=feed_info_window, + feed_info_matches=feed_info_matches, + previous_dataset_id=previous_dataset.stable_id if previous_dataset else None, + overlap_days=overlap_days, + gap_days=gap_days, + files=cls._files(dataset), + ) diff --git a/api/src/shared/db_models/service_date_window_impl.py b/api/src/shared/db_models/service_date_window_impl.py new file mode 100644 index 000000000..6f5d212a8 --- /dev/null +++ b/api/src/shared/db_models/service_date_window_impl.py @@ -0,0 +1,35 @@ +import datetime +from typing import Optional + +from feeds_gen.models.service_date_window import ServiceDateWindow +from shared.common.continuous_coverage import as_date, window_days + + +class ServiceDateWindowImpl(ServiceDateWindow): + """Implementation of the `ServiceDateWindow` model. + + A window is a pair of columns rather than a row of its own, so this class is built from the two + dates instead of from an ORM object. + """ + + class Config: + """Pydantic configuration. + Enabling `from_attributes` method to create a model instance from a SQLAlchemy row object.""" + + from_attributes = True + + @classmethod + def from_dates( + cls, + start: Optional[datetime.date | datetime.datetime], + end: Optional[datetime.date | datetime.datetime], + ) -> Optional[ServiceDateWindow]: + """Build a window from a start and end date, or None when either is missing. + + A half-open window is reported as no window at all: both bounds are required to say + anything about coverage, and serving one bound would invite a client to treat it as a range. + """ + start_date, end_date = as_date(start), as_date(end) + if start_date is None or end_date is None: + return None + return cls(start=start_date, end=end_date, days=window_days(start_date, end_date)) diff --git a/api/src/utils/date_utils.py b/api/src/utils/date_utils.py index 14f9bca53..466f766ac 100644 --- a/api/src/utils/date_utils.py +++ b/api/src/utils/date_utils.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from typing import Final, Optional import re @@ -14,3 +15,17 @@ def valid_iso_date(date_string: Optional[str]) -> bool: if date_string is None or date_string.strip() == "": return True return re.match(iso_pattern, date_string) is not None + + +def parse_iso_datetime(date_string: Optional[str]) -> Optional[datetime]: + """Parse a `valid_iso_date`-validated string, defaulting to UTC when it carries no offset. + + `valid_iso_date`'s offset group is optional, so `2024-01-01T00:00:00` and + `2024-01-01T00:00:00Z` are equally valid input, but `fromisoformat` returns a naive datetime + for the first and a timezone-aware one for the second - comparing one of each raises + `TypeError`. Normalizing here means two independently-parsed values are always comparable. + """ + if not date_string: + return None + parsed = datetime.fromisoformat(date_string) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) diff --git a/api/tests/unittest/models/test_continuous_coverage.py b/api/tests/unittest/models/test_continuous_coverage.py new file mode 100644 index 000000000..a23f96609 --- /dev/null +++ b/api/tests/unittest/models/test_continuous_coverage.py @@ -0,0 +1,103 @@ +import unittest +from datetime import date, datetime, timedelta, timezone + +from shared.common.continuous_coverage import ( + MAX_COVERAGE_WINDOW, + as_date, + overlap_and_gap, + window_days, + within_max_coverage_window, +) + + +class TestAsDate(unittest.TestCase): + """Test `as_date`, which reconciles the two ways a service bound is stored.""" + + def test_datetime_is_narrowed(self): + """Service dates are stored as timestamps but describe whole days.""" + assert as_date(datetime(2026, 9, 16, 13, 45, tzinfo=timezone.utc)) == date(2026, 9, 16) + + def test_date_passes_through(self): + """`feed_info.txt` dates are already dates.""" + assert as_date(date(2026, 9, 16)) == date(2026, 9, 16) + + def test_none(self): + assert as_date(None) is None + + +class TestWindowDays(unittest.TestCase): + """Test `window_days`, the inclusive length of a service window.""" + + def test_counts_both_bounds(self): + """Sep 16 through Sep 30 is 15 service days, not 14.""" + assert window_days(date(2026, 9, 16), date(2026, 9, 30)) == 15 + + def test_single_day_window(self): + """A window covering one day covers one day, not zero.""" + assert window_days(date(2026, 9, 16), date(2026, 9, 16)) == 1 + + def test_incomplete_window(self): + assert window_days(None, date(2026, 9, 30)) is None + assert window_days(date(2026, 9, 16), None) is None + + def test_inverted_window(self): + """An end date before its start is no measurement, not a negative one.""" + assert window_days(date(2026, 9, 30), date(2026, 9, 16)) is None + + +class TestWithinMaxCoverageWindow(unittest.TestCase): + """Test the two-year limit.""" + + def test_inside_the_limit(self): + start = date(2026, 9, 16) + assert within_max_coverage_window(start, start + MAX_COVERAGE_WINDOW - timedelta(days=1)) is True + + def test_exactly_at_the_limit(self): + """The limit is inclusive: a window exactly two years long still qualifies.""" + start = date(2026, 9, 16) + assert within_max_coverage_window(start, start + MAX_COVERAGE_WINDOW) is True + + def test_beyond_the_limit(self): + start = date(2026, 9, 16) + assert within_max_coverage_window(start, start + MAX_COVERAGE_WINDOW + timedelta(days=1)) is False + + def test_no_window_is_not_a_pass(self): + """A missing window must not read as satisfying the limit.""" + assert within_max_coverage_window(None, date(2027, 7, 28)) is None + + +class TestOverlapAndGap(unittest.TestCase): + """Test how successive datasets are judged to meet, overlap or leave a gap.""" + + def test_overlapping_windows(self): + """An older window ending Sep 30 and a newer one starting Sep 16 share 15 days.""" + assert overlap_and_gap(date(2026, 9, 30), date(2026, 9, 16)) == (15, None) + + def test_windows_meeting_exactly(self): + """The newer window starting the day after the older ends is continuous with nothing spare. + + That is reported as zero overlap rather than as a gap: no service day is uncovered. + """ + assert overlap_and_gap(date(2026, 9, 30), date(2026, 10, 1)) == (0, None) + + def test_same_day_boundary(self): + """Both windows covering Sep 30 share exactly that one day.""" + assert overlap_and_gap(date(2026, 9, 30), date(2026, 9, 30)) == (1, None) + + def test_one_uncovered_day(self): + """A single missing service day reads as a gap of 1, not 2.""" + assert overlap_and_gap(date(2026, 9, 30), date(2026, 10, 2)) == (None, 1) + + def test_larger_gap(self): + assert overlap_and_gap(date(2026, 9, 30), date(2026, 10, 15)) == (None, 14) + + def test_missing_bound_is_not_a_gap(self): + """An absent window says nothing about continuity, so neither value is reported.""" + assert overlap_and_gap(None, date(2026, 9, 16)) == (None, None) + assert overlap_and_gap(date(2026, 9, 30), None) == (None, None) + + def test_accepts_timestamps_via_as_date(self): + """Mixed storage types compare correctly once narrowed.""" + older_end = as_date(datetime(2026, 9, 30, 23, 59, tzinfo=timezone.utc)) + newer_start = as_date(date(2026, 9, 16)) + assert overlap_and_gap(older_end, newer_start) == (15, None) diff --git a/api/tests/unittest/models/test_gtfs_feed_continuous_coverage_impl.py b/api/tests/unittest/models/test_gtfs_feed_continuous_coverage_impl.py new file mode 100644 index 000000000..d8da190b7 --- /dev/null +++ b/api/tests/unittest/models/test_gtfs_feed_continuous_coverage_impl.py @@ -0,0 +1,167 @@ +import unittest +from datetime import date, datetime, timedelta, timezone + +from shared.common.continuous_coverage import ( + COVERAGE_FILES, + MAX_COVERAGE_WINDOW, + SOURCE_FEED_INFO, + SOURCE_SERVICE_DATES, +) +from shared.database_gen.sqlacodegen_models import Feedinfo, Gtfsdataset, Gtfsfile +from shared.db_models.gtfs_feed_continuous_coverage_impl import GtfsFeedContinuousCoverageImpl + + +def make_dataset( + stable_id="mdb-1-202606280029", + service_start=datetime(2026, 9, 16, tzinfo=timezone.utc), + service_end=datetime(2027, 7, 28, tzinfo=timezone.utc), + feed_info=None, + files=COVERAGE_FILES, + downloaded_at=datetime(2026, 6, 28, tzinfo=timezone.utc), +): + """A dataset whose validated service window is Sep 16 2026 - Jul 28 2027.""" + return Gtfsdataset( + id=stable_id, + stable_id=stable_id, + downloaded_at=downloaded_at, + service_date_range_start=service_start, + service_date_range_end=service_end, + feed_info=feed_info, + gtfsfiles=[ + Gtfsfile(id=f"{stable_id}-{name}", gtfs_dataset_id=stable_id, file_name=name, file_size_bytes=1) + for name in files + ], + ) + + +def make_feed_info(start=date(2026, 9, 16), end=date(2027, 7, 28)): + return Feedinfo(file_hash="hash", feed_start_date=start, feed_end_date=end) + + +class TestGtfsFeedContinuousCoverageImpl(unittest.TestCase): + """Test the `GtfsFeedContinuousCoverageImpl` model.""" + + def test_no_dataset_returns_none(self): + assert GtfsFeedContinuousCoverageImpl.from_orm(None) is None + + def test_service_dates_are_preferred(self): + """The validator's service dates are the calculation's input when present.""" + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(feed_info=make_feed_info())) + + assert result.coverage_window_source == SOURCE_SERVICE_DATES + assert result.coverage_window.start == date(2026, 9, 16) + assert result.coverage_window.end == date(2027, 7, 28) + assert result.coverage_window.days == 316 + + def test_feed_info_is_the_fallback(self): + """A dataset the validator produced no service dates for falls back on `feed_info.txt`.""" + dataset = make_dataset(service_start=None, service_end=None, feed_info=make_feed_info()) + result = GtfsFeedContinuousCoverageImpl.from_orm(dataset) + + assert result.coverage_window_source == SOURCE_FEED_INFO + assert result.coverage_window.start == date(2026, 9, 16) + assert result.service_window is None + + def test_no_window_at_all(self): + """With neither input there is no window, and no verdict on the two-year limit.""" + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(service_start=None, service_end=None)) + + assert result.coverage_window is None + assert result.coverage_window_source is None + assert result.within_max_coverage_window is None + + def test_windows_are_reported_separately(self): + """Both inputs are served even when they agree, so a client can show the two bars.""" + dataset = make_dataset(feed_info=make_feed_info()) + result = GtfsFeedContinuousCoverageImpl.from_orm(dataset) + + assert result.service_window.start == date(2026, 9, 16) + assert result.feed_info_window.start == date(2026, 9, 16) + assert result.feed_info_matches is True + + def test_feed_info_mismatch(self): + """A `feed_info.txt` disagreeing with the calendars is reported, not resolved silently.""" + dataset = make_dataset(feed_info=make_feed_info(start=date(2026, 10, 1))) + result = GtfsFeedContinuousCoverageImpl.from_orm(dataset) + + assert result.feed_info_matches is False + + def test_missing_feed_info_is_not_a_mismatch(self): + """A dataset with no `feed_info.txt` has not contradicted its calendars.""" + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset()) + + assert result.feed_info_window is None + assert result.feed_info_matches is None + + def test_beyond_the_two_year_limit(self): + """A dataset declaring service more than two years out fails the limit.""" + start = datetime(2026, 9, 16, tzinfo=timezone.utc) + dataset = make_dataset(service_start=start, service_end=start + MAX_COVERAGE_WINDOW + timedelta(days=1)) + result = GtfsFeedContinuousCoverageImpl.from_orm(dataset) + + assert result.within_max_coverage_window is False + + def test_exactly_at_the_two_year_limit(self): + """The limit is inclusive, so a window exactly two years long still passes.""" + start = datetime(2026, 9, 16, tzinfo=timezone.utc) + dataset = make_dataset(service_start=start, service_end=start + MAX_COVERAGE_WINDOW) + result = GtfsFeedContinuousCoverageImpl.from_orm(dataset) + + assert result.within_max_coverage_window is True + + def test_overlap_with_previous_dataset(self): + """The mock's case: an older window ending Sep 30 and a newer one starting Sep 16.""" + previous = make_dataset( + stable_id="mdb-1-202604290029", + service_start=datetime(2026, 5, 1, tzinfo=timezone.utc), + service_end=datetime(2026, 9, 30, tzinfo=timezone.utc), + ) + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(), previous_dataset=previous) + + assert result.previous_dataset_id == "mdb-1-202604290029" + assert result.overlap_days == 15 + assert result.gap_days is None + + def test_gap_from_previous_dataset(self): + """Service uncovered between two datasets is what the criterion is meant to catch.""" + previous = make_dataset( + stable_id="mdb-1-202604290029", + service_start=datetime(2026, 5, 1, tzinfo=timezone.utc), + service_end=datetime(2026, 9, 1, tzinfo=timezone.utc), + ) + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(), previous_dataset=previous) + + assert result.overlap_days is None + assert result.gap_days == 14 + + def test_previous_dataset_window_uses_the_same_fallback(self): + """The older dataset's window is resolved the same way, not read off service dates only.""" + previous = make_dataset( + stable_id="mdb-1-202604290029", + service_start=None, + service_end=None, + feed_info=make_feed_info(start=date(2026, 5, 1), end=date(2026, 9, 30)), + ) + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(), previous_dataset=previous) + + assert result.overlap_days == 15 + + def test_no_previous_dataset(self): + """The oldest dataset of a feed has nothing to overlap with.""" + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset()) + + assert result.previous_dataset_id is None + assert result.overlap_days is None + assert result.gap_days is None + + def test_files_always_reported_in_full(self): + """All three files get an entry so a client can render a fixed row.""" + result = GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(files=("calendar.txt", "stops.txt"))) + + assert [file.name for file in result.files] == list(COVERAGE_FILES) + assert [file.present for file in result.files] == [False, True, False] + + def test_is_latest_is_passed_through(self): + """Whether a dataset is the feed's latest is the caller's knowledge, not the row's.""" + assert GtfsFeedContinuousCoverageImpl.from_orm(make_dataset(), is_latest=True).is_latest is True + assert GtfsFeedContinuousCoverageImpl.from_orm(make_dataset()).is_latest is False diff --git a/api/tests/unittest/test_feeds.py b/api/tests/unittest/test_feeds.py index 1ae240bbc..cbd6cfdb9 100644 --- a/api/tests/unittest/test_feeds.py +++ b/api/tests/unittest/test_feeds.py @@ -1,16 +1,25 @@ import contextlib import copy -from datetime import datetime, timedelta, timezone +from datetime import date, datetime, timedelta, timezone from unittest.mock import Mock, MagicMock import json +import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient +from feeds.impl.datasets_api_impl import DatasetsApiImpl +from feeds.impl.feeds_api_impl import FeedsApiImpl +from feeds_gen.models.gtfs_feed_continuous_coverage import GtfsFeedContinuousCoverage +from feeds_gen.models.gtfs_feed_continuous_coverage_file import GtfsFeedContinuousCoverageFile +from feeds_gen.models.service_date_window import ServiceDateWindow +from shared.common.continuous_coverage import COVERAGE_FILES from shared.database_gen.sqlacodegen_models import GtfsFeedAvailabilityCheck as DbAvailabilityCheck from shared.common.error_handling import InternalHTTPException, unknown_seal_criterion from shared.db_models.feed_impl import FeedImpl from shared.db_models.feed_reliability_report_impl import FeedReliabilityReportImpl from shared.db_models.gtfs_feed_availability_check_impl import GtfsFeedAvailabilityCheckImpl +from shared.db_models.gtfs_feed_continuous_coverage_impl import GtfsFeedContinuousCoverageImpl from shared.database.database import Database from shared.database_gen.sqlacodegen_models import ( Feed, @@ -576,3 +585,319 @@ def test_gtfs_feed_get_without_seal_reports_null(client: TestClient): assert response.status_code == 200, f"Response status code was {response.status_code} instead of 200" assert response.json()["reliability_seal"] is None + + +# ---- Unit tests for the continuous coverage endpoint's `latest_*` root fields ---- + + +def test_latest_continuous_coverage_no_latest_dataset(): + """A feed with no `latest_dataset_id` has no latest dataset to summarize.""" + feed = Gtfsfeed(latest_dataset_id=None) + assert FeedsApiImpl._latest_continuous_coverage(feed, MagicMock()) is None + + +def test_latest_continuous_coverage_dataset_not_found(): + """`latest_dataset_id` pointing at a row the query can't find is treated as no latest dataset.""" + feed = Gtfsfeed(latest_dataset_id="dataset-latest") + feed_datasets = MagicMock() + feed_datasets.filter.return_value.options.return_value.first.return_value = None + + assert FeedsApiImpl._latest_continuous_coverage(feed, feed_datasets) is None + + +def test_latest_continuous_coverage_delegates_to_the_model_impl(mocker): + """The snapshot is computed the same way as an `items[]` entry - `is_latest=True`, and measured + against its own predecessor rather than whichever dataset happens to lead the requested page.""" + feed = Gtfsfeed(latest_dataset_id="dataset-latest") + latest_dataset = Gtfsdataset(id="dataset-latest", stable_id="dataset-latest") + previous_dataset = Gtfsdataset(id="dataset-previous", stable_id="dataset-previous") + + feed_datasets = MagicMock() + feed_datasets.filter.return_value.options.return_value.first.return_value = latest_dataset + mocker.patch.object(FeedsApiImpl, "_previous_dataset", return_value=previous_dataset) + from_orm = mocker.patch.object(GtfsFeedContinuousCoverageImpl, "from_orm") + + result = FeedsApiImpl._latest_continuous_coverage(feed, feed_datasets) + + feed_datasets.filter.assert_called_once() + from_orm.assert_called_once_with(latest_dataset, previous_dataset=previous_dataset, is_latest=True) + assert result is from_orm.return_value + + +def test_get_gtfs_feed_continuous_coverage_maps_latest_fields(mocker): + """The response's root `latest_*` fields are the `_latest_continuous_coverage` snapshot, not the + first item of whatever page was requested.""" + feed = Gtfsfeed(latest_dataset_id="dataset-latest") + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + latest_coverage = GtfsFeedContinuousCoverage( + dataset_id="dataset-latest", + is_latest=True, + coverage_window=ServiceDateWindow(start=date(2026, 9, 16), end=date(2027, 7, 28), days=316), + coverage_window_source="service_dates", + within_max_coverage_window=True, + service_window=ServiceDateWindow(start=date(2026, 9, 16), end=date(2027, 7, 28), days=316), + feed_info_window=None, + feed_info_matches=None, + overlap_days=15, + gap_days=None, + files=[GtfsFeedContinuousCoverageFile(name=name, present=True) for name in COVERAGE_FILES], + ) + mocker.patch.object(FeedsApiImpl, "_latest_continuous_coverage", return_value=latest_coverage) + + db_session = MagicMock() + db_session.query.return_value.filter.return_value.count.return_value = 0 + empty_page = db_session.query.return_value.filter.return_value.order_by.return_value + empty_page.offset.return_value.limit.return_value.options.return_value.all.return_value = [] + + response = FeedsApiImpl().get_gtfs_feed_continuous_coverage( + id="mdb-1", + downloaded_after=None, + downloaded_before=None, + limit=20, + offset=0, + db_session=db_session, + ) + + assert response.latest_coverage_window.start == date(2026, 9, 16) + assert response.latest_coverage_window_source == "service_dates" + assert response.latest_within_max_coverage_window is True + assert response.latest_service_window.end == date(2027, 7, 28) + assert response.latest_feed_info_window is None + assert response.latest_feed_info_matches is None + assert response.latest_overlap_days == 15 + assert response.latest_gap_days is None + assert [f.present for f in response.latest_files] == [True] * len(COVERAGE_FILES) + + +def test_get_gtfs_feed_continuous_coverage_no_latest_dataset(mocker): + """A feed with no datasets still returns the required `latest_files` list, with every file + reported absent rather than the field being omitted.""" + feed = Gtfsfeed(latest_dataset_id=None) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + db_session = MagicMock() + db_session.query.return_value.filter.return_value.count.return_value = 0 + empty_page = db_session.query.return_value.filter.return_value.order_by.return_value + empty_page.offset.return_value.limit.return_value.options.return_value.all.return_value = [] + + response = FeedsApiImpl().get_gtfs_feed_continuous_coverage( + id="mdb-1", + downloaded_after=None, + downloaded_before=None, + limit=20, + offset=0, + db_session=db_session, + ) + + assert response.latest_coverage_window is None + assert [f.name for f in response.latest_files] == list(COVERAGE_FILES) + assert [f.present for f in response.latest_files] == [False] * len(COVERAGE_FILES) + + +# ---- Regression tests: `datetime.fromisoformat` parses `Z`-suffixed dates directly on Python +# 3.11+ (https://github.com/python/cpython/issues/80010), so `downloaded_after`/`downloaded_before`/ +# `_from`/`to` no longer need a `Z` -> `+00:00` rewrite before parsing. ---- + + +def test_get_gtfs_feed_continuous_coverage_accepts_z_suffixed_dates(mocker): + """A `Z`-suffixed `downloaded_after`/`downloaded_before` - what GTFS timestamps actually look + like - must parse without raising, now that the manual rewrite is gone.""" + feed = Gtfsfeed(latest_dataset_id=None) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + feed_datasets = MagicMock() + feed_datasets.filter.return_value = feed_datasets # chained `.filter()` calls stay on one mock + feed_datasets.count.return_value = 0 + feed_datasets.order_by.return_value.offset.return_value.limit.return_value.options.return_value.all.return_value = ( + [] + ) + + db_session = MagicMock() + db_session.query.return_value = feed_datasets + + response = FeedsApiImpl().get_gtfs_feed_continuous_coverage( + id="mdb-1", + downloaded_after="2024-01-01T00:00:00Z", + downloaded_before="2024-06-01T00:00:00Z", + limit=20, + offset=0, + db_session=db_session, + ) + + assert response.total == 0 + assert response.items == [] + + +def test_get_gtfs_feed_availability_accepts_z_suffixed_dates(mocker): + """Same guarantee for the availability endpoint's `_from`/`to` parameters.""" + feed = Gtfsfeed(id=1) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + query = MagicMock() + query.filter.return_value = query + query.count.return_value = 0 + query.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [] + + db_session = MagicMock() + db_session.query.return_value = query + + response = FeedsApiImpl().get_gtfs_feed_availability( + id="mdb-1", + _from="2024-01-01T00:00:00Z", + to="2024-06-01T00:00:00Z", + limit=20, + offset=0, + sort="desc", + db_session=db_session, + ) + + assert response.total == 0 + assert response.checks == [] + + +def test_get_gtfs_feed_datasets_accepts_z_suffixed_dates(mocker): + """Same guarantee for the datasets endpoint, checked precisely: the `Z` suffix must resolve to + UTC, not be silently dropped as a naive datetime.""" + feed = Gtfsfeed(id=1) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + mocker.patch.object(DatasetsApiImpl, "create_dataset_query", return_value=MagicMock()) + mocker.patch.object(DatasetsApiImpl, "get_datasets_gtfs", return_value=[]) + filter_cls = mocker.patch("feeds.impl.feeds_api_impl.GtfsDatasetFilter") + + result = FeedsApiImpl().get_gtfs_feed_datasets( + gtfs_feed_id="mdb-1", + latest=False, + limit=20, + offset=0, + downloaded_after="2024-01-01T00:00:00Z", + downloaded_before="2024-06-01T00:00:00Z", + db_session=MagicMock(), + ) + + assert result == [] + _, kwargs = filter_cls.call_args + assert kwargs["downloaded_at__gte"] == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert kwargs["downloaded_at__lte"] == datetime(2024, 6, 1, tzinfo=timezone.utc) + + +# ---- Regression tests: one date param carrying a UTC offset and the other not must not raise +# TypeError ("can't compare offset-naive and offset-aware datetimes") - both are equally valid per +# `valid_iso_date`, so `parse_iso_datetime` normalizes an offset-less value to UTC before any +# after/before comparison happens. ---- + + +def test_get_gtfs_feed_continuous_coverage_mixed_naive_and_aware_dates(mocker): + feed = Gtfsfeed(latest_dataset_id=None) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + feed_datasets = MagicMock() + feed_datasets.filter.return_value = feed_datasets + feed_datasets.count.return_value = 0 + feed_datasets.order_by.return_value.offset.return_value.limit.return_value.options.return_value.all.return_value = ( + [] + ) + + db_session = MagicMock() + db_session.query.return_value = feed_datasets + + response = FeedsApiImpl().get_gtfs_feed_continuous_coverage( + id="mdb-1", + downloaded_after="2024-01-01T00:00:00", + downloaded_before="2024-06-01T00:00:00Z", + limit=20, + offset=0, + db_session=db_session, + ) + + assert response.total == 0 + + +def test_get_gtfs_feed_continuous_coverage_mixed_dates_still_validates_order(mocker): + """Normalizing to UTC must not paper over a genuinely reversed range.""" + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=Gtfsfeed(latest_dataset_id=None)) + + with pytest.raises(HTTPException) as exc_info: + FeedsApiImpl().get_gtfs_feed_continuous_coverage( + id="mdb-1", + downloaded_after="2024-06-01T00:00:00", + downloaded_before="2024-01-01T00:00:00Z", + limit=20, + offset=0, + db_session=MagicMock(), + ) + assert exc_info.value.status_code == 422 + + +def test_get_gtfs_feed_availability_mixed_naive_and_aware_dates(mocker): + feed = Gtfsfeed(id=1) + mocker.patch.object(FeedsApiImpl, "_get_gtfs_feed", return_value=feed) + + query = MagicMock() + query.filter.return_value = query + query.count.return_value = 0 + query.order_by.return_value.offset.return_value.limit.return_value.all.return_value = [] + + db_session = MagicMock() + db_session.query.return_value = query + + response = FeedsApiImpl().get_gtfs_feed_availability( + id="mdb-1", + _from="2024-01-01T00:00:00", + to="2024-06-01T00:00:00Z", + limit=20, + offset=0, + sort="desc", + db_session=db_session, + ) + + assert response.total == 0 + + +# ---- Unit tests for `_predecessor`: a page row's positional neighbour is only a valid predecessor +# when both rows have a real `downloaded_at`. `_continuous_coverage_order`'s `nullslast` sorts +# undated datasets to the end of the page, so a positional neighbour next to one of them can't be +# trusted - `_predecessor` must fall back to `_previous_dataset` instead. ---- + + +def test_predecessor_uses_positional_next_when_both_dated(): + dataset = Gtfsdataset(id="a", downloaded_at=datetime(2024, 6, 1, tzinfo=timezone.utc)) + next_dataset = Gtfsdataset(id="b", downloaded_at=datetime(2024, 1, 1, tzinfo=timezone.utc)) + + assert FeedsApiImpl._predecessor(MagicMock(), dataset, next_dataset) is next_dataset + + +def test_predecessor_falls_back_when_dataset_itself_is_undated(mocker): + dataset = Gtfsdataset(id="a", downloaded_at=None) + next_dataset = Gtfsdataset(id="b", downloaded_at=datetime(2024, 1, 1, tzinfo=timezone.utc)) + previous_dataset = mocker.patch.object(FeedsApiImpl, "_previous_dataset", return_value=None) + + result = FeedsApiImpl._predecessor(MagicMock(), dataset, next_dataset) + + assert result is None + previous_dataset.assert_called_once() + + +def test_predecessor_falls_back_when_positional_next_is_undated(mocker): + """A dated dataset immediately followed, in the page, by an undated one must not report that + undated neighbour as its predecessor - we don't know when it was actually downloaded.""" + dataset = Gtfsdataset(id="a", downloaded_at=datetime(2024, 6, 1, tzinfo=timezone.utc)) + next_dataset = Gtfsdataset(id="b", downloaded_at=None) + real_previous = Gtfsdataset(id="c", downloaded_at=datetime(2024, 1, 1, tzinfo=timezone.utc)) + previous_dataset = mocker.patch.object(FeedsApiImpl, "_previous_dataset", return_value=real_previous) + + result = FeedsApiImpl._predecessor(MagicMock(), dataset, next_dataset) + + assert result is real_previous + previous_dataset.assert_called_once() + + +def test_predecessor_falls_back_for_the_last_page_row(mocker): + """No positional neighbour at all (the page's last row) - unchanged from before this fix.""" + dataset = Gtfsdataset(id="a", downloaded_at=datetime(2024, 6, 1, tzinfo=timezone.utc)) + previous_dataset = mocker.patch.object(FeedsApiImpl, "_previous_dataset", return_value=None) + + result = FeedsApiImpl._predecessor(MagicMock(), dataset, None) + + assert result is None + previous_dataset.assert_called_once() diff --git a/api/tests/utils/test_date_utils.py b/api/tests/utils/test_date_utils.py index 67c6def8e..e852a1c77 100644 --- a/api/tests/utils/test_date_utils.py +++ b/api/tests/utils/test_date_utils.py @@ -1,4 +1,6 @@ -from utils.date_utils import valid_iso_date +from datetime import datetime, timezone + +from utils.date_utils import parse_iso_datetime, valid_iso_date def test_valid_iso_date_valid_format(): @@ -33,3 +35,25 @@ def test_invalid_iso_date_valid_format(): """Test valid_iso_date function with invalids ISO 8601 date formats.""" assert not valid_iso_date("2021-01-01") assert not valid_iso_date("June 2021") + + +def test_parse_iso_datetime_none_and_empty(): + assert parse_iso_datetime(None) is None + assert parse_iso_datetime("") is None + + +def test_parse_iso_datetime_naive_defaults_to_utc(): + assert parse_iso_datetime("2024-01-01T00:00:00") == datetime(2024, 1, 1, tzinfo=timezone.utc) + + +def test_parse_iso_datetime_aware_is_preserved(): + assert parse_iso_datetime("2024-01-01T00:00:00Z") == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert parse_iso_datetime("2024-01-01T00:00:00+01:00") == datetime(2023, 12, 31, 23, 0, tzinfo=timezone.utc) + + +def test_parse_iso_datetime_naive_and_aware_are_comparable(): + """The exact scenario `valid_iso_date` allows through and `fromisoformat` alone can't compare: + one input with an offset and one without.""" + naive = parse_iso_datetime("2024-01-01T00:00:00") + aware = parse_iso_datetime("2024-06-01T00:00:00Z") + assert naive < aware diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index 98868bb6d..f47442dff 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -327,6 +327,39 @@ paths: 500: description: Internal server error. + /v1/gtfs_feeds/{id}/continuous_coverage: + parameters: + - $ref: "#/components/parameters/feed_id_path_param" + get: + description: > + Returns the continuous coverage history for a GTFS feed: one entry per dataset, ordered by + `downloaded_at` from newest to oldest. Each entry carries the service window the dataset + covers, the window declared in its `feed_info.txt`, whether the two agree, and how much that + dataset overlaps the previous (older) one. + tags: + - "feeds" + operationId: getGtfsFeedContinuousCoverage + parameters: + - $ref: "#/components/parameters/continuous_coverage_downloaded_after" + - $ref: "#/components/parameters/continuous_coverage_downloaded_before" + - $ref: "#/components/parameters/limit_query_param_continuous_coverage_endpoint" + - $ref: "#/components/parameters/offset" + security: + - Authentication: [] + responses: + 200: + description: Continuous coverage history for the GTFS feed, ordered by downloaded_at (newest first). + content: + application/json: + schema: + $ref: "#/components/schemas/GtfsFeedContinuousCoverageResponse" + 400: + description: Invalid request parameters. + 404: + description: GTFS feed not found. + 500: + description: Internal server error. + /v1/datasets/gtfs/{id}: get: description: Get the specified dataset from the Mobility Database. @@ -1339,6 +1372,237 @@ components: description: Machine-readable error category when the check failed. example: timeout + GtfsFeedContinuousCoverageResponse: + type: object + required: + - feed_id + - items + - latest_files + - total + - offset + - limit + properties: + feed_id: + type: string + description: Unique identifier of the GTFS feed. + example: mdb-123 + latest_files: + type: array + description: > + The files the calculation reads for the feed's latest dataset (the `items[]` entry + with `is_latest: true`), and whether each was present. Always returned in the same + order with one entry per file, so a client can render a fixed row. + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverageFile" + latest_coverage_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_coverage_window_source: + type: string + nullable: true + description: > + Which input the latest dataset's `latest_coverage_window` was taken from. + + * `service_dates` - the service dates derived by the validator from `calendar.txt` and + `calendar_dates.txt`. + * `feed_info` - the dates declared in `feed_info.txt`, used only when the service dates + are missing. + enum: + - service_dates + - feed_info + example: service_dates + latest_within_max_coverage_window: + type: boolean + nullable: true + description: > + Whether the latest dataset's `latest_coverage_window` stays inside the maximum + coverage window the seal allows (two years). Null when there is no coverage window to + measure. + example: true + latest_service_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_feed_info_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_feed_info_matches: + type: boolean + nullable: true + description: > + Whether the latest dataset's `latest_feed_info_window` agrees with + `latest_service_window` on both bounds. Null when either window is missing, which is + not the same as a mismatch. + example: true + latest_overlap_days: + type: integer + nullable: true + description: > + Days of overlap between the latest dataset's coverage window and that of the dataset + immediately older than it. Zero means the windows meet exactly; a gap is reported as + `latest_gap_days` instead. Null when either window is missing or there is no older + dataset. + example: 15 + latest_gap_days: + type: integer + nullable: true + description: > + Days of uncovered service between the end of the older dataset's window and the start + of the latest dataset's window. Null when the windows overlap or meet, which is the + passing case. + example: 3 + total: + type: integer + description: Total number of matching datasets regardless of limit and offset. + example: 42 + offset: + type: integer + description: Offset of the first returned item. + example: 0 + limit: + type: integer + description: Maximum number of items returned. + example: 20 + items: + type: array + description: > + One entry per dataset, ordered by downloaded_at from newest to oldest. The first entry of + the unpaged list is the feed's current coverage; it is marked with `is_latest`. + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverage" + + GtfsFeedContinuousCoverage: + type: object + description: > + The coverage one dataset contributes, and how it lines up with the dataset downloaded just + before it. + + + Three windows are reported. `service_window` is the service dates the validator derived from + `calendar.txt` and `calendar_dates.txt`; `feed_info_window` is what the dataset's + `feed_info.txt` declares; `coverage_window` is the one the calculation actually used, with + `coverage_window_source` naming which of the two it came from. Any of them may be absent when + the dataset did not supply the underlying files. + required: + - dataset_id + - is_latest + - files + properties: + dataset_id: + type: string + description: Stable identifier of the dataset this entry describes. + example: mdb-123-202604290029 + is_latest: + type: boolean + description: > + Whether this is the feed's latest dataset. Exactly one entry in the unpaged list has this + set, so a client can identify the headline entry without assuming it is on the current page. + example: true + downloaded_at: + type: string + format: date-time + nullable: true + description: Timestamp when the dataset was downloaded. + example: "2026-06-28T00:29:00Z" + coverage_window: + $ref: "#/components/schemas/ServiceDateWindow" + coverage_window_source: + type: string + nullable: true + description: > + Which input `coverage_window` was taken from. + + * `service_dates` - the service dates derived by the validator from `calendar.txt` and + `calendar_dates.txt`. + * `feed_info` - the dates declared in `feed_info.txt`, used only when the service dates + are missing. + enum: + - service_dates + - feed_info + example: service_dates + within_max_coverage_window: + type: boolean + nullable: true + description: > + Whether `coverage_window` stays inside the maximum coverage window the seal allows + (two years). Null when there is no coverage window to measure. + example: true + service_window: + $ref: "#/components/schemas/ServiceDateWindow" + feed_info_window: + $ref: "#/components/schemas/ServiceDateWindow" + feed_info_matches: + type: boolean + nullable: true + description: > + Whether `feed_info_window` agrees with `service_window` on both bounds. Null when either + window is missing, which is not the same as a mismatch. + example: true + previous_dataset_id: + type: string + nullable: true + description: > + Stable identifier of the dataset downloaded immediately before this one. Null for the + oldest dataset of the feed. Populated even when that dataset falls outside the requested + page or date range, so overlap is never reported as absent merely because of paging. + example: mdb-123-202604290029 + overlap_days: + type: integer + nullable: true + description: > + Days of overlap between this dataset's coverage window and that of the dataset + immediately older than it. Zero means the windows meet exactly; a gap is reported as + `gap_days` instead. Null when either window is missing or there is no older dataset. + example: 15 + gap_days: + type: integer + nullable: true + description: > + Days of uncovered service between the end of the older dataset's window and the start of + this one. Null when the windows overlap or meet, which is the passing case. + example: 3 + files: + type: array + description: > + The files the calculation reads, and whether each was present in this dataset. Always + returned in the same order with one entry per file, so a client can render a fixed row. + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverageFile" + + GtfsFeedContinuousCoverageFile: + type: object + required: + - name + - present + properties: + name: + type: string + description: Name of the GTFS file. + example: calendar.txt + present: + type: boolean + description: Whether the file was present in the dataset. + example: true + + ServiceDateWindow: + type: object + description: A closed range of service dates, with its length in days. + required: + - start + - end + properties: + start: + type: string + format: date + description: First date covered by the window. + example: "2026-09-16" + end: + type: string + format: date + description: Last date covered by the window. + example: "2027-07-28" + days: + type: integer + nullable: true + description: Length of the window in days, counting both bounds. + example: 316 + LatestDataset: type: object properties: @@ -2429,6 +2693,43 @@ components: format: date-time example: "2026-05-01T00:00:00Z" + limit_query_param_continuous_coverage_endpoint: + name: limit + in: query + description: The number of items to be returned. Maximum is 100. + required: False + schema: + type: integer + minimum: 0 + maximum: 100 + default: 20 + example: 2 + + continuous_coverage_downloaded_after: + name: downloaded_after + in: query + description: > + Only include datasets downloaded at or after this timestamp. Date should be in ISO 8601 + date-time format. The dataset immediately older than the oldest included one is still used to + compute its overlap. + required: False + schema: + type: string + format: date-time + example: "2026-02-24T00:00:00Z" + + continuous_coverage_downloaded_before: + name: downloaded_before + in: query + description: > + Only include datasets downloaded at or before this timestamp. Date should be in ISO 8601 + date-time format. + required: False + schema: + type: string + format: date-time + example: "2026-08-24T00:00:00Z" + availability_sort: name: sort in: query diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 5febae2de..274149ae1 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -1418,6 +1418,225 @@ components: nullable: true description: Machine-readable error category when the check failed. example: timeout + GtfsFeedContinuousCoverageResponse: + type: object + required: + - feed_id + - items + - latest_files + - total + - offset + - limit + properties: + feed_id: + type: string + description: Unique identifier of the GTFS feed. + example: mdb-123 + latest_files: + type: array + description: > + The files the calculation reads for the feed's latest dataset (the `items[]` entry with `is_latest: true`), and whether each was present. Always returned in the same order with one entry per file, so a client can render a fixed row. + + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverageFile" + latest_coverage_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_coverage_window_source: + type: string + nullable: true + description: > + Which input the latest dataset's `latest_coverage_window` was taken from. + + * `service_dates` - the service dates derived by the validator from `calendar.txt` and + + `calendar_dates.txt`. + * `feed_info` - the dates declared in `feed_info.txt`, used only when the service dates + + are missing. + enum: + - service_dates + - feed_info + example: service_dates + latest_within_max_coverage_window: + type: boolean + nullable: true + description: > + Whether the latest dataset's `latest_coverage_window` stays inside the maximum coverage window the seal allows (two years). Null when there is no coverage window to measure. + + example: true + latest_service_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_feed_info_window: + $ref: "#/components/schemas/ServiceDateWindow" + latest_feed_info_matches: + type: boolean + nullable: true + description: > + Whether the latest dataset's `latest_feed_info_window` agrees with `latest_service_window` on both bounds. Null when either window is missing, which is not the same as a mismatch. + + example: true + latest_overlap_days: + type: integer + nullable: true + description: > + Days of overlap between the latest dataset's coverage window and that of the dataset immediately older than it. Zero means the windows meet exactly; a gap is reported as `latest_gap_days` instead. Null when either window is missing or there is no older dataset. + + example: 15 + latest_gap_days: + type: integer + nullable: true + description: > + Days of uncovered service between the end of the older dataset's window and the start of the latest dataset's window. Null when the windows overlap or meet, which is the passing case. + + example: 3 + total: + type: integer + description: Total number of matching datasets regardless of limit and offset. + example: 42 + offset: + type: integer + description: Offset of the first returned item. + example: 0 + limit: + type: integer + description: Maximum number of items returned. + example: 20 + items: + type: array + description: > + One entry per dataset, ordered by downloaded_at from newest to oldest. The first entry of the unpaged list is the feed's current coverage; it is marked with `is_latest`. + + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverage" + GtfsFeedContinuousCoverage: + type: object + description: > + The coverage one dataset contributes, and how it lines up with the dataset downloaded just before it. + + + Three windows are reported. `service_window` is the service dates the validator derived from `calendar.txt` and `calendar_dates.txt`; `feed_info_window` is what the dataset's `feed_info.txt` declares; `coverage_window` is the one the calculation actually used, with `coverage_window_source` naming which of the two it came from. Any of them may be absent when the dataset did not supply the underlying files. + + required: + - dataset_id + - is_latest + - files + properties: + dataset_id: + type: string + description: Stable identifier of the dataset this entry describes. + example: mdb-123-202604290029 + is_latest: + type: boolean + description: > + Whether this is the feed's latest dataset. Exactly one entry in the unpaged list has this set, so a client can identify the headline entry without assuming it is on the current page. + + example: true + downloaded_at: + type: string + format: date-time + nullable: true + description: Timestamp when the dataset was downloaded. + example: "2026-06-28T00:29:00Z" + coverage_window: + $ref: "#/components/schemas/ServiceDateWindow" + coverage_window_source: + type: string + nullable: true + description: > + Which input `coverage_window` was taken from. + + * `service_dates` - the service dates derived by the validator from `calendar.txt` and + + `calendar_dates.txt`. + * `feed_info` - the dates declared in `feed_info.txt`, used only when the service dates + + are missing. + enum: + - service_dates + - feed_info + example: service_dates + within_max_coverage_window: + type: boolean + nullable: true + description: > + Whether `coverage_window` stays inside the maximum coverage window the seal allows (two years). Null when there is no coverage window to measure. + + example: true + service_window: + $ref: "#/components/schemas/ServiceDateWindow" + feed_info_window: + $ref: "#/components/schemas/ServiceDateWindow" + feed_info_matches: + type: boolean + nullable: true + description: > + Whether `feed_info_window` agrees with `service_window` on both bounds. Null when either window is missing, which is not the same as a mismatch. + + example: true + previous_dataset_id: + type: string + nullable: true + description: > + Stable identifier of the dataset downloaded immediately before this one. Null for the oldest dataset of the feed. Populated even when that dataset falls outside the requested page or date range, so overlap is never reported as absent merely because of paging. + + example: mdb-123-202604290029 + overlap_days: + type: integer + nullable: true + description: > + Days of overlap between this dataset's coverage window and that of the dataset immediately older than it. Zero means the windows meet exactly; a gap is reported as `gap_days` instead. Null when either window is missing or there is no older dataset. + + example: 15 + gap_days: + type: integer + nullable: true + description: > + Days of uncovered service between the end of the older dataset's window and the start of this one. Null when the windows overlap or meet, which is the passing case. + + example: 3 + files: + type: array + description: > + The files the calculation reads, and whether each was present in this dataset. Always returned in the same order with one entry per file, so a client can render a fixed row. + + items: + $ref: "#/components/schemas/GtfsFeedContinuousCoverageFile" + GtfsFeedContinuousCoverageFile: + type: object + required: + - name + - present + properties: + name: + type: string + description: Name of the GTFS file. + example: calendar.txt + present: + type: boolean + description: Whether the file was present in the dataset. + example: true + ServiceDateWindow: + type: object + description: A closed range of service dates, with its length in days. + required: + - start + - end + properties: + start: + type: string + format: date + description: First date covered by the window. + example: "2026-09-16" + end: + type: string + format: date + description: Last date covered by the window. + example: "2027-07-28" + days: + type: integer + nullable: true + description: Length of the window in days, counting both bounds. + example: 316 LatestDataset: type: object properties: @@ -2270,6 +2489,7 @@ components: The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2396,6 +2616,7 @@ components: The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2497,6 +2718,7 @@ components: Describes status of the Feed. Should be one of + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2516,6 +2738,7 @@ components: Describes data type of a feed. Should be one of + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. diff --git a/functions-python/operations_api/.openapi-generator/FILES b/functions-python/operations_api/.openapi-generator/FILES index 4f59597fe..15144f1a8 100644 --- a/functions-python/operations_api/.openapi-generator/FILES +++ b/functions-python/operations_api/.openapi-generator/FILES @@ -31,6 +31,9 @@ src/feeds_gen/models/gtfs_dataset.py src/feeds_gen/models/gtfs_feed.py src/feeds_gen/models/gtfs_feed_availability_check.py src/feeds_gen/models/gtfs_feed_availability_response.py +src/feeds_gen/models/gtfs_feed_continuous_coverage.py +src/feeds_gen/models/gtfs_feed_continuous_coverage_file.py +src/feeds_gen/models/gtfs_feed_continuous_coverage_response.py src/feeds_gen/models/gtfs_rt_feed.py src/feeds_gen/models/latest_dataset.py src/feeds_gen/models/latest_dataset_validation_report.py @@ -57,6 +60,7 @@ src/feeds_gen/models/put_user_feature_flags_request.py src/feeds_gen/models/redirect.py src/feeds_gen/models/reliability_criterion.py src/feeds_gen/models/search_feed_item_result.py +src/feeds_gen/models/service_date_window.py src/feeds_gen/models/source_info.py src/feeds_gen/models/update_feature_flag_request.py src/feeds_gen/models/update_request_gtfs_feed.py diff --git a/functions-python/process_validation_report/src/main.py b/functions-python/process_validation_report/src/main.py index b253fbf13..ccfa8f8de 100644 --- a/functions-python/process_validation_report/src/main.py +++ b/functions-python/process_validation_report/src/main.py @@ -109,7 +109,7 @@ def parse_json_report(json_report): """ try: dt = json_report["summary"]["validatedAt"] - validated_at = datetime.fromisoformat(dt.replace("Z", "+00:00")) + validated_at = datetime.fromisoformat(dt) version = None if "validatorVersion" in json_report["summary"]: version = json_report["summary"]["validatorVersion"] diff --git a/functions-python/tasks_executor/src/tasks/users/migrate_firebase_users.py b/functions-python/tasks_executor/src/tasks/users/migrate_firebase_users.py index b7f025bee..f78f9ab9a 100644 --- a/functions-python/tasks_executor/src/tasks/users/migrate_firebase_users.py +++ b/functions-python/tasks_executor/src/tasks/users/migrate_firebase_users.py @@ -125,7 +125,7 @@ def _parse_datastore_timestamp(value) -> datetime | None: return value if value.tzinfo else value.replace(tzinfo=timezone.utc) if isinstance(value, str): try: - dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + dt = datetime.fromisoformat(value) return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc) except ValueError: logger.warning("Cannot parse Datastore timestamp string: %r", value)