feat: api continuous coverage endpoints

api/.openapi-generator/FILES

@@ -34,6 +34,9 @@ src/feeds_gen/models/gtfs_dataset.py
 src/feeds_gen/models/gtfs_feed.py
 src/feeds_gen/models/gtfs_feed_availability_check.py
 src/feeds_gen/models/gtfs_feed_availability_response.py
+src/feeds_gen/models/gtfs_feed_continuous_coverage.py
+src/feeds_gen/models/gtfs_feed_continuous_coverage_file.py
+src/feeds_gen/models/gtfs_feed_continuous_coverage_response.py
 src/feeds_gen/models/gtfs_rt_feed.py
 src/feeds_gen/models/latest_dataset.py
 src/feeds_gen/models/latest_dataset_validation_report.py
@@ -49,6 +52,27 @@ src/feeds_gen/models/redirect.py
 src/feeds_gen/models/reliability_criterion.py
 src/feeds_gen/models/search_feed_item_result.py
 src/feeds_gen/models/search_feeds200_response.py
+src/feeds_gen/models/service_date_window.py
 src/feeds_gen/models/source_info.py
 src/feeds_gen/models/validation_report.py
 src/feeds_gen/security_api.py
+src/user_service/impl/__init__.py
+src/user_service_gen/apis/__init__.py
+src/user_service_gen/apis/notifications_api.py
+src/user_service_gen/apis/notifications_api_base.py
+src/user_service_gen/apis/subscriptions_api.py
+src/user_service_gen/apis/subscriptions_api_base.py
+src/user_service_gen/apis/users_api.py
+src/user_service_gen/apis/users_api_base.py
+src/user_service_gen/main.py
+src/user_service_gen/models/__init__.py
+src/user_service_gen/models/create_notification_subscription_request.py
+src/user_service_gen/models/extra_models.py
+src/user_service_gen/models/feature_flag.py
+src/user_service_gen/models/notification_subscription.py
+src/user_service_gen/models/notification_type.py
+src/user_service_gen/models/subscription_feed.py
+src/user_service_gen/models/update_notification_subscription_request.py
+src/user_service_gen/models/update_user_request.py
+src/user_service_gen/models/user_profile.py
+src/user_service_gen/security_api.py

api/src/feeds/impl/feeds_api_impl.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from typing import List, Union, TypeVar, Optional
 
-from sqlalchemy import or_
+from sqlalchemy import or_, desc, nullslast
 from sqlalchemy.orm import contains_eager, selectinload, Session
 from sqlalchemy.orm.query import Query
 
@@ -11,6 +11,7 @@
 from shared.db_models.feed_reliability_report_impl import FeedReliabilityReportImpl
 from shared.db_models.gbfs_feed_impl import GbfsFeedImpl
 from shared.db_models.gtfs_feed_availability_check_impl import GtfsFeedAvailabilityCheckImpl
+from shared.db_models.gtfs_feed_continuous_coverage_impl import GtfsFeedContinuousCoverageImpl
 from shared.db_models.gtfs_feed_impl import GtfsFeedImpl
 from shared.db_models.gtfs_rt_feed_impl import GtfsRTFeedImpl
 from feeds_gen.apis.feeds_api_base import BaseFeedsApi
@@ -20,8 +21,12 @@
 from feeds_gen.models.gtfs_dataset import GtfsDataset
 from feeds_gen.models.gtfs_feed import GtfsFeed
 from feeds_gen.models.gtfs_feed_availability_response import GtfsFeedAvailabilityResponse
+from feeds_gen.models.gtfs_feed_continuous_coverage import GtfsFeedContinuousCoverage
+from feeds_gen.models.gtfs_feed_continuous_coverage_file import GtfsFeedContinuousCoverageFile
+from feeds_gen.models.gtfs_feed_continuous_coverage_response import GtfsFeedContinuousCoverageResponse
 from feeds_gen.models.gtfs_rt_feed import GtfsRTFeed
 from middleware.request_context import is_user_email_restricted
+from shared.common.continuous_coverage import COVERAGE_FILES
 from shared.common.db_utils import (
     get_gtfs_feeds_query,
     get_gtfs_rt_feeds_query,
@@ -31,6 +36,7 @@
 )
 from shared.common.error_handling import (
     availability_from_after_to,
+    continuous_coverage_downloaded_after_before,
     invalid_date_message,
     feed_not_found,
     gtfs_feed_not_found,
@@ -177,15 +183,9 @@ def get_gtfs_feed_datasets(
         if not feed:
             raise_http_error(404, f"FeedOrm with id {gtfs_feed_id} not found")
 
-        # Replace Z with +00:00 to make the datetime object timezone aware
-        # Due to https://github.com/python/cpython/issues/80010, once migrate to Python 3.11, we can use fromisoformat
         query = GtfsDatasetFilter(
-            downloaded_at__lte=(
-                datetime.fromisoformat(downloaded_before.replace("Z", "+00:00")) if downloaded_before else None
-            ),
-            downloaded_at__gte=(
-                datetime.fromisoformat(downloaded_after.replace("Z", "+00:00")) if downloaded_after else None
-            ),
+            downloaded_at__lte=datetime.fromisoformat(downloaded_before) if downloaded_before else None,
+            downloaded_at__gte=datetime.fromisoformat(downloaded_after) if downloaded_after else None,
         ).filter(DatasetsApiImpl.create_dataset_query().filter(FeedOrm.stable_id == gtfs_feed_id))
 
         if latest:
@@ -348,8 +348,8 @@ def get_gtfs_feed_availability(
         if to and not valid_iso_date(to):
             raise_http_validation_error(invalid_date_message.format("to"))
 
-        from_dt = datetime.fromisoformat(_from.replace("Z", "+00:00")) if _from else None
-        to_dt = datetime.fromisoformat(to.replace("Z", "+00:00")) if to else None
+        from_dt = datetime.fromisoformat(_from) if _from else None
+        to_dt = datetime.fromisoformat(to) if to else None
 
         if from_dt and to_dt and from_dt > to_dt:
             raise_http_validation_error(availability_from_after_to)
@@ -378,6 +378,131 @@ def get_gtfs_feed_availability(
             checks=[GtfsFeedAvailabilityCheckImpl.from_orm(c) for c in checks],
         )
 
+    @with_db_session
+    def get_gtfs_feed_continuous_coverage(
+        self,
+        id: str,
+        downloaded_after: str,
+        downloaded_before: str,
+        limit: int,
+        offset: int,
+        db_session: Session,
+    ) -> GtfsFeedContinuousCoverageResponse:
+        """Returns the continuous coverage history for a GTFS feed."""
+        if downloaded_after and not valid_iso_date(downloaded_after):
+            raise_http_validation_error(invalid_date_message.format("downloaded_after"))
+        if downloaded_before and not valid_iso_date(downloaded_before):
+            raise_http_validation_error(invalid_date_message.format("downloaded_before"))
+
+        after_dt = datetime.fromisoformat(downloaded_after) if downloaded_after else None
+        before_dt = datetime.fromisoformat(downloaded_before) if downloaded_before else None
+
+        if after_dt and before_dt and after_dt > before_dt:
+            raise_http_validation_error(continuous_coverage_downloaded_after_before)
+
+        feed = self._get_gtfs_feed(id, db_session, include_options_for_joinedload=False)
+        if not feed:
+            raise_http_error(404, gtfs_feed_not_found.format(id))
+
+        # Kept separate from the filtered query: the dataset immediately older than the page is
+        # looked up here, and it may well be one the date filters excluded.
+        feed_datasets = db_session.query(Gtfsdataset).filter(Gtfsdataset.feed_id == feed.id)
+
+        query = feed_datasets
+        if after_dt:
+            query = query.filter(Gtfsdataset.downloaded_at >= after_dt)
+        if before_dt:
+            query = query.filter(Gtfsdataset.downloaded_at <= before_dt)
+
+        total = query.count()
+        page = (
+            query.order_by(*self._continuous_coverage_order())
+            .offset(offset)
+            .limit(limit)
+            .options(selectinload(Gtfsdataset.feed_info), selectinload(Gtfsdataset.gtfsfiles))
+            .all()
+        )
+
+        # Each item's overlap is measured against the dataset downloaded just before it. Within the
+        # page that is simply the next item, but the oldest item's neighbour lies outside the page,
+        # so it is fetched from the feed's unfiltered datasets - otherwise every page would report a
+        # missing overlap at its bottom edge and look like a gap.
+        predecessors = page[1:] + [self._previous_dataset(feed_datasets, page[-1]) if page else None]
+
+        latest_coverage = self._latest_continuous_coverage(feed, feed_datasets)
+
+        return GtfsFeedContinuousCoverageResponse(
+            feed_id=id,
+            total=total,
+            offset=offset,
+            limit=limit,
+            latest_files=(
+                latest_coverage.files
+                if latest_coverage
+                else [GtfsFeedContinuousCoverageFile(name=name, present=False) for name in COVERAGE_FILES]
+            ),
+            latest_coverage_window=latest_coverage.coverage_window if latest_coverage else None,
+            latest_coverage_window_source=latest_coverage.coverage_window_source if latest_coverage else None,
+            latest_within_max_coverage_window=(latest_coverage.within_max_coverage_window if latest_coverage else None),
+            latest_service_window=latest_coverage.service_window if latest_coverage else None,
+            latest_feed_info_window=latest_coverage.feed_info_window if latest_coverage else None,
+            latest_feed_info_matches=latest_coverage.feed_info_matches if latest_coverage else None,
+            latest_overlap_days=latest_coverage.overlap_days if latest_coverage else None,
+            latest_gap_days=latest_coverage.gap_days if latest_coverage else None,
+            items=[
+                GtfsFeedContinuousCoverageImpl.from_orm(
+                    dataset,
+                    previous_dataset=previous,
+                    is_latest=dataset.id == feed.latest_dataset_id,
+                )
+                for dataset, previous in zip(page, predecessors)
+            ],
+        )
+
+    @staticmethod
+    def _latest_continuous_coverage(feed: Gtfsfeed, feed_datasets: Query) -> Optional[GtfsFeedContinuousCoverage]:
+        """The coverage snapshot for the feed's latest dataset, independent of the requested page or
+        date filters - the root `latest_*` response fields always describe this dataset, even when it
+        falls outside the current page or date range.
+        """
+        if feed.latest_dataset_id is None:
+            return None
+        latest_dataset = (
+            feed_datasets.filter(Gtfsdataset.id == feed.latest_dataset_id)
+            .options(selectinload(Gtfsdataset.feed_info), selectinload(Gtfsdataset.gtfsfiles))
+            .first()
+        )
+        if latest_dataset is None:
+            return None
+        previous = FeedsApiImpl._previous_dataset(feed_datasets, latest_dataset)
+        return GtfsFeedContinuousCoverageImpl.from_orm(latest_dataset, previous_dataset=previous, is_latest=True)
+
+    @staticmethod
+    def _continuous_coverage_order() -> tuple:
+        """Newest dataset first, with a deterministic tiebreak.
+
+        `downloaded_at` is nullable, and a dataset with no download timestamp cannot be placed in a
+        chronological chain at all, so those sort last rather than ahead of everything. `stable_id`
+        breaks ties so that paging over datasets sharing a timestamp cannot repeat or skip a row.
+        """
+        return nullslast(desc(Gtfsdataset.downloaded_at)), desc(Gtfsdataset.stable_id)
+
+    @staticmethod
+    def _previous_dataset(feed_datasets: Query, dataset: Gtfsdataset) -> Optional[Gtfsdataset]:
+        """The feed's dataset downloaded immediately before `dataset`, ignoring any date filter.
+
+        Returns None for a dataset with no download timestamp: there is no "before" to look for, and
+        ordering it against timestamped datasets would invent a neighbour.
+        """
+        if dataset.downloaded_at is None:
+            return None
+        return (
+            feed_datasets.filter(Gtfsdataset.downloaded_at < dataset.downloaded_at)
+            .order_by(*FeedsApiImpl._continuous_coverage_order())
+            .options(selectinload(Gtfsdataset.feed_info))
+            .first()
+        )
+
     @with_db_session
     def get_gbfs_feed(
         self,

api/src/shared/common/continuous_coverage.py

@@ -0,0 +1,96 @@
+"""Continuous coverage policy values and the arithmetic that applies them.
+
+The maximum coverage window and the list of files the calculation reads are the published
+definition of continuous coverage, so - like the rest of the seal policy in
+`shared.common.seal_criteria` - they live in code rather than in DB config: changing either
+changes which feeds qualify, and that should go through code review, tests and a deploy.
+
+This module is under `shared/` so that the read API and the nightly `fresh_continuous`
+evaluator (`functions-python/tasks_executor`, which symlinks `api/src/shared/*`) can agree on
+what "successive datasets overlap" means. The API computes these values on the fly from the
+datasets it already stores; nothing here is persisted.
+
+See #1761 for the seal algorithm and MobilityData/product-tasks#215 for the endpoint.
+"""
+
+import datetime
+from datetime import timedelta
+from typing import Final, Optional, Tuple
+
+# The longest service window the seal accepts. A dataset declaring service further out than
+# this is not evidence of continuous coverage - it is more likely a placeholder calendar.
+MAX_COVERAGE_WINDOW: Final[timedelta] = timedelta(days=730)
+
+# The files the calculation reads, in the order they are reported. `feed_info.txt` supplies the
+# declared window; the two calendar files are what the validator derives the service window
+# from. Order is fixed so a client can render one row of chips per dataset without sorting.
+COVERAGE_FILES: Final[Tuple[str, ...]] = ("feed_info.txt", "calendar.txt", "calendar_dates.txt")
+
+# Which input a coverage window was taken from. Values match the `coverage_window_source` enum
+# in the API spec.
+SOURCE_SERVICE_DATES: Final[str] = "service_dates"
+SOURCE_FEED_INFO: Final[str] = "feed_info"
+
+
+def as_date(value: Optional[datetime.date | datetime.datetime]) -> Optional[datetime.date]:
+    """Reduce a stored value to a plain date.
+
+    The validated service dates are stored as timestamps and the `feed_info.txt` dates as dates,
+    but both describe whole service days. Comparing them as timestamps would make a window that
+    starts at midnight look like it starts before one stored as a date, so both are narrowed here
+    before any arithmetic.
+    """
+    if value is None:
+        return None
+    return value.date() if isinstance(value, datetime.datetime) else value
+
+
+def window_days(start: Optional[datetime.date], end: Optional[datetime.date]) -> Optional[int]:
+    """Length of a closed date window, counting both bounds.
+
+    A window that starts and ends on the same day covers one day, not zero - the bounds are
+    service dates, not instants. Returns None for an incomplete window and for an inverted one:
+    a producer whose end date precedes its start date has no window to measure, and reporting a
+    negative length would read as a real measurement.
+    """
+    if start is None or end is None or end < start:
+        return None
+    return (end - start).days + 1
+
+
+def within_max_coverage_window(start: Optional[datetime.date], end: Optional[datetime.date]) -> Optional[bool]:
+    """Whether a window stays inside `MAX_COVERAGE_WINDOW`.
+
+    Returns None when there is no window to measure, which a client must not read as passing.
+    """
+    days = window_days(start, end)
+    if days is None:
+        return None
+    return end - start <= MAX_COVERAGE_WINDOW
+
+
+def overlap_and_gap(
+    older_end: Optional[datetime.date], newer_start: Optional[datetime.date]
+) -> Tuple[Optional[int], Optional[int]]:
+    """How a dataset's window meets the window of the dataset downloaded just before it.
+
+    Returns `(overlap_days, gap_days)`, of which at most one is ever set:
+
+    * The windows overlap - the newer one starts on or before the older one ends - so the shared
+      days are returned as `overlap_days`, counting both bounds. An older window ending Sep 30
+      and a newer one starting Sep 16 share 15 days.
+    * The windows meet exactly, the newer starting the day after the older ends. That is
+      continuous with nothing to spare, reported as `overlap_days` of 0 rather than as a gap.
+    * Service is uncovered in between, returned as `gap_days` - the count of uncovered days, so
+      one missing day reads as 1.
+
+    Both are None when either bound is missing: an absent window is not a gap.
+    """
+    if older_end is None or newer_start is None:
+        return None, None
+    # Days from the end of the older window to the start of the newer one. <= 0 means they
+    # overlap, exactly 1 means they meet, more than 1 leaves days uncovered.
+    delta = (newer_start - older_end).days
+    if delta <= 1:
+        return 1 - delta, None
+    return None, delta - 1

api/src/shared/common/error_handling.py

@@ -4,6 +4,9 @@
     "Invalid date format for '{}'. Expected ISO 8601 format, example: '2021-01-01T00:00:00Z'"
 )
 availability_from_after_to: Final[str] = "'from' timestamp must be before 'to' timestamp"
+continuous_coverage_downloaded_after_before: Final[str] = (
+    "'downloaded_after' timestamp must be before 'downloaded_before' timestamp"
+)
 invalid_bounding_coordinates: Final[str] = "Invalid bounding coordinates {} {}"
 invalid_bounding_method: Final[str] = "Invalid bounding_filter_method {}"
 feed_not_found: Final[str] = "Feed '{}' not found"