From 7d7437bb5ebc37254940ff57b890e1834c7ecabe Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 31 Aug 2026 16:25:24 +0200 Subject: [PATCH 1/7] Add a grid-snapped Overture building cache Building footprints come from Overture Maps, which is not in our database, so they have to be read from the source and cached. Raw viewport bboxes are arbitrary floats that never repeat, so caching them directly would never hit. Requests are therefore snapped outward to a fixed XYZ grid at GRID_ZOOM and cached per cell, so a pan that shifts the viewport by a cell only pays for the new column, and two people looking at the same place share the same cells. Cells that turn out to have no buildings are cached too, or an empty area is re-queried on every view. The Overture release is part of the cache key, so repinning OVERTURE_RELEASE invalidates itself. The cell count is capped: the count is what upstream cost scales with, and a whole-world bbox covers 2**28 cells, so buildings_for_bbox refuses anything past MAX_BUILDING_CELLS before it does anything else. count_covering_cells answers that from the corner tiles rather than by listing cells, since listing them is what the cap exists to prevent. overturemaps is imported lazily - it pulls in pyarrow, worth ~40MB of RSS - and its failures are normalised to BuildingDataUnavailable so an unreadable upstream cannot be mistaken for an area with no buildings. Co-Authored-By: Claude Opus 5 --- commcare_connect/microplanning/buildings.py | 228 +++++++++++++ commcare_connect/microplanning/const.py | 19 ++ commcare_connect/microplanning/exceptions.py | 6 + .../microplanning/tests/conftest.py | 10 + .../microplanning/tests/test_buildings.py | 309 ++++++++++++++++++ pyproject.toml | 2 + uv.lock | 75 ++++- 7 files changed, 643 insertions(+), 6 deletions(-) create mode 100644 commcare_connect/microplanning/buildings.py create mode 100644 commcare_connect/microplanning/exceptions.py create mode 100644 commcare_connect/microplanning/tests/conftest.py create mode 100644 commcare_connect/microplanning/tests/test_buildings.py diff --git a/commcare_connect/microplanning/buildings.py b/commcare_connect/microplanning/buildings.py new file mode 100644 index 0000000000..ba6ac53de7 --- /dev/null +++ b/commcare_connect/microplanning/buildings.py @@ -0,0 +1,228 @@ +""" +Building footprints for the microplanning map. + +Footprints come from Overture Maps, which is not in our database, so requests are served straight +from the source and cached. To keep that cache useful the requested viewport bbox is snapped +outward to a fixed XYZ grid (``GRID_ZOOM``): raw viewport bboxes are arbitrary floats that never +repeat, while grid tiles are shared between pans and between users looking at the same place. +""" + +import mercantile +import shapely +from django.core.cache import cache + +from commcare_connect.microplanning.const import ( + BUILDINGS_CACHE_KEY, + BUILDINGS_CACHE_TIMEOUT, + GRID_ZOOM, + MAX_BUILDING_GRID_TILES, + OVERTURE_BUILDING_TYPE, + OVERTURE_CONNECT_TIMEOUT, + OVERTURE_REQUEST_TIMEOUT, +) +from commcare_connect.microplanning.exceptions import AreaTooLarge, BuildingDataUnavailable + +# The Overture release footprints are read from. We could probably store this in the DB and read +# it on a weekly basis. +OVERTURE_RELEASE = "2026-08-19.0" + +# Web Mercator cannot represent the poles; this is the latitude the projection is cut off at. +MAX_MERCATOR_LATITUDE = 85.0511 + + +def parse_bbox(raw): + """Parse a ``west,south,east,north`` query parameter, raising ValueError if it is unusable.""" + if not raw: + raise ValueError("bbox is required") + + parts = raw.split(",") + if len(parts) != 4: + raise ValueError("bbox must have four comma-separated values: west,south,east,north") + + try: + west, south, east, north = (float(part) for part in parts) + except ValueError: + raise ValueError("bbox values must be numbers") + + if not (-180 <= west <= 180 and -180 <= east <= 180): + raise ValueError("bbox longitudes must be between -180 and 180") + if not (-90 <= south <= 90 and -90 <= north <= 90): + raise ValueError("bbox latitudes must be between -90 and 90") + if west >= east or south >= north: + raise ValueError("bbox must be west,south,east,north with west < east and south < north") + + return west, south, east, north + + +def buildings_for_bbox(west, south, east, north): + """ + Return a GeoJSON FeatureCollection of buildings covering the given bbox. + + Its ``bbox`` member is the snapped grid area actually covered, which is always at least the + requested one. + + Grid tiles are read from the cache, and any that are missing are fetched from Overture together and + written back, so a pan that shifts the viewport by a grid tile only pays for the new column. + + Raises ``AreaTooLarge`` for a bbox covering more grid tiles than one request may fetch. + """ + if count_covering_grid_tiles(west, south, east, north) > MAX_BUILDING_GRID_TILES: + raise AreaTooLarge(f"{west},{south},{east},{north} covers more than {MAX_BUILDING_GRID_TILES} grid_tiles") + + grid_tiles = covering_grid_tiles(west, south, east, north) + cache_tile_keys = {grid_tile: cache_key_for_grid_tile(grid_tile) for grid_tile in grid_tiles} + # Read once and pass it on: a second read could miss a grid tile this one hit, and that grid + # tile is not in `fetched_buildings` precisely because the first read found it. + cached_buildings = cache.get_many(list(cache_tile_keys.values())) + + missing_grid_tiles = [grid_tile for grid_tile in grid_tiles if cache_tile_keys[grid_tile] not in cached_buildings] + # `bounds_for_grid_tiles` has no answer for an empty set, and there is nothing to fetch anyway. + fetched_buildings = fetch_buildings_for_grid_tiles(missing_grid_tiles) if missing_grid_tiles else {} + cache.set_many( + {cache_tile_keys[grid_tile]: features for grid_tile, features in fetched_buildings.items()}, + BUILDINGS_CACHE_TIMEOUT, + ) + + features = _dedupe_features_across_grid_tiles(grid_tiles, cache_tile_keys, cached_buildings, fetched_buildings) + return {"type": "FeatureCollection", "bbox": list(bounds_for_grid_tiles(grid_tiles)), "features": features} + + +def _dedupe_features_across_grid_tiles(grid_tiles, cache_tile_keys, cached_buildings, fetched_buildings): + """ + Flatten the per-grid-tile features into one list, in grid tile order, each building once. + """ + features = [] + seen_ids = set() + for grid_tile in grid_tiles: + for feature in cached_buildings.get(cache_tile_keys[grid_tile], fetched_buildings.get(grid_tile, [])): + # A building straddling a grid tile boundary is stored under both grid tiles. + feature_id = feature.get("properties", {}).get("id") + if feature_id is not None: + if feature_id in seen_ids: + continue + seen_ids.add(feature_id) + features.append(feature) + return features + + +def fetch_buildings_for_grid_tiles(grid_tiles): + """ + Fetch the given grid tiles from Overture, returning ``{(x, y): [feature, ...]}``. + + Every grid tile asked for is present in the result, empty ones included, so the caller can cache the + fact that an area has no buildings. One query covers the rectangle enclosing all of them rather + than one query per grid tile: each Overture read pays seconds of fixed overhead opening the remote + dataset, and neighbouring grid tiles largely share the parquet row groups that then get scanned. + """ + try: + reader = _overture_building_reader(bounds_for_grid_tiles(grid_tiles)) + table = reader.read_all() + except BuildingDataUnavailable: + raise + except Exception as e: + raise BuildingDataUnavailable("Could not read building data from Overture") from e + + return _features_by_grid_tile(table, grid_tiles) + + +def _overture_building_reader(bounds): + # Imported lazily: overturemaps pulls in pyarrow, which is slow and memory-hungry to import, + # and most requests to this process never ask for buildings. + from overturemaps import record_batch_reader + + reader = record_batch_reader( + OVERTURE_BUILDING_TYPE, + bbox=bounds, + release=OVERTURE_RELEASE, + connect_timeout=OVERTURE_CONNECT_TIMEOUT, + request_timeout=OVERTURE_REQUEST_TIMEOUT, + stac=True, + ) + if reader is None: + # None means STAC found no parquet files intersecting the bbox; Treat it as an + # error rather than as "no buildings here", or we would cache an empty result for an area + # we never managed to read. + raise BuildingDataUnavailable(f"Overture returned no reader for {bounds}") + return reader + + +def _features_by_grid_tile(table, grid_tiles): + """Turn an Overture record batch into GeoJSON features, bucketed by the grid tiles they fall in.""" + by_grid_tile = {grid_tile: [] for grid_tile in grid_tiles} + wanted = set(grid_tiles) + + ids = table.column("id").to_pylist() + geometries = shapely.from_wkb(table.column("geometry").to_pylist()) + bounds = table.column("bbox").to_pylist() + + for feature_id, geometry, feature_bounds in zip(ids, geometries, bounds): + if geometry is None: + continue + # A building may overlap several grid tiles, and may reach outside the ones we asked for: the + # query covers a rectangle enclosing them, and buildings on its edge stick out. Only the + # grid tiles we are about to cache are filled in. + overlapped = [ + grid_tile + for grid_tile in covering_grid_tiles( + feature_bounds["xmin"], feature_bounds["ymin"], feature_bounds["xmax"], feature_bounds["ymax"] + ) + if grid_tile in wanted + ] + if not overlapped: + continue + + feature = { + "type": "Feature", + "geometry": shapely.geometry.mapping(geometry), + "properties": {"id": feature_id}, + } + for grid_tile in overlapped: + by_grid_tile[grid_tile].append(feature) + + return by_grid_tile + + +def covering_grid_tiles(west, south, east, north, zoom=GRID_ZOOM): + """ + Return the ``(x, y)`` grid tiles at ``zoom`` that together cover the given bbox. + """ + grid_tiles = [(tile.x, tile.y) for tile in mercantile.tiles(west, south, east, north, zoom, truncate=True)] + if grid_tiles: + return grid_tiles + + corner = mercantile.tile(west, min(north, MAX_MERCATOR_LATITUDE), zoom, truncate=True) + return [(corner.x, corner.y)] + + +def count_covering_grid_tiles(west, south, east, north, zoom=GRID_ZOOM): + """ + How many grid tiles ``covering_grid_tiles`` would return, without building the list. + """ + # mercantile.tile cannot project the poles, and clamps nothing itself. mercantile.tiles does + # this for us; here it is ours to do. + north = min(north, MAX_MERCATOR_LATITUDE) + south = max(south, -MAX_MERCATOR_LATITUDE) + + top_left = mercantile.tile(west, north, zoom, truncate=True) + bottom_right = mercantile.tile(east - mercantile.LL_EPSILON, south + mercantile.LL_EPSILON, zoom, truncate=True) + + # A bbox smaller than one grid tile, or landing exactly on a boundary, still covers it. + return (max(bottom_right.x - top_left.x, 0) + 1) * (max(bottom_right.y - top_left.y, 0) + 1) + + +def bounds_for_grid_tiles(grid_tiles, zoom=GRID_ZOOM): + """ + Return the ``(west, south, east, north)`` bounds enclosing every grid tile in ``grid_tiles``. + """ + corners = [mercantile.bounds(x, y, zoom) for x, y in grid_tiles] + return ( + min(west for west, _, _, _ in corners), + min(south for _, south, _, _ in corners), + max(east for _, _, east, _ in corners), + max(north for _, _, _, north in corners), + ) + + +def cache_key_for_grid_tile(grid_tile, zoom=GRID_ZOOM): + x, y = grid_tile + return BUILDINGS_CACHE_KEY.format(release=OVERTURE_RELEASE, z=zoom, x=x, y=y) diff --git a/commcare_connect/microplanning/const.py b/commcare_connect/microplanning/const.py index 7f747277ef..3392857737 100644 --- a/commcare_connect/microplanning/const.py +++ b/commcare_connect/microplanning/const.py @@ -32,3 +32,22 @@ "wag": "work_area_group", "ia": "implementation_area", } + +BUILDING_MIN_ZOOM = 15 + +# Each zoom level divides the map into fixed grid tiles. We can use these grid tiles to cache +# results on so GRID_ZOOM defines the zoom level for which we cache the grid tiles. One grid tile is +# 1024px on screen at BUILDING_MIN_ZOOM, which is what makes MAX_BUILDING_GRID_TILES work out. +GRID_ZOOM = 14 + +# The most (GRID_ZOOM-size) grid tiles one request's boundary box may cover, so that a single view +# cannot fan out into an unbounded number of upstream fetches. +MAX_BUILDING_GRID_TILES = 16 + +# The release is part of the key, so a cached grid tile never goes stale under the release it was read +# from and bumping buildings.OVERTURE_RELEASE invalidates every one of them at once. +BUILDINGS_CACHE_KEY = "buildings:{release}:{z}:{x}:{y}" +BUILDINGS_CACHE_TIMEOUT = 60 * 60 * 24 * 7 +OVERTURE_BUILDING_TYPE = "building" +OVERTURE_CONNECT_TIMEOUT = 5 +OVERTURE_REQUEST_TIMEOUT = 30 diff --git a/commcare_connect/microplanning/exceptions.py b/commcare_connect/microplanning/exceptions.py new file mode 100644 index 0000000000..8e49890f08 --- /dev/null +++ b/commcare_connect/microplanning/exceptions.py @@ -0,0 +1,6 @@ +class BuildingDataUnavailable(Exception): + """Raised when Overture could not be read. Distinct from an area that genuinely has no buildings.""" + + +class AreaTooLarge(Exception): + """Raised when more grid tiles were asked for than one request may fetch.""" diff --git a/commcare_connect/microplanning/tests/conftest.py b/commcare_connect/microplanning/tests/conftest.py new file mode 100644 index 0000000000..cbf2f48a7c --- /dev/null +++ b/commcare_connect/microplanning/tests/conftest.py @@ -0,0 +1,10 @@ +import pytest +from django.core.cache import cache + + +@pytest.fixture +def local_cache(settings): + """Keep cache reads off the shared Redis, and out of each other's way.""" + settings.CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}} + cache.clear() + return cache diff --git a/commcare_connect/microplanning/tests/test_buildings.py b/commcare_connect/microplanning/tests/test_buildings.py new file mode 100644 index 0000000000..21e068432d --- /dev/null +++ b/commcare_connect/microplanning/tests/test_buildings.py @@ -0,0 +1,309 @@ +from unittest.mock import Mock, patch + +import mercantile +import pyarrow as pa +import pytest +import shapely + +from commcare_connect.microplanning.buildings import ( + _features_by_grid_tile, + bounds_for_grid_tiles, + buildings_for_bbox, + cache_key_for_grid_tile, + count_covering_grid_tiles, + covering_grid_tiles, + fetch_buildings_for_grid_tiles, + parse_bbox, +) +from commcare_connect.microplanning.const import ( + BUILDING_MIN_ZOOM, + GRID_ZOOM, + MAX_BUILDING_GRID_TILES, +) +from commcare_connect.microplanning.exceptions import AreaTooLarge, BuildingDataUnavailable + +# The largest map we draw, in css pixels, and where those numbers come from. They live here rather +# than in const.py because they describe the browser, not the endpoint: the cap is a limit on what +# the server will fetch, and this is only the check that the limit leaves room for a real viewport. +# Sizes follow #map-wrapper in microplanning/home.html - a w-96 sidebar plus its gap beside the map, +# and 300px of page above it. +LARGEST_SCREEN_PX = (3840, 2160) # a 4K display +SIDEBAR_PX = 384 + 32 +PAGE_CHROME_PX = 300 +MAPBOX_TILE_PX = 512 # Mapbox GL serves 512px tiles, so the world is this * 2**zoom px across + + +def _arrow_table(rows): + """A minimal stand-in for an Overture record batch: id, WKB geometry and a bbox struct.""" + return pa.table( + { + "id": pa.array([feature_id for feature_id, _ in rows], pa.string()), + "geometry": pa.array([shapely.to_wkb(geometry) for _, geometry in rows], pa.binary()), + "bbox": pa.array( + [dict(zip(("xmin", "ymin", "xmax", "ymax"), geometry.bounds)) for _, geometry in rows], + pa.struct([(name, pa.float64()) for name in ("xmin", "ymin", "xmax", "ymax")]), + ), + } + ) + + +def building(feature_id, lon=8.65, lat=9.05): + return { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": [[[lon, lat], [lon, lat], [lon, lat], [lon, lat]]]}, + "properties": {"id": feature_id}, + } + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("8.65,9.05,8.70,9.09", (8.65, 9.05, 8.70, 9.09)), + ("-1.5,-2.5,3,4", (-1.5, -2.5, 3.0, 4.0)), + ], +) +def test_parse_bbox_accepts_valid_values(raw, expected): + assert parse_bbox(raw) == expected + + +@pytest.mark.parametrize( + "raw", + [ + None, + "", + "8.65,9.05,8.70", + "8.65,9.05,8.70,9.09,1", + "8.65,9.05,8.70,north", + "-181,9.05,8.70,9.09", + "8.65,-91,8.70,9.09", + "8.70,9.05,8.65,9.09", # west past east + "8.65,9.09,8.70,9.05", # south past north + "8.65,9.05,8.65,9.09", # zero width + ], +) +def test_parse_bbox_rejects_bad_values(raw): + with pytest.raises(ValueError): + parse_bbox(raw) + + +def test_covering_grid_tiles_returns_a_single_grid_tile_for_a_bbox_inside_one(): + # A hundred metres or so either side of the origin, well inside one z14 grid tile. + assert covering_grid_tiles(0.001, 0.001, 0.002, 0.002) == [(8192, 8191)] + + +def test_covering_grid_tiles_spans_every_grid_tile_a_bbox_touches(): + # Straddles the prime meridian and the equator, so it covers the four grid tiles that meet there. + assert set(covering_grid_tiles(-0.01, -0.01, 0.01, 0.01)) == { + (8191, 8191), + (8191, 8192), + (8192, 8191), + (8192, 8192), + } + + +def test_covering_grid_tiles_clamps_to_the_world_at_zoom_zero(): + assert covering_grid_tiles(-180, -85, 180, 85, zoom=0) == [(0, 0)] + + +@pytest.mark.parametrize( + "bbox", + [ + (8.65, 9.05, 8.70, 9.09), # a zoom-15-ish viewport in northern Nigeria + (-0.01, -0.01, 0.01, 0.01), + (0.001, 0.001, 0.002, 0.002), + (-73.99, 40.70, -73.96, 40.73), + ], +) +def test_covered_area_always_contains_the_requested_bbox(bbox): + west, south, east, north = bbox + covered_west, covered_south, covered_east, covered_north = bounds_for_grid_tiles(covering_grid_tiles(*bbox)) + + assert covered_west <= west + assert covered_south <= south + assert covered_east >= east + assert covered_north >= north + + +@pytest.mark.parametrize( + "bbox", + [ + (8.65, 9.05, 8.70, 9.09), # a zoom-15-ish viewport + (8.6501, 9.0501, 8.6502, 9.0502), # smaller than one grid tile + # Narrower than the epsilon mercantile.tiles trims off the closing edges, and sitting on a + # grid tile boundary, which is the case that made the list come back empty while the count said + # one - leaving buildings_for_bbox with no grid tiles to take bounds from. + (0.0, 0.0, 1e-13, 1e-13), + (-0.01, -0.01, 0.01, 0.01), # across the prime meridian and the equator + (3.0, 4.0, 14.0, 14.0), # far larger than the cap allows + ], +) +def test_counting_grid_tiles_agrees_with_listing_them(bbox): + assert count_covering_grid_tiles(*bbox) == len(covering_grid_tiles(*bbox)) + + +def test_a_bbox_narrower_than_the_tile_epsilon_still_covers_its_corner_grid_tile(): + bbox = (0.0, 0.0, 1e-13, 1e-13) + + corner = mercantile.tile(0.0, 0.0, GRID_ZOOM) + + assert covering_grid_tiles(*bbox) == [(corner.x, corner.y)] + assert count_covering_grid_tiles(*bbox) == 1 + + +def test_counting_grid_tiles_is_cheap_for_a_bbox_that_could_never_be_served(): + """ + parse_bbox admits the whole world, and the view counts grid tiles before anything else, so counting + must not go anywhere near building the list: this bbox covers 2**28 grid tiles, tens of GB of tuples. + """ + whole_grid = 2**GRID_ZOOM * 2**GRID_ZOOM + + assert count_covering_grid_tiles(-180, -90, 180, 90) == whole_grid + assert whole_grid > MAX_BUILDING_GRID_TILES + + +def test_an_area_past_the_cap_is_refused_before_anything_is_fetched(local_cache): + with patch("commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles") as fetch: + with pytest.raises(AreaTooLarge): + buildings_for_bbox(3.0, 4.0, 14.0, 14.0) + + fetch.assert_not_called() + + +def test_the_cap_admits_the_largest_viewport_we_support(): + assert count_covering_grid_tiles(*_viewport_bbox(*LARGEST_SCREEN_PX, BUILDING_MIN_ZOOM)) <= MAX_BUILDING_GRID_TILES + + +def _viewport_bbox(screen_width_px, screen_height_px, zoom, west=8.65, south=9.05): + """ + The bbox the map would ask for on a screen of this size, at this zoom. + + Degrees per pixel is taken at the equator, where a pixel spans the most longitude, and the + fetched area is snapped outward to whole grid tiles, so this is the worst case for the count. + """ + width_px = screen_width_px - SIDEBAR_PX + height_px = screen_height_px - PAGE_CHROME_PX + degrees_per_px = 360 / (MAPBOX_TILE_PX * 2**zoom) + return (west, south, west + width_px * degrees_per_px, south + height_px * degrees_per_px) + + +def test_buildings_for_bbox_returns_a_feature_collection_covering_the_request(local_cache): + bbox = (8.65, 9.05, 8.70, 9.09) + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: { + grid_tile: [building(f"{grid_tile[0]}-{grid_tile[1]}")] for grid_tile in grid_tiles + }, + ): + collection = buildings_for_bbox(*bbox) + + assert collection["type"] == "FeatureCollection" + assert len(collection["features"]) == len(covering_grid_tiles(*bbox)) + covered_west, covered_south, covered_east, covered_north = collection["bbox"] + assert covered_west <= bbox[0] and covered_south <= bbox[1] + assert covered_east >= bbox[2] and covered_north >= bbox[3] + + +def test_buildings_for_bbox_dedupes_a_building_returned_by_several_grid_tiles(local_cache): + # A building straddling a grid tile boundary comes back from every grid tile it touches. + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: { + grid_tile: [building("straddler"), building(f"{grid_tile[0]}-{grid_tile[1]}")] for grid_tile in grid_tiles + }, + ): + collection = buildings_for_bbox(-0.01, -0.01, 0.01, 0.01) + + ids = [feature["properties"]["id"] for feature in collection["features"]] + assert ids.count("straddler") == 1 + assert len(ids) == len(set(ids)) == 5 # one straddler plus one per grid tile + + +def test_buildings_for_bbox_serves_cached_grid_tiles_without_calling_the_service(local_cache): + bbox = (0.001, 0.001, 0.002, 0.002) + local_cache.set(cache_key_for_grid_tile(covering_grid_tiles(*bbox)[0]), [building("cached")]) + + with patch("commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles") as fetch: + collection = buildings_for_bbox(*bbox) + + fetch.assert_not_called() + assert [feature["properties"]["id"] for feature in collection["features"]] == ["cached"] + + +@pytest.mark.parametrize( + "fetched_ids", + [ + ["fetched"], + # An area with nothing in it is worth remembering, or every view of it re-queries Overture. + [], + ], +) +def test_buildings_for_bbox_caches_what_it_fetched(local_cache, fetched_ids): + bbox = (0.001, 0.001, 0.002, 0.002) + grid_tile = covering_grid_tiles(*bbox)[0] + + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: {c: [building(i) for i in fetched_ids] for c in grid_tiles}, + ): + buildings_for_bbox(*bbox) + + cached = local_cache.get(cache_key_for_grid_tile(grid_tile)) + assert [feature["properties"]["id"] for feature in cached] == fetched_ids + + +def test_buildings_for_bbox_only_fetches_the_grid_tiles_it_is_missing(local_cache): + bbox = (-0.01, -0.01, 0.01, 0.01) + grid_tiles = covering_grid_tiles(*bbox) + cached_grid_tile = grid_tiles[0] + local_cache.set(cache_key_for_grid_tile(cached_grid_tile), [building("cached")]) + + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: {c: [] for c in grid_tiles}, + ) as fetch: + buildings_for_bbox(*bbox) + + assert fetch.call_args.args[0] == [grid_tile for grid_tile in grid_tiles if grid_tile != cached_grid_tile] + + +def _unreadable_reader(): + reader = Mock() + reader.read_all.side_effect = OSError("connection reset") + return reader + + +@pytest.mark.parametrize( + "patch_kwargs", + [ + # STAC found no parquet files for the bbox, which the package signals by returning None. + {"return_value": None}, + # The dataset opened, but reading its rows failed. + {"return_value": _unreadable_reader()}, + # It never opened: looking up the parquet files and reading their footers happens before + # any row does, and raises rather than returning None. + {"side_effect": Exception("Could not open dataset: ")}, + ], + ids=["no reader", "unreadable data", "dataset would not open"], +) +def test_fetch_raises_when_overture_cannot_be_read(patch_kwargs): + with patch("overturemaps.record_batch_reader", **patch_kwargs): + with pytest.raises(BuildingDataUnavailable): + fetch_buildings_for_grid_tiles([(8192, 8191)]) + + +def test_features_by_grid_tile_buckets_each_building_into_every_grid_tile_it_touches(): + boundary_lon = mercantile.bounds(8192, 8191, GRID_ZOOM).west + grid_tiles = [(8191, 8191), (8192, 8191)] + straddler = shapely.box(boundary_lon - 0.001, 0.001, boundary_lon + 0.001, 0.002) + inside = shapely.box(boundary_lon + 0.003, 0.001, boundary_lon + 0.004, 0.002) + table = _arrow_table([("straddler", straddler), ("inside", inside)]) + + by_grid_tile = _features_by_grid_tile(table, grid_tiles) + + assert [f["properties"]["id"] for f in by_grid_tile[(8191, 8191)]] == ["straddler"] + assert [f["properties"]["id"] for f in by_grid_tile[(8192, 8191)]] == ["straddler", "inside"] + assert by_grid_tile[(8192, 8191)][1]["geometry"]["type"] == "Polygon" + + +def test_features_by_grid_tile_returns_an_entry_for_every_requested_grid_tile(): + assert _features_by_grid_tile(_arrow_table([]), [(1, 2), (3, 4)]) == {(1, 2): [], (3, 4): []} diff --git a/pyproject.toml b/pyproject.toml index fe668a5190..893ffbeef0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,8 @@ dependencies = [ "hiredis>=2.2.3", "httpx[http2]>=0.24.1", "jsonpath-ng>=1.5.3", + "mercantile>=1.2.1", + "overturemaps>=1.0.2", "pillow>=10.0.0", "pyproj>=3.7.2", "python-slugify>=8.0.1", diff --git a/uv.lock b/uv.lock index 850c3e5213..dea16e2fc6 100644 --- a/uv.lock +++ b/uv.lock @@ -326,14 +326,11 @@ wheels = [ [[package]] name = "click" -version = "8.1.6" +version = "8.5.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/72/bd/fedc277e7351917b6c4e0ac751853a97af261278a4c7808babafa8ef2120/click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd", size = 336051, upload-time = "2023-07-18T20:05:13.823Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/70/e63223f8116931d365993d4a6b7ef653a4d920b41d03de7c59499962821f/click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5", size = 97909, upload-time = "2023-07-18T20:05:12.481Z" }, + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, ] [[package]] @@ -413,6 +410,8 @@ dependencies = [ { name = "hiredis" }, { name = "httpx", extra = ["http2"] }, { name = "jsonpath-ng" }, + { name = "mercantile" }, + { name = "overturemaps" }, { name = "pillow" }, { name = "pyproj" }, { name = "python-slugify" }, @@ -478,6 +477,8 @@ requires-dist = [ { name = "hiredis", specifier = ">=2.2.3" }, { name = "httpx", extras = ["http2"], specifier = ">=0.24.1" }, { name = "jsonpath-ng", specifier = ">=1.5.3" }, + { name = "mercantile", specifier = ">=1.2.1" }, + { name = "overturemaps", specifier = ">=1.0.2" }, { name = "pillow", specifier = ">=10.0.0" }, { name = "pyproj", specifier = ">=3.7.2" }, { name = "python-slugify", specifier = ">=8.0.1" }, @@ -1558,6 +1559,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/94/a59521de836ef0da54aaf50da6c4da8fb4072fb3053fa71f052fd9399e7a/openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5", size = 249985, upload-time = "2023-03-11T16:58:36.257Z" }, ] +[[package]] +name = "orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/1a/a7075a8e8b0d3f5097d17ac3099017104b6b7b42012041147995d5b2da05/orjson-3.12.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a94f0f0c6fcbb2b5bd9734c57a489c7584a732bbdf04a39e8c83b861e9d03e92", size = 223409, upload-time = "2026-08-14T16:12:12.654Z" }, + { url = "https://files.pythonhosted.org/packages/05/34/c2eb3b2900e5597db7841a4c6416ac2d90081bd956b02d4dd1833fa2b96b/orjson-3.12.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a696529ec96a90d9a5f9570207efe403c8b08f8e4aa2783ee3403511e2fdfa10", size = 124015, upload-time = "2026-08-14T16:12:14.025Z" }, + { url = "https://files.pythonhosted.org/packages/1c/df/b49081766a75b6a37b3d33bdc0a39e492abab8441dd25e3e1998e7b83fcb/orjson-3.12.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:e4ac5059baab4b3acbd99485de019ff8cda0fdf34b61fa74f7197a53db78bfe8", size = 113471, upload-time = "2026-08-14T16:12:15.81Z" }, + { url = "https://files.pythonhosted.org/packages/48/d4/58ea28eeef95c2a27358ed927380a621162cf20bd740bbccf9c3f09a200a/orjson-3.12.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e29957429c35bbb5a185a119c523aa2428b7bbf1a293724c7b9375ed8f892a3", size = 129998, upload-time = "2026-08-14T16:12:17.503Z" }, + { url = "https://files.pythonhosted.org/packages/e2/f4/1e82aa2efc9916422d804697876ce433c907a1abd7c7e5c6d3d48565e5f9/orjson-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce0166feb0a737ab84f598c9a338cbc0b764a036617aa686194f53c7eba0c3e", size = 130891, upload-time = "2026-08-14T16:12:18.762Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e1/15169e9d22b59a406264f99d6db387c0b0b12b6357a8a0169917c2a713eb/orjson-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9caf3d09f47c3c70c4451ada20ef9bc4a4cdffa26f49862cf0a253b329aae2d5", size = 131285, upload-time = "2026-08-14T16:12:20.251Z" }, + { url = "https://files.pythonhosted.org/packages/a4/3a/763dbd426290d044ec3e615a05e70adb6d8b6f95bf17dc355c0081a5e8b6/orjson-3.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b9dca132b1fda5565088e65a6b6e742285e0aeceb6fae549fa8863e16c7d3998", size = 135707, upload-time = "2026-08-14T16:12:21.652Z" }, + { url = "https://files.pythonhosted.org/packages/04/d1/3b2038ed168d22e14182ed715d6963f9c073a83a2ba43cfe918a4fc43c64/orjson-3.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a791f793b287bbc135b8e87c34e35c8bfc693e2a8a620fab1ae682b925f9a32e", size = 127669, upload-time = "2026-08-14T16:12:22.926Z" }, + { url = "https://files.pythonhosted.org/packages/88/ae/b84b3d3e65f5629ada0edcb1d2bccc55d7c5f89d8b981537ecdc3d6f31ec/orjson-3.12.0-cp311-cp311-win32.whl", hash = "sha256:31ed278a36304390adc3eec5d7f6fd593a7c3e99e5a06cd07866396c4b1b4710", size = 128043, upload-time = "2026-08-14T16:12:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/35/24/2ed0e6f51ea3d0af45d807233a851175af75bec83ef5fd0d6a2601904ec0/orjson-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb2539159dfe8d371914f354360fa50e4a577cc89222a3828b9650a5e5040252", size = 122084, upload-time = "2026-08-14T16:12:25.813Z" }, + { url = "https://files.pythonhosted.org/packages/21/dd/95d25fcfbc9471799ef6bb01c552d64ee5cde93ee40ba2f423dd3442c708/orjson-3.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:61318b6de893c7a9d9f3e5ecbadccbfc26a7eb417ccc7bbf0771de3b4d72f868", size = 127035, upload-time = "2026-08-14T16:12:27.201Z" }, +] + +[[package]] +name = "overturemaps" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama" }, + { name = "numpy" }, + { name = "orjson" }, + { name = "pyarrow" }, + { name = "pyfiglet" }, + { name = "shapely" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/6b/d02503bba3a90fc333d6188b892554bcfccb30b6e3728086fa0fa4c2857f/overturemaps-1.0.2.tar.gz", hash = "sha256:e92355dcc2961da0ce95ab9837a59f2d15bcc357be51d0c415ceab3d812fc97d", size = 111372, upload-time = "2026-08-25T15:59:26.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/55/e3545154339873cdeeae7c5dca00c22e58de1192bf2769795cb31e319ba6/overturemaps-1.0.2-py3-none-any.whl", hash = "sha256:5c3a0e607d9a9be60467f3a6c8977220702dcfdf35e842bf48e72a326b9ac6ce", size = 25451, upload-time = "2026-08-25T15:59:25.838Z" }, +] + [[package]] name = "packaging" version = "23.1" @@ -1723,6 +1762,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/27/77f9d5684e6bce929f5cfe18d6cfbe5133013c06cb2fbf5933670e60761d/pure_eval-0.2.2-py3-none-any.whl", hash = "sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350", size = 11693, upload-time = "2022-01-22T15:41:27.814Z" }, ] +[[package]] +name = "pyarrow" +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, +] + [[package]] name = "pycparser" version = "2.21" @@ -1741,6 +1795,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/22/11/47efe2f66ba848a107adfd490b508f5c0cedc82127950553dca44d29e6c4/pydyf-0.12.1-py3-none-any.whl", hash = "sha256:ea25b4e1fe7911195cb57067560daaa266639184e8335365cc3ee5214e7eaadc", size = 8028, upload-time = "2025-12-02T14:52:12.938Z" }, ] +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615, upload-time = "2025-08-15T18:32:47.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118, upload-time = "2025-08-15T18:32:45.556Z" }, +] + [[package]] name = "pygments" version = "2.15.1" From fb7697cc3e4511b0f50fc03bda9dcc5cfada0d1d Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 31 Aug 2026 16:25:57 +0200 Subject: [PATCH 2/7] Serve building footprints from a flag-gated endpoint Adds GET /buildings/?bbox=west,south,east,north, returning a GeoJSON FeatureCollection for the cells covering that bbox. Its bbox member is the snapped grid area actually covered, always at least what was asked for, so the caller can tell whether it has panned past the data it holds. Behind the same decorators as the rest of the microplanning views. The bbox comes from the client rather than from anything in our database, so it is parsed and range-checked, and a malformed one is answered with 400 rather than being allowed to reach the grid arithmetic. The view is non-atomic: reading from Overture takes seconds and ATOMIC_REQUESTS would otherwise hold a database transaction open for all of it, which this view has no use for since it only reads. An unreadable upstream is reported as 503 rather than as an empty map, so the client can tell "no buildings here" from "we could not find out". Co-Authored-By: Claude Opus 5 --- .../microplanning/tests/test_views.py | 104 ++++++++++++++++++ commcare_connect/microplanning/urls.py | 1 + commcare_connect/microplanning/views.py | 31 ++++++ 3 files changed, 136 insertions(+) diff --git a/commcare_connect/microplanning/tests/test_views.py b/commcare_connect/microplanning/tests/test_views.py index 8f3823b658..ef92a02b96 100644 --- a/commcare_connect/microplanning/tests/test_views.py +++ b/commcare_connect/microplanning/tests/test_views.py @@ -27,6 +27,7 @@ SEARCH_KIND_FILTERS, SERVICE_DELIVERY_UNIT_SLUG, ) +from commcare_connect.microplanning.exceptions import BuildingDataUnavailable from commcare_connect.microplanning.filters import WorkAreaMapFilterSet from commcare_connect.microplanning.forms import AssignmentModeForm from commcare_connect.microplanning.models import ( @@ -221,6 +222,109 @@ def test_clear_deletes_areas_and_keeps_work_area_names(self, client, org_user_ad assert work_area.implementation_area_name == "Ward North" +@pytest.mark.django_db +class TestBuildingsGeojson(BaseMicroplanningFlagTest): + @pytest.fixture(autouse=True) + def isolated_cache(self, local_cache): + """Grid tile lookups must not carry over between tests, or a fetch under test never happens.""" + + def url(self, org_slug, opp_id): + return reverse( + "microplanning:buildings_geojson", + kwargs={"org_slug": org_slug, "opp_id": opp_id}, + ) + + def test_returns_feature_collection(self, client, org_user_admin, organization, opportunity): + feature = { + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": [[[8.65, 9.05], [8.66, 9.05], [8.66, 9.06], [8.65, 9.05]]]}, + "properties": {"id": "building-1"}, + } + client.force_login(org_user_admin) + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: { + grid_tiles[0]: [feature], + **{grid_tile: [] for grid_tile in grid_tiles[1:]}, + }, + ): + response = client.get( + self.url(organization.slug, opportunity.opportunity_id), {"bbox": "8.65,9.05,8.70,9.09"} + ) + + assert response.status_code == 200 + body = response.json() + assert body["type"] == "FeatureCollection" + assert body["features"] == [feature] + # The bbox reported back is the snapped area, so it always contains what was asked for. + west, south, east, north = body["bbox"] + assert west <= 8.65 and south <= 9.05 and east >= 8.70 and north >= 9.09 + + @pytest.mark.parametrize( + "bbox", + [None, "", "8.65,9.05,8.70", "8.65,9.05,8.70,north", "8.70,9.05,8.65,9.09"], + ) + def test_bad_bbox_is_rejected(self, client, org_user_admin, organization, opportunity, bbox): + client.force_login(org_user_admin) + params = {} if bbox is None else {"bbox": bbox} + response = client.get(self.url(organization.slug, opportunity.opportunity_id), params) + assert response.status_code == 400 + assert response.json()["error"] + + def test_too_large_an_area_is_rejected(self, client, org_user_admin, organization, opportunity): + client.force_login(org_user_admin) + response = client.get(self.url(organization.slug, opportunity.opportunity_id), {"bbox": "3.0,4.0,14.0,14.0"}) + assert response.status_code == 400 + assert "too large" in response.json()["error"] + + def test_unavailable_upstream_is_reported_as_503(self, client, org_user_admin, organization, opportunity): + client.force_login(org_user_admin) + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=BuildingDataUnavailable("overture is down"), + ): + response = client.get( + self.url(organization.slug, opportunity.opportunity_id), {"bbox": "8.65,9.05,8.70,9.09"} + ) + + assert response.status_code == 503 + assert response.json()["error"] + + def test_overture_failing_to_open_is_reported_as_503(self, client, org_user_admin, organization, opportunity): + """The package raises rather than returning None when it cannot resolve the release.""" + client.force_login(org_user_admin) + with patch( + "overturemaps.record_batch_reader", + side_effect=Exception("Could not fetch STAC catalog: "), + ): + response = client.get( + self.url(organization.slug, opportunity.opportunity_id), {"bbox": "8.65,9.05,8.70,9.09"} + ) + + assert response.status_code == 503 + + def test_a_bbox_too_narrow_to_cover_a_grid_tile_is_not_a_server_error( + self, client, org_user_admin, organization, opportunity + ): + """parse_bbox admits this, so it must reach the fetch rather than blowing up on empty grid tiles.""" + client.force_login(org_user_admin) + with patch( + "commcare_connect.microplanning.buildings.fetch_buildings_for_grid_tiles", + side_effect=lambda grid_tiles: {grid_tile: [] for grid_tile in grid_tiles}, + ): + response = client.get( + self.url(organization.slug, opportunity.opportunity_id), + {"bbox": "0.0,0.0,0.0000000000001,0.0000000000001"}, + ) + + assert response.status_code == 200 + + def test_requires_org_admin(self, client, org_user_member, organization, opportunity): + client.force_login(org_user_member) + response = client.get(self.url(organization.slug, opportunity.opportunity_id), {"bbox": "8.65,9.05,8.70,9.09"}) + assert response.status_code == 404 + + @pytest.mark.django_db class TestImplementationAreasGeojson(BaseMicroplanningFlagTest): def url(self, org_slug, opp_id): diff --git a/commcare_connect/microplanning/urls.py b/commcare_connect/microplanning/urls.py index b76b891781..42a38dfce2 100644 --- a/commcare_connect/microplanning/urls.py +++ b/commcare_connect/microplanning/urls.py @@ -51,6 +51,7 @@ views.implementation_areas_geojson, name="implementation_areas_geojson", ), + path("/buildings/", views.buildings_geojson, name="buildings_geojson"), path("/cluster_work_areas/", views.cluster_work_areas, name="cluster_work_areas"), path("/clear_work_areas/", views.clear_work_areas, name="clear_work_areas"), path("/clear_work_area_groups/", views.clear_work_area_groups, name="clear_work_area_groups"), diff --git a/commcare_connect/microplanning/views.py b/commcare_connect/microplanning/views.py index 9be657d1d5..7457b53429 100644 --- a/commcare_connect/microplanning/views.py +++ b/commcare_connect/microplanning/views.py @@ -47,7 +47,9 @@ from commcare_connect.commcarehq.api import create_or_update_case_by_work_area from commcare_connect.flags.flag_names import MICROPLANNING +from commcare_connect.microplanning.buildings import buildings_for_bbox, parse_bbox from commcare_connect.microplanning.const import ( + BUILDING_MIN_ZOOM, MAX_EXCLUDE_WORK_AREAS, MAX_UNASSIGN_WORK_AREAS, REQUIRED_DELIVER_UNIT_SLUGS, @@ -59,6 +61,7 @@ annotate_approved_visit_counts, missing_deliver_units, ) +from commcare_connect.microplanning.exceptions import AreaTooLarge, BuildingDataUnavailable from commcare_connect.microplanning.filters import ( CoverageProgressFilterSet, UserVisitMapFilterSet, @@ -197,6 +200,10 @@ def microplanning_home(request, *args, **kwargs): kwargs={"org_slug": request.org.slug, "opp_id": opportunity.opportunity_id}, ) + buildings_url = reverse( + "microplanning:buildings_geojson", kwargs={"org_slug": request.org.slug, "opp_id": opportunity.opportunity_id} + ) + work_area_detail_url = reverse( "microplanning:work_area_detail", args=[request.org.slug, opportunity.opportunity_id, 0], @@ -257,8 +264,10 @@ def microplanning_home(request, *args, **kwargs): "visit_tiles_url": visit_tiles_url, "groups_url": groups_url, "implementation_areas_url": implementation_areas_url, + "buildings_url": buildings_url, "status_meta": status_meta, "workarea_min_zoom": WORKAREA_MIN_ZOOM, + "building_min_zoom": BUILDING_MIN_ZOOM, "edit_work_area_url": edit_work_area_url, "user_visit_data_url": user_visit_data_url, "download_url": download_url, @@ -802,6 +811,28 @@ def implementation_areas_geojson(request, org_slug, opp_id): return JsonResponse({"implementation_area_features": features}) +# Reading from Overture takes seconds, and ATOMIC_REQUESTS would otherwise hold a database +# transaction open for all of it. This view only reads, so it does not need one. +@transaction.non_atomic_requests +@require_GET +@org_admin_required +@opportunity_required +@waffle_flag(MICROPLANNING) +def buildings_geojson(request, org_slug, opp_id): + try: + bbox = parse_bbox(request.GET.get("bbox")) + except ValueError as e: + return JsonResponse({"error": str(e)}, status=400) + + try: + return JsonResponse(buildings_for_bbox(*bbox)) + except AreaTooLarge: + return JsonResponse({"error": _("Requested area is too large.")}, status=400) + except BuildingDataUnavailable: + logger.exception("Could not fetch building data for bbox %s", bbox) + return JsonResponse({"error": _("Building data is unavailable right now.")}, status=503) + + @org_admin_required @opportunity_required @require_POST From bfac789845a2df7c5b7ef8d03f3fee8448256946 Mon Sep 17 00:00:00 2001 From: Charl Smit Date: Mon, 31 Aug 2026 16:26:16 +0200 Subject: [PATCH 3/7] Draw the footprints as a toggleable map overlay Adds a Show Building Data toggle over the map, on both the progress map and assignment mode, drawing footprints as a faint fill with an outline: the fill keeps the Work Area status colours underneath readable, and the outline is what makes small rural footprints visible at all. The layers are added after the Work Area layers so they draw on top. Footprints load for the viewport when the toggle goes on, and again whenever a pan or zoom takes the viewport outside the grid area already loaded, so moving to the next area loads it without the user asking. Each response adds to what is drawn rather than replacing it, so panning or zooming back never blanks an area they have already seen, and the map keeps a bounded number of areas so a long session roaming a country cannot grow without limit. One request is in flight at a time and the viewport is re-checked when it lands, so a few quick pans settle on where the user actually stopped instead of queueing stale fetches. A spinner is pinned over the area being fetched rather than to the screen, so it stays with the area it belongs to if they keep moving. The layers carry minzoom, as the other zoom-limited layers here do, and the control is absent below that zoom rather than disabled, so it only appears when it can be used. The footprints are kept, just not drawn, so zooming back in restores them without refetching. Co-Authored-By: Claude Opus 5 --- .../templates/microplanning/home.html | 5 +- .../templates/microplanning/map_handler.html | 207 ++++++++++++++++++ .../microplanning/map_layer_toggles.html | 20 ++ tailwind/tailwind.css | 8 + 4 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 commcare_connect/templates/microplanning/map_layer_toggles.html diff --git a/commcare_connect/templates/microplanning/home.html b/commcare_connect/templates/microplanning/home.html index 567b061e28..41425855c4 100644 --- a/commcare_connect/templates/microplanning/home.html +++ b/commcare_connect/templates/microplanning/home.html @@ -185,7 +185,7 @@

+ class="pointer-events-none absolute top-4 right-4 z-50 flex items-center gap-3 px-4 py-3 rounded-lg shadow-lg text-sm text-white opacity-0 transition-opacity duration-300">
@@ -193,10 +193,11 @@

+ class="absolute top-20 right-2 z-40 pointer-events-none px-2 py-2 rounded-md shadow-lg border border-white/10 text-sm text-white bg-black/50"> {% translate "Click to select group ยท Shift+click to select single area" %} {% endif %} + {% include "microplanning/map_layer_toggles.html" %}
diff --git a/commcare_connect/templates/microplanning/map_handler.html b/commcare_connect/templates/microplanning/map_handler.html index bb913e0cec..aae589c217 100644 --- a/commcare_connect/templates/microplanning/map_handler.html +++ b/commcare_connect/templates/microplanning/map_handler.html @@ -1,5 +1,42 @@ {% load i18n %}