Api route hookups

.github/workflows/etl.yml

@@ -0,0 +1,49 @@
+name: Vermont Data Collaborative ETL
+
+on:
+  schedule:
+    # Runs once a month on the 30th day (@2:00 AM)
+    - cron: '0 2 30 * *'
+  workflow_dispatch:
+
+jobs:
+  run-etl:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        version: '0.11.21'
+        enable-cache: true
+
+    - name: Install Podman
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y podman
+        podman --version
+
+    - name: Install Just and DuckDB
+      run: |
+        curl --proto '=https' --tlsv1.2 -sSf https://just.systems | bash -s -- --to /usr/local/bin
+        wget https://github.com
+
+    - name: Run ETL Pipeline
+      # Grab data just from the latest year to add to the database (2024)
+      run: just run-etl 2024 2024
+
+    - name: Commit and Push Updated DuckDB Database
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+        git add backend/Data/warehouse.duckdb
+        git diff-index --quiet HEAD || git commit -m "Automated ETL: Update DuckDB data [skip ci]"
+        git push

.gitignore

@@ -17,6 +17,8 @@ venv/
 backend/Data/Parcels/
 
 #DuckLake files
+
+# DuckLake catalogue
 backend/Data/lake
 backend/Data/lake.files/
 

README.md

@@ -8,13 +8,13 @@ A **React-based Website** for exploring, visualizing, and interpreting Vermont d
 
 Install these before you start. Every one of them is used by the standard workflow.
 
-| Tool                                          | Why it's needed                                                                                      |
-| --------------------------------------------- | ---------------------------------------------------------------------------------------------------- |
-| [git-lfs](https://git-lfs.com/)               | The datasets in `Data/` are tracked with Git LFS.                                                    |
-| [just](https://just.systems/man/en/)          | Task runner. Every dev command in this project is a `just` recipe (see [justfile](justfile)).        |
-| [uv](https://docs.astral.sh/uv/)              | Python dependency management and script running for the backend.                                     |
-| [Node + npm](https://nodejs.org/)             | Frontend dependencies and the Next.js dev server.                                                    |
-| [podman](https://podman.io/docs/installation) | Builds and runs the containerized stack.  |
+| Tool                                          | Why it's needed                                                                               |
+| --------------------------------------------- | --------------------------------------------------------------------------------------------- |
+| [git-lfs](https://git-lfs.com/)               | The datasets in `Data/` are tracked with Git LFS.                                             |
+| [just](https://just.systems/man/en/)          | Task runner. Every dev command in this project is a `just` recipe (see [justfile](justfile)). |
+| [uv](https://docs.astral.sh/uv/)              | Python dependency management and script running for the backend.                              |
+| [Node + npm](https://nodejs.org/)             | Frontend dependencies and the Next.js dev server.                                             |
+| [podman](https://podman.io/docs/installation) | Builds and runs the containerized stack.                                                      |
 
 > **Note:** podman is required even for the non-containerized workflow, because the `local-*` recipes call `just down` first to make sure a running container isn't already holding the ports.
 
@@ -137,11 +137,8 @@ This project is open-source under the **MIT License**.
 
 # VM Deployment
 
-
-
 1. sudo su - appuser0
 
-
 ## Credits
 
 - Developed by Ian Sargent and Fitzwilliam Keenan-Koch

backend/.sqlfluff

@@ -25,9 +25,9 @@ exclude_rules = CP02, CP03, RF04, RF05, ST06, ST07
 case_sensitive = True
 
 [sqlfluff:templater:jinja:context]
-table = acs5_b10_census
+table = acs5_demographics_tidy
 where_string = WHERE "Measure" IN ($1)
-cte_filter_block = WITH f0 AS (SELECT DISTINCT OBJECT_ID FROM zoning_info WHERE "County" IN ($1))
+cte_filter_block = WITH f0 AS (SELECT DISTINCT OBJECT_ID FROM VersoZoning_info WHERE "County" IN ($1))
 join_filter_block = JOIN f0 USING (OBJECT_ID)
 info_string = OBJECT_ID, County
 rule_string = CAST("Residential_Min_Lot" AS VARCHAR) AS residential_min_lot

backend/ETL/dockerfile.clean

@@ -22,7 +22,7 @@ COPY --chown=app:app ./api /code/api
 COPY --chown=app:app ./logger /code/logger
 # TODO: Remove the dependency on app_utils
 COPY --chown=app:app ./app_utils /code/app_utils
-COPY --chown=app:app ./datastore /code/datastore
+COPY --chown=app:app ./lake_build.py /code/lake_build.py
 
 
 ENV DATA_DIR=/data

backend/ETL/dockerfile.collect

@@ -21,7 +21,7 @@ COPY --chown=app:app ./run_data_collection.py /code/run_data_collection.py
 # TODO: Remove the dependency on app_utils
 COPY --chown=app:app ./app_utils /code/app_utils
 
-COPY --chown=app:app ./datastore /code/datastore
+COPY --chown=app:app ./lake_build.py /code/lake_build.py
 
 
 ENV DATA_DIR=/data

backend/ETL/dockerfile.lake

@@ -0,0 +1,24 @@
+FROM python:3.12-slim-trixie
+COPY --from=ghcr.io/astral-sh/uv:0.11.21 /uv /uvx /bin/
+
+# Set working directory
+WORKDIR /code
+RUN useradd -m -u 1000 app && chown app:app /code /home/app
+USER app
+
+# Dependencies
+COPY --chown=app:app pyproject.toml uv.lock /code/
+RUN --mount=type=cache,target=/home/app/.cache/uv,uid=1000,gid=1000 \
+    uv sync --frozen --no-dev --no-install-project
+
+# Spatial extension installation
+RUN uv run python -c "import duckdb; duckdb.connect().execute('INSTALL spatial')"
+
+# The source data collection scripts (source --> destination)
+COPY --chown=app:app ./lake_build.py /code/lake_build.py
+
+ENV DATA_DIR=/data
+ENV PYTHONPATH=/code
+
+# Command to run the DuckLake builder script (container entrypoint) 
+ENTRYPOINT ["uv", "run", "python", "-u", "-m", "lake_build"]

backend/ETL/dockerfile.load

@@ -19,7 +19,7 @@ COPY --chown=app:app ./run_data_loading.py /code/run_data_loading.py
 # TODO: Remove the dependency on app_utils
 COPY --chown=app:app ./app_utils /code/app_utils
 # For lake connection import
-COPY --chown=app:app ./datastore /code/datastore
+COPY --chown=app:app ./lake_build.py /code/lake_build.py
 
 
 ENV DATA_DIR=/data

backend/api/models/request_models.py

@@ -1,7 +1,10 @@
+from datetime import datetime
 from typing import Literal
 
 from pydantic import BaseModel, model_validator
 
+MAX_YEAR = datetime.now().year - 2
+
 
 class RangeFilter(BaseModel):
     min: float | None = None
@@ -30,7 +33,7 @@ class DPSeriesRequest(BaseModel):
     variable: str
     measure: str
     year_min: int = 2009
-    year_max: int = 2024
+    year_max: int = MAX_YEAR
 
 
 join_types = Literal["inner", "left", "spatial_intersect"]

backend/api/routes/get_routes/get_wholedata.py

@@ -5,6 +5,7 @@
 
 from app_utils import data_loading
 from app_utils.flooding import add_flood_color
+from query.production_db import DB
 
 logger = logging.getLogger(__name__)
 
@@ -16,14 +17,33 @@ def read_root():
     return {"Default Message": "No endpoint specified"}
 
 
-# Flood Endpoint (Hardcoded for now)
+# Flood Endpoint
 @router.get("/load/mapping/flood_legal")
 async def read_flood_data():
-    data = data_loading.masterload(name="flood_legal")
-    # Re-apply zone-based colors at serve time so the static JSON
-    # does not need to be regenerated when the color scheme changes.
-    data = add_flood_color(data)
-    return json.loads(data.to_json())
+    result = DB.execute("""--sql
+        SELECT
+            *,
+            ST_AsGeoJSON(geometry)::JSON AS geometry_json
+        FROM FEMA_floodHazard_geom
+    """).df()
+
+    result = add_flood_color(result)
+
+    features = []
+    for _, row in result.iterrows():
+        properties = row.drop(["geometry", "geometry_json"]).to_dict()
+        features.append(
+            {
+                "type": "Feature",
+                "geometry": json.loads(row["geometry_json"]),
+                "properties": properties,
+            }
+        )
+
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+    }
 
 
 # Soil Septic Endpoint (Hardcoded for now)