diff --git a/backend/Data/flood-hazard/VT_Flood b/backend/Data/flood-hazard/VT_Flood deleted file mode 100644 index d3f5a12f..00000000 --- a/backend/Data/flood-hazard/VT_Flood +++ /dev/null @@ -1 +0,0 @@ - diff --git a/backend/Data/flood-hazard/table_build.qmd b/backend/Data/flood-hazard/table_build.qmd new file mode 100644 index 00000000..aa753994 --- /dev/null +++ b/backend/Data/flood-hazard/table_build.qmd @@ -0,0 +1,87 @@ +--- +title: "Flood Hazard Table Build" +format: html +--- + +## Load dataset + +```{python} +import duckdb +import pandas as pd + +con = duckdb.connect("flood.db") +raw = con.execute("SELECT * FROM raw").df() +raw.head() + +raw.isna().sum().sort_values(ascending=False) + +for col in raw.columns: + print("COLUMN:", col) + try: + print(raw[col].value_counts().head()) + except Exception as e: + print("Could not compute value counts:", e) + print("\n") + + +``` + +## Null values + +```{python} +for col in raw.columns: + print(col) + print(raw[col].value_counts(dropna=False).head(10)) + print("\n") +import numpy as np + +raw["VELOCITY"] = raw["VELOCITY"].replace(-9999, np.nan) + + +``` + +## Dropping columns + +```{python} + + +drop_cols = [ + "AR_SUBTRV", + "BFE_REVERT", + "DEP_REVERT" +] + +drop_cols + +info_cols = [ + "OBJECTID", + "AR_REVERT", + "DFIRM_ID", + "VERSION_ID", + "FLD_AR_ID", + "STUDY_TYP", + "FLD_ZONE", + "ZONE_SUBTYP", + "SFHA_TF", + "STATIC_BFE", + "V_DATUM", + "DEPTH", + "LEN_UNIT", + "VELOCITY", + "VEL_UNIT", + "DUAL_ZONE", + "SOURCE_CIT" +] + +info_cols + +``` + +```{python} +geom_cols = ["OBJECTID", "geometry"] +geom_cols + +final_cols = info_cols + ["geometry"] +final_cols + +``` \ No newline at end of file diff --git a/backend/Data/flood-hazard/vt-flood-hazard.parquet b/backend/Data/flood-hazard/vt-flood-hazard.parquet new file mode 100644 index 00000000..a1161ae4 Binary files /dev/null and b/backend/Data/flood-hazard/vt-flood-hazard.parquet differ diff --git a/backend/build/flooding.py b/backend/build/flooding.py new file mode 100644 index 00000000..6302cc12 --- /dev/null +++ b/backend/build/flooding.py @@ -0,0 +1,30 @@ +from build import BACKEND, CON, data_dir + +PARQUET = data_dir / "Flooding" / "vt-flood-hazard.parquet" +proc_dir = BACKEND / "Data" / "_Processed" / "flood" +SQL_DIR = BACKEND / "build" / "sql" + + +def main(): + + # Load raw table from parquet + CON.execute(f""" + CREATE OR REPLACE VIEW raw AS + SELECT * + FROM read_parquet('{PARQUET}') + """) + + # Run SQL files in order + for name in ["flood_info.sql", "flood_geom.sql", "flood_build.sql"]: + CON.execute((SQL_DIR / name).read_text()) + + for table in ["flood_hazard"]: + CON.execute( + f"COPY (SELECT * FROM {table}) TO '{proc_dir / 'Flooding' / f'{table}.parquet'}' " + ) + + print("Flood hazard tables built.") + + +if __name__ == "__main__": + main() diff --git a/backend/build/sql/flood_build.sql b/backend/build/sql/flood_build.sql new file mode 100644 index 00000000..173e6cbb --- /dev/null +++ b/backend/build/sql/flood_build.sql @@ -0,0 +1,21 @@ +CREATE OR REPLACE TABLE hazard AS +SELECT + i.id, + i.DFIRM_ID, + i.FLD_AR_ID, + i.STUDY_TYP, + i.FLD_ZONE, + i.ZONE_SUBTY, + i.SFHA_TF, + i.STATIC_BFE, + i.V_DATUM, + i.DEPTH, + i.LEN_UNIT, + i.VELOCITY, + i.VEL_UNIT, + i.AR_REVERT, + i.DUAL_ZONE, + i.SOURCE_CIT, + g.geometry AS geom +FROM info AS i +INNER JOIN geom AS g USING (id); diff --git a/backend/build/sql/flood_geom.sql b/backend/build/sql/flood_geom.sql new file mode 100644 index 00000000..5d8b6f11 --- /dev/null +++ b/backend/build/sql/flood_geom.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE TABLE geom AS +SELECT + OBJECTID AS id, + geometry +FROM raw; diff --git a/backend/build/sql/flood_info.sql b/backend/build/sql/flood_info.sql new file mode 100644 index 00000000..6cecfae1 --- /dev/null +++ b/backend/build/sql/flood_info.sql @@ -0,0 +1,54 @@ +CREATE OR REPLACE TABLE info AS +SELECT + + -- Unique ID assigned by FEMA + OBJECTID AS id, + + -- Indicates whether the flood zone is under heightened risk + AR_REVERT, + + -- FEMA Digital Flood Insurance Rate Map identifier + DFIRM_ID, + + -- Version number of the FEMA flood hazard dataset + VERSION_ID, + + -- Unique flood area identifier used by FEMA + FLD_AR_ID, + + -- Type of flood study conducted + STUDY_TYP, + + -- FEMA flood zone designation + FLD_ZONE, + + --Subtype of the flood zone providing additional classification detail + ZONE_SUBTY, + + -- Indicates whether the area is a Special Flood Hazard Area + SFHA_TF, + + -- Base Flood Elevation in feet + STATIC_BFE, + + -- Vertical datum used for the BFE measurement + V_DATUM, + + -- Flood depth in feet + DEPTH, + + -- Unit of measurement for linear values + LEN_UNIT, + + -- Flood velocity in feet per second + VELOCITY, + + -- Unit of measurement for velocity values + VEL_UNIT, + + -- Indicates whether the area is part of dual-zone classification + DUAL_ZONE, + + -- Citation or reference for the source of the flood hazard data + SOURCE_CIT +FROM raw; diff --git a/backend/notebooks/Flooding /table_build.qmd b/backend/notebooks/Flooding /table_build.qmd new file mode 100644 index 00000000..9cba13de --- /dev/null +++ b/backend/notebooks/Flooding /table_build.qmd @@ -0,0 +1,220 @@ +--- +title: "Flood Hazard Table Build" +author: Fitz Koch and Ethan Elbert +description: Builds SQL tables from flood hazard data. +format: + html: + html-math-method: mathjax + fig-responsive: true + toc: true + toc-location: left + theme: cosmo + page-layout: full + ipynb: + wrap: none +--- + +# Setup + +Navigate to root: +```{python} +import sys, os +from pathlib import Path + +_project_root = Path.cwd() +while not (_project_root / "api").exists(): + _project_root = _project_root.parent +os.chdir(_project_root) +print(os.getcwd()) +``` + + +```{python} +import pandas as pd +import geopandas as gpd +import pyogrio +from pathlib import Path +import duckdb +import re + +from app_utils.data_loading import crs_set + +``` + +```{python} +con = duckdb.connect() +``` + +## Load dataset + +```{python} +con.execute("INSTALL spatial ; LOAD spatial") +``` + +```{python} +con.execute(f""" + CREATE OR REPLACE TABLE raw AS + SELECT * + FROM read_parquet("Data/flood-hazard/vt-flood-hazard.parquet") +""") +``` + +```{python} +raw = con.execute("SELECT * FROM raw").df() +raw.head() + +raw.isna().sum().sort_values(ascending=False) + +for col in raw.columns: + print("COLUMN:", col) + try: + print(raw[col].value_counts().head()) + except Exception as e: + print("Could not compute value counts:", e) + print("\n") +``` + +# Analysis +```{python} +con.execute("""DESCRIBE raw""").df() +``` + +Per the describe, the dataset contains the following columns: + +- OBJECTID (int32) — unique feature identifier +- DFIRM_ID (object) — FEMA map ID +- VERSION_ID (object) +- FLD_AR_ID (object) +- STUDY_TYP (object) +- FLD_ZONE (object) +- ZONE_SUBTYP (object) +- SFHA_TF (object) +- STATIC_BFE (int32) +- V_DATUM (object) +- DEPTH (int32) +- LEN_UNIT (object) +- VELOCITY (int32) +- VEL_UNIT (object) +- AR_REVERT (object) +- AR_SUBTRV (int32) +- BFE_REVERT (int32) +- DEP_REVERT (int32) +- DUAL_ZONE (Int32) +- SOURCE_CIT (object) +- geometry (object) + + +Now we want to look at the what these columns mean. First, go to [this link](https://www.fema.gov/sites/default/files/2020-02/FIRM_Database_Technical_Reference_Feb_2019.pdf) and search (ctrl + f) for the column names, then see what values we actually have. + +## study type +For example, for the column `STUDY_TYP`, we can see that it is the study type based on the manual. Then we run a query to see what value counts we have for that column. + +```{python} +df = con.execute("SELECT * FROM raw").df() +df["STUDY_TYP"].value_counts() +``` + +All the values are `NP`, which means "No Study". This is not very useful, so we we'll make a mental note to drop this column. + + +## static_bfe +For another example, we'll check out another column, the `STATIC_BFE` column. This is the static base flood elevation, which is the elevation of the base flood in feet above the vertical datum. That is, the area where there is *always* a flood, which are basically in lakes and costal zones. To quote the manual, + +> Static Base Flood Elevation. This field will be populated for areas that have been determined to have a constant Base Flood Elevation (BFE) over a flood zone. The BFE value will be shown beneath the zone label. In this situation the same BFE applies to the entire polygon. This normally occurs in lakes or coastal zones. + +```{python} +df["STATIC_BFE"].value_counts() +``` + +Hmm, this is kind of weird. Lots of `-9999`... let's check the manual (see above) to see what that means... okay, so it means the null value. That will be annoying if we want to do any kind of math on this, so we'll convert these to NULL in a format that makes sense for DuckDB. Namely, we'll convert them to `NULL` in the SQL table creation step. + +For a preview of this, where we are updating the `raw` table instead of creating a new table (which is what we'll do in the final SQL file), we can run the following: + +```{python} +con.execute("SELECT STATIC_BFE, COUNT(*) FROM raw GROUP BY STATIC_BFE").df() +``` + + +This code is good, we want to include it in our final version. +```{python} +con.execute(""" + UPDATE raw + SET STATIC_BFE = NULL + WHERE STATIC_BFE = -9999 +""") +``` + + + + +```{python} +con.execute("SELECT STATIC_BFE, COUNT(*) FROM raw GROUP BY STATIC_BFE").df() +``` + +```{python} +df = con.execute("SELECT * FROM raw").df() +df["STATIC_BFE"].value_counts(dropna=False) +``` + +Okay, that looks better, now they're correctly stored as Null values. + + +## sfha + +Basically this column means whether the area is in a Special Flood Hazard Area (SFHA) or not. The values are either `Y` or `N`, which is pretty straightforward. We don't see any nulls or anything that doesn't make sense, so we can leave this column as is. + +```{python} +con.execute(""" + SELECT SFHA_TF, COUNT(*) + FROM raw + GROUP BY SFHA_TF +""").df() +``` + + +## Metadata tables. + +```{python} +con.execute(""" + CREATE OR REPLACE TABLE metadata AS + SELECT VERSION_ID, STUDY_TYP, FLD_ZONE, ZONE_SUBTYP, SFHA_TF, V_DATUM +""") +``` + + + +# Ethan Analysis + +Ethan, please do similar checks for the **all** the other columns, either cleaning the data (e.g., converting -9999 to NULL) or dropping columns that are not useful. + +Once you have figured out all that logic, combine it into a single SQL file that creates the final tables. This is what goes in the 'backend/build/sql' folder. Then we can run the build script to create the final tables (I moved them there, you can update them once you've done this). + + +# DuckDB, SQL, and Pandas + +```{python} +raw = con.execute("SELECT * FROM raw").df() +``` + +```{python} +con.execute("SELECT * FROM raw").df() +``` + +```{python} +import numpy as np +raw["VELOCITY"] = raw["VELOCITY"].replace(-9999, np.nan) +raw["VELOCITY"].value_counts(dropna=False) +``` + +```{python} +con.execute(""" + SELECT VELOCITY, COUNT(*) + FROM raw + GROUP BY VELOCITY +""").df() +``` + +```{python} +raw_new = con.execute("SELECT * FROM raw").df() +raw_new["VELOCITY"].value_counts(dropna=False) +``` \ No newline at end of file