Generate impact reports in PDF and PNG formats

.github/workflows/engine_nightly_webui.yml

@@ -51,9 +51,16 @@ jobs:
           set -x
           date
           wget https://downloads.openquake.org/test_data/exposure.hdf5
+          wget https://downloads.openquake.org/test_data/worldcities.csv
+          wget https://downloads.openquake.org/test_data/countries_info.csv
+          wget https://downloads.openquake.org/test_data/World_Adm1_updated.gpkg
+          wget https://downloads.openquake.org/test_data/fonts.zip
+          unzip fonts.zip
           date
           mv exposure.hdf5 openquake/qa_tests_data/mosaic/
           source ~/openquake/bin/activate
+          PY_VER=$(echo "${{ matrix.python-version }}" | tr -d '.')
+          pip install -r requirements-impact-py${PY_VER}-linux64.txt
           oq engine --upgrade-db
           touch ~/webui-access.log
           pip install https://wheelhouse.openquake.org/v3/py/pytest_django-4.9.0-py3-none-any.whl

.github/workflows/engine_pr_test.yml

@@ -37,7 +37,7 @@ jobs:
         run: |
           set -e
           source ~/openquake/bin/activate
-          pip install https://wheelhouse.openquake.org/v3/py/rtgmpy-1.0.1-py3-none-any.whl
+          pip install -r requirements-aelo-py-linux64.txt
           oq engine --upgrade-db
           pytest --doctest-modules -x --disable-warnings --color=yes --durations=10 openquake/calculators -k "not classical"
 
@@ -70,7 +70,8 @@ jobs:
         run: |
           set -e
           source ~/openquake/bin/activate
-          pip install pyshp pytest flake8 ruff https://wheelhouse.openquake.org/v3/py/rtgmpy-1.0.1-py3-none-any.whl
+          pip install -r requirements-aelo-py-linux64.txt
+          pip install pyshp pytest flake8 ruff
           oq engine --upgrade-db
           ruff check openquake
           curl https://downloads.openquake.org/test_data/exposure.hdf5 -o openquake/qa_tests_data/mosaic/exposure.hdf5
@@ -110,7 +111,14 @@ jobs:
           source ~/openquake/bin/activate
           oq engine --upgrade-db
           pip install https://wheelhouse.openquake.org/v3/py/pytest_django-4.9.0-py3-none-any.whl
+          pip install -r requirements-impact-py311-linux64.txt
+          pip install -r requirements-aelo-py-linux64.txt
           curl https://downloads.openquake.org/test_data/exposure.hdf5 -o openquake/qa_tests_data/mosaic/exposure.hdf5; ln -s openquake/qa_tests_data/mosaic/exposure.hdf5
+          wget https://downloads.openquake.org/test_data/worldcities.csv
+          wget https://downloads.openquake.org/test_data/countries_info.csv
+          wget https://downloads.openquake.org/test_data/World_Adm1_updated.gpkg
+          wget https://downloads.openquake.org/test_data/fonts.zip
+          unzip fonts.zip
           OQ_APPLICATION_MODE=PUBLIC pytest -vx openquake/server/tests/test_public_mode.py
           OQ_APPLICATION_MODE=RESTRICTED pytest -vx openquake/server/tests/test_restricted_mode.py
           OQ_APPLICATION_MODE=READ_ONLY pytest -vx openquake/server/tests/test_read_only_mode.py

.github/workflows/engine_weekly_test.yml

@@ -417,6 +417,9 @@ jobs:
           else
             python install.py devel --version master
           fi
+          PY_VER=`echo py${{ matrix.python-version }} | tr -d .`
+          echo $PY_VER
+          pip3 install -r requirements-impact-$PY_VER-linux64.txt
       - name: Actualize 'impact' templates for email notifications
         run: |
           for file in ./openquake/server/templates/registration/*.impact.tmpl; do
@@ -429,6 +432,11 @@ jobs:
           set -x
           date
           wget https://downloads.openquake.org/test_data/exposure.hdf5
+          wget https://downloads.openquake.org/test_data/worldcities.csv
+          wget https://downloads.openquake.org/test_data/countries_info.csv
+          wget https://downloads.openquake.org/test_data/World_Adm1_updated.gpkg
+          wget https://downloads.openquake.org/test_data/fonts.zip
+          unzip fonts.zip
           date
           mv exposure.hdf5 openquake/qa_tests_data/mosaic/
           source ~/openquake/bin/activate

bin/extract_impact_reports.py

@@ -0,0 +1,35 @@
+import sys
+import subprocess
+from pathlib import Path
+from openquake.commonlib import datastore
+
+
+def main(calc_id: int):
+    dstore = datastore.read(calc_id)
+    if "impact" not in dstore:
+        raise RuntimeError("No 'impact' group found in datastore")
+    impact_group = dstore["impact"]
+    iso3_codes = list(impact_group.keys())
+    if not iso3_codes:
+        raise RuntimeError("No ISO3 reports found under 'impact/'")
+    print(f"Found ISO3 reports: {iso3_codes}")
+    for iso3 in iso3_codes:
+        country_group = impact_group[iso3]
+        if "report_pdf" not in country_group:
+            print(f"No report_pdf found for {iso3}")
+            continue
+        pdf_bytes = country_group["report_pdf"][()]
+        fname = Path(f"impact_report_{calc_id}_{iso3}.pdf")
+        with open(fname, "wb") as f:
+            f.write(pdf_bytes)
+        print(f"Wrote {fname}")
+        subprocess.Popen(["xdg-open", str(fname)])
+    dstore.close()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: extract_impact_reports.py <calc_id>")
+        sys.exit(1)
+
+    main(int(sys.argv[1]))

openquake/calculators/event_based.py

@@ -46,6 +46,7 @@
     RuptureProxy, EBRupture, get_ruptures_aw)
 from openquake.hazardlib.shakemap.parsers import adjust_hypocenter
 from openquake.commonlib import util, logs, readinput, datastore
+from openquake.commonlib.readinput import get_close_mosaic_models
 from openquake.commonlib.calc import (
     gmvs_to_poes, make_hmaps, slice_dt, build_slice_by_event, RuptureImporter,
     SLICE_BY_EVENT_NSITES, get_proxies, get_model_lts)
@@ -589,7 +590,7 @@ def read_gsim_lt(oq):
             elif oq.rupture_xml:
                 hypo = readinput.get_rupture(oq).hypocenter
                 lon, lat = [hypo.x, hypo.y]
-            mosaic_models = readinput.get_close_mosaic_models(lon, lat, 5)
+            mosaic_models = get_close_mosaic_models(lon, lat, 5)
             # NOTE: using the first mosaic model
             oq.mosaic_model = mosaic_models[0]
             if len(mosaic_models) > 1:

openquake/calculators/export/risk.py

@@ -26,11 +26,11 @@
 from openquake.baselib import hdf5, writers, general, node
 from openquake.baselib.general import decode
 from openquake.hazardlib import nrml
-from openquake.hazardlib.stats import compute_stats2, mean_curve
+from openquake.hazardlib.stats import mean_curve
 from openquake.risklib import scientific
 from openquake.commonlib import readinput
 from openquake.calculators.extract import (
-    extract, sanitize, avglosses, aggexp_tags)
+    extract, sanitize, _get_data, aggexp_tags)
 from openquake.calculators import base, post_risk
 from openquake.calculators.export import export, loss_curves
 from openquake.calculators.export.hazard import savez
@@ -225,32 +225,6 @@ def export_exposure_by_lse(ekey, dstore):
     return [dest]
 
 
-def _get_data(dstore, dskey, loss_types, stats):
-    name, kind = dskey.split('-')  # i.e. ('avg_losses', 'stats')
-    if kind == 'stats':
-        try:
-            weights = dstore['weights'][()]
-        except KeyError:
-            # there is single realization, like in classical_risk/case_2
-            weights = [1.]
-        if dskey in set(dstore):  # precomputed
-            rlzs_or_stats = list(stats)
-            statfuncs = [stats[ros] for ros in stats]
-            value = avglosses(dstore, loss_types, 'stats')  # shape (A, S, L)
-        elif dstore['oqparam'].collect_rlzs:
-            rlzs_or_stats = list(stats)
-            value = avglosses(dstore, loss_types, 'rlzs')
-        else:  # compute on the fly
-            rlzs_or_stats, statfuncs = zip(*stats.items())
-            value = compute_stats2(
-                avglosses(dstore, loss_types, 'rlzs'), statfuncs, weights)
-    else:  # rlzs
-        value = avglosses(dstore, loss_types, kind)  # shape (A, R, L)
-        R = value.shape[1]
-        rlzs_or_stats = ['rlz-%03d' % r for r in range(R)]
-    return name, value, rlzs_or_stats
-
-
 # this is used by event_based_risk, classical_risk and scenario_risk
 @export.add(('avg_losses-rlzs', 'csv'), ('avg_losses-stats', 'csv'))
 def export_avg_losses(ekey, dstore):
@@ -856,7 +830,8 @@ def export_fragility_xml(dstore):
     crm = dstore.read_df('crm')
     ddic = {peril: {} for peril in crm.peril.unique()}
     for (peril, loss_type), df in crm.groupby(['peril', 'loss_type']):
-        nodeobj = convert_df_to_fragility(peril, loss_type, oq.limit_states, df)
+        nodeobj = convert_df_to_fragility(
+            peril, loss_type, oq.limit_states, df)
         dest = dstore.export_path('%s_%s_fragility.xml' % (peril, loss_type))
         with open(dest, 'wb') as out:
             nrml.write([nodeobj], out)

openquake/calculators/extract.py

@@ -38,7 +38,7 @@
 from openquake.hazardlib.contexts import (
     ContextMaker, read_cmakers, read_ctx_by_grp)
 from openquake.hazardlib.calc import disagg, stochastic, filters
-from openquake.hazardlib.stats import calc_stats
+from openquake.hazardlib.stats import calc_stats, compute_stats2
 from openquake.hazardlib.source import rupture
 from openquake.risklib.scientific import LOSSTYPE, LOSSID
 from openquake.risklib.asset import tagset
@@ -867,15 +867,15 @@ def extract_mmi_tags(dstore, what):
 
 def ensure_npy_serializable(df):
     """
-    Cast object-dtype columns of a DataFrame to fixed-width numpy byte
-    strings (dtype ``|S<n>``) so that a structured array built from it
-    is serializable with ``allow_pickle=False``.
+    Cast object-dtype and categorical columns of a DataFrame to fixed-width
+    numpy byte strings (dtype ``|S<n>``) so that a structured array built
+    from it is serializable with ``allow_pickle=False``.
 
     :param df: a :class:`pandas.DataFrame`
-    :returns: the same DataFrame with object columns replaced in-place
+    :returns: the same DataFrame with string columns replaced in-place
     """
     for col in df.columns:
-        if df[col].dtype != object:
+        if df[col].dtype != object and df[col].dtype.name != 'category':
             continue
         # Explicitly encode strings to UTF-8 bytes to handle non-ASCII
         # characters (e.g., 'é', 'ñ') before casting to the fixed-width byte
@@ -1942,6 +1942,52 @@ def extract_high_sites(dstore, what):
     return (max_hazard > .2).all(axis=1)  # shape N
 
 
+def _get_data(dstore, dskey, loss_types, stats):
+    name, kind = dskey.split('-')  # i.e. ('avg_losses', 'stats')
+    if kind == 'stats':
+        try:
+            weights = dstore['weights'][()]
+        except KeyError:
+            # there is single realization, like in classical_risk/case_2
+            weights = [1.]
+        if dskey in set(dstore):  # precomputed
+            rlzs_or_stats = list(stats)
+            statfuncs = [stats[ros] for ros in stats]
+            value = avglosses(dstore, loss_types, 'stats')  # shape (A, S, L)
+        elif dstore['oqparam'].collect_rlzs:
+            rlzs_or_stats = list(stats)
+            value = avglosses(dstore, loss_types, 'rlzs')
+        else:  # compute on the fly
+            rlzs_or_stats, statfuncs = zip(*stats.items())
+            value = compute_stats2(
+                avglosses(dstore, loss_types, 'rlzs'), statfuncs, weights)
+    else:  # rlzs
+        value = avglosses(dstore, loss_types, kind)  # shape (A, R, L)
+        R = value.shape[1]
+        rlzs_or_stats = ['rlz-%03d' % r for r in range(R)]
+    return name, value, rlzs_or_stats
+
+
+@extract.add('avg_losses')
+def extract_avg_losses(dstore, what):
+    """
+    Example:
+    http://127.0.0.1:8800/v1/calc/30/extract/avg_losses?kind=stats
+    """
+    oq = dstore['oqparam']
+    qdict = parse(what)
+    dskey = f"avg_losses-{qdict['kind'][0]}"
+    name, value, rlzs_or_stats = _get_data(
+        dstore, dskey, oq.ext_loss_types, oq.hazard_stats())
+    assets = util.get_assets(dstore)
+    dt = [(lt, F32) for lt in oq.ext_loss_types]
+    for ros, values in zip(rlzs_or_stats, value.transpose(1, 0, 2)):
+        array = numpy.zeros(len(values), dt)
+        for lt, ln in enumerate(oq.ext_loss_types):
+            array[ln] = values[:, lt]
+        yield ros, util.compose_arrays(assets, array)
+
+
 # #####################  extraction from the WebAPI ###################### #
 
 class WebAPIError(RuntimeError):