From d737a9dd3baad737f4c5a39c131135e7c874a17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 10:47:32 -0400 Subject: [PATCH 01/76] fix: add small fixes to create Items out of RDPS and HRDPS data sets from PR #104 --- CHANGES.md | 6 +- STACpopulator/extensions/datacube.py | 2 +- STACpopulator/extensions/thredds.py | 15 + STACpopulator/input.py | 2 +- tests/data/cordex6_ncml.json | 2 +- tests/data/cordex6_raw.json | 2 +- tests/data/hrdps_p_tt.json | 197 +++++++++++ tests/data/hrdps_sfc.json | 197 +++++++++++ tests/data/rdps.json | 489 +++++++++++++++++++++++++++ tests/data/update_data.py | 50 +++ tests/test_cordex.py | 33 -- tests/test_rdps.py | 28 ++ 12 files changed, 985 insertions(+), 38 deletions(-) create mode 100644 tests/data/hrdps_p_tt.json create mode 100644 tests/data/hrdps_sfc.json create mode 100644 tests/data/rdps.json create mode 100644 tests/data/update_data.py create mode 100644 tests/test_rdps.py diff --git a/CHANGES.md b/CHANGES.md index 9a275d0..e44cbb1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,11 @@ * Fix bug in `THREDDSLoader` iterator introduced in 0.8.0. Simple iteration over `catalog_refs` returns the catalog names (strings), whereas we want an object with a `follow` method. * Remove `log_debug` option from the `CORDEXCMIP6_Ouranos` runner. * Add attributes to CORDEX IDs to avoid duplicate IDs in the STAC catalog. -* Update test data for `CORDEXCMIP6_Ouranos`. +* Update test data for `CORDEXCMIP6_Ouranos`. +* Add default `create_uid` to `THREDDSCatalogDataModel`. +* Fix bug in `DataCubeHelper` for vertical coordinate extents. +* Split and clean script to update test data. +* Add tests for RDPS and HRDPS attributes with no custom extension. * Make sure that all tests will pass even when not connected to the internet. ## [0.8.0](https://github.com/crim-ca/stac-populator/tree/0.8.0) (2025-06-11) diff --git a/STACpopulator/extensions/datacube.py b/STACpopulator/extensions/datacube.py index 24be404..6167acc 100644 --- a/STACpopulator/extensions/datacube.py +++ b/STACpopulator/extensions/datacube.py @@ -166,7 +166,7 @@ def dimensions(self) -> dict[str, Dimension]: elif key in ["T", "time"]: extent = self.temporal_extent() else: - extent = ["", ""] + extent = [None, None] properties = dict( type=type_.value, diff --git a/STACpopulator/extensions/thredds.py b/STACpopulator/extensions/thredds.py index c4390c3..c352c2b 100644 --- a/STACpopulator/extensions/thredds.py +++ b/STACpopulator/extensions/thredds.py @@ -209,6 +209,21 @@ def thredds_helper(cls, data: dict) -> dict: data["thredds"] = THREDDSHelper(data["data"]["access_urls"]) return data + def create_uid(self) -> str: + """Return a unique ID from the server location. + + For datasets with a DRS, it might might more sense to use the dataset's metadata instead. + """ + if "HTTPServer" in self.data["access_urls"]: + location = self.data["access_urls"]["HTTPServer"].split("/fileServer/")[1] + elif "OpenDAP" in self.data["access_urls"]: + location = self.data["access_urls"]["OPENDAP"].split("/dodsC/")[1] + elif "NCML" in self.data["access_urls"]: + location = self.data["access_urls"]["NCML"].split("/ncml/")[1] + else: + raise ValueError("No valid access URL found in data.") + return location.replace("/", "__") + # TODO: Validate services links exist ? # @field_validator("access_urls") diff --git a/STACpopulator/input.py b/STACpopulator/input.py index f90ff1a..584f671 100644 --- a/STACpopulator/input.py +++ b/STACpopulator/input.py @@ -103,7 +103,7 @@ def __iter__(self) -> Iterator[Tuple[str, str, MutableMapping[str, Any]]]: url = f"{current_catalog.catalog_url}?dataset={dataset.id}" yield item_name, url, attrs - for name, ref in current_catalog.catalog_refs.items(): + for ref in current_catalog.catalog_refs.values(): catalogs.put(ref.follow()) def __getitem__(self, dataset: str) -> dict: diff --git a/tests/data/cordex6_ncml.json b/tests/data/cordex6_ncml.json index 8b03ed6..33122bc 100644 --- a/tests/data/cordex6_ncml.json +++ b/tests/data/cordex6_ncml.json @@ -88,7 +88,7 @@ }, "NCISOMetadata": { "attributes": { - "metadata_creation": "2025-06-25", + "metadata_creation": "2025-07-01", "nciso_version": "2.4.6" } } diff --git a/tests/data/cordex6_raw.json b/tests/data/cordex6_raw.json index 81b66f6..2ced795 100644 --- a/tests/data/cordex6_raw.json +++ b/tests/data/cordex6_raw.json @@ -81,7 +81,7 @@ }, "NCISOMetadata": { "attributes": { - "metadata_creation": "2025-06-25", + "metadata_creation": "2025-07-01", "nciso_version": "2.4.6" } } diff --git a/tests/data/hrdps_p_tt.json b/tests/data/hrdps_p_tt.json new file mode 100644 index 0000000..e229a96 --- /dev/null +++ b/tests/data/hrdps_p_tt.json @@ -0,0 +1,197 @@ +{ + "@location": "Not provided because of security concerns.", + "@xmlns": { + "": "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" + }, + "attributes": { + "_NCProperties": "version=2,netcdf=4.9.2,hdf5=1.14.2", + "Remarks": "Variable names are following the convention ___. Variables with level '10000' are at surface level. The height [m] of variables with level '0XXXX' needs to be inferrred using the corresponding fields of geopotential height (GZ_0XXXX-GZ_10000). The variables UUC, VVC, UVC, and WDC are not modelled but inferred from UU and VV for convenience of the users. Precipitation (PR) is reported as 6-hr accumulations for CaPA_fine and CaPA_coarse. Precipitation (PR) are accumulations since beginning of the forecast for GEPS, GDPS, REPS, RDPS, HRDPS, and CaLDAS. The re-analysis product RDRS_v2 contains two variables for precipitation: 'RDRS_v2_P_PR0_SFC' is the model precipitation (trial field used by CaPA) and 'RDRS_v2_A_PR0_SFC' is precipitations adjusted with CaPA 24h precipitation (as in 'RDRS_P_PR0_SFC'). Please be aware that the baseflow 'O1' of the current version of WCPS is not reliable during the spring melt period.", + "License": "These data are provided by the Canadian Surface Prediction Archive CaSPar. You should have received a copy of the License agreement with the data. Otherwise you can find them under http://caspar-data.ca/doc/caspar_license.txt or email caspar.data@uwaterloo.ca.", + "product": "HRDPS", + "Conventions": "CF-1.6", + "_CoordSysBuilder": "ucar.nc2.internal.dataset.conv.CF1Convention" + }, + "dimensions": { + "rlon": 2540, + "rlat": 1290, + "time": 13 + }, + "groups": { + "CFMetadata": { + "attributes": { + "geospatial_lon_min": [ + -152.7306365966797 + ], + "geospatial_lat_min": [ + 27.284589767456055 + ], + "geospatial_lon_max": [ + -40.708560943603516 + ], + "geospatial_lat_max": [ + 70.61150360107422 + ], + "geospatial_lon_units": "degrees_east", + "geospatial_lat_units": "degrees_north", + "geospatial_lon_resolution": "3.418852275416474E-5", + "geospatial_lat_resolution": "1.3223137054096467E-5", + "time_coverage_start": "2024-01-01T00:00:00Z", + "time_coverage_end": "2024-01-01T12:00:00Z", + "time_coverage_units": "seconds", + "time_coverage_resolution": "3600.0", + "time_coverage_duration": "P0Y0M0DT12H0M0.000S" + } + }, + "NCISOMetadata": { + "attributes": { + "metadata_creation": "2025-07-01", + "nciso_version": "2.4.6" + } + } + }, + "variables": { + "rlon": { + "shape": [ + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "longitude in rotated pole grid", + "units": "degrees", + "eccc_grid_definition": "grtyp: E, ig1: 1430, ig2: 500, ig3: 56000, ig4: 44000", + "standard_name": "grid_longitude", + "axis": "X", + "_ChunkSizes": [ + 2540 + ], + "_CoordinateAxisType": "GeoX" + } + }, + "rlat": { + "shape": [ + "rlat" + ], + "type": "float", + "attributes": { + "long_name": "latitude in rotated pole grid", + "units": "degrees", + "eccc_grid_definition": "grtyp: E, ig1: 1430, ig2: 500, ig3: 56000, ig4: 44000", + "standard_name": "grid_latitude", + "axis": "Y", + "_ChunkSizes": [ + 1290 + ], + "_CoordinateAxisType": "GeoY" + } + }, + "time": { + "shape": [ + "time" + ], + "type": "int", + "attributes": { + "long_name": "time", + "units": "hours since 2024-01-01 00:00:00", + "calendar": "gregorian", + "standard_name": "time", + "axis": "T", + "_ChunkSizes": [ + 1024 + ], + "_CoordinateAxisType": "Time" + } + }, + "HRDPS_P_TT_10000": { + "shape": [ + "time", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "Forecast: Air temperature", + "units": "deg_C", + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1290, + 2540 + ] + } + }, + "rotated_pole": { + "shape": [ + "" + ], + "type": "float", + "attributes": { + "grid_north_pole_longitude": [ + 65.3051528930664 + ], + "north_pole_grid_longitude": [ + 0.0 + ], + "longitude_of_prime_meridian": [ + 0.0 + ], + "grid_north_pole_latitude": [ + 36.088523864746094 + ], + "earth_radius": [ + 6371220.0 + ], + "long_name": "coordinates of the rotated North Pole", + "grid_mapping_name": "rotated_latitude_longitude", + "_CoordinateTransformType": "Projection", + "_CoordinateAxisTypes": "GeoX GeoY" + } + }, + "lon": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "longitude", + "units": "degrees_east", + "eccc_grid_definition": "grtyp: Z, ig1: 76826, ig2: 74520, ig3: 1, ig4: 0", + "standard_name": "longitude", + "_ChunkSizes": [ + 1290, + 2540 + ], + "_CoordinateAxisType": "Lon" + } + }, + "lat": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "latitude", + "units": "degrees_north", + "eccc_grid_definition": "grtyp: Z, ig1: 76826, ig2: 74520, ig3: 1, ig4: 0", + "standard_name": "latitude", + "_ChunkSizes": [ + 1290, + 2540 + ], + "_CoordinateAxisType": "Lat" + } + } + }, + "access_urls": { + "HTTPServer": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/fileServer/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "OpenDAP": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "NCML": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncml/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "UDDC": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/uddc/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "ISO": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/iso/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "WCS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wcs/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "WMS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc", + "NetcdfSubset": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncss/point/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/2024010100.nc" + } +} \ No newline at end of file diff --git a/tests/data/hrdps_sfc.json b/tests/data/hrdps_sfc.json new file mode 100644 index 0000000..def60b1 --- /dev/null +++ b/tests/data/hrdps_sfc.json @@ -0,0 +1,197 @@ +{ + "@location": "Not provided because of security concerns.", + "@xmlns": { + "": "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" + }, + "attributes": { + "_NCProperties": "version=2,netcdf=4.9.2,hdf5=1.14.2", + "Remarks": "Variable names are following the convention ___. Variables with level '10000' are at surface level. The height [m] of variables with level '0XXXX' needs to be inferrred using the corresponding fields of geopotential height (GZ_0XXXX-GZ_10000). The variables UUC, VVC, UVC, and WDC are not modelled but inferred from UU and VV for convenience of the users. Precipitation (PR) is reported as 6-hr accumulations for CaPA_fine and CaPA_coarse. Precipitation (PR) are accumulations since beginning of the forecast for GEPS, GDPS, REPS, RDPS, HRDPS, and CaLDAS. The re-analysis product RDRS_v2 contains two variables for precipitation: 'RDRS_v2_P_PR0_SFC' is the model precipitation (trial field used by CaPA) and 'RDRS_v2_A_PR0_SFC' is precipitations adjusted with CaPA 24h precipitation (as in 'RDRS_P_PR0_SFC'). Please be aware that the baseflow 'O1' of the current version of WCPS is not reliable during the spring melt period.", + "License": "These data are provided by the Canadian Surface Prediction Archive CaSPar. You should have received a copy of the License agreement with the data. Otherwise you can find them under http://caspar-data.ca/doc/caspar_license.txt or email caspar.data@uwaterloo.ca.", + "product": "HRDPS", + "Conventions": "CF-1.6", + "_CoordSysBuilder": "ucar.nc2.internal.dataset.conv.CF1Convention" + }, + "dimensions": { + "rlon": 2540, + "rlat": 1290, + "time": 13 + }, + "groups": { + "CFMetadata": { + "attributes": { + "geospatial_lon_min": [ + -152.7306365966797 + ], + "geospatial_lat_min": [ + 27.284589767456055 + ], + "geospatial_lon_max": [ + -40.708560943603516 + ], + "geospatial_lat_max": [ + 70.61150360107422 + ], + "geospatial_lon_units": "degrees_east", + "geospatial_lat_units": "degrees_north", + "geospatial_lon_resolution": "3.418852275416474E-5", + "geospatial_lat_resolution": "1.3223137054096467E-5", + "time_coverage_start": "2024-01-01T00:00:00Z", + "time_coverage_end": "2024-01-01T12:00:00Z", + "time_coverage_units": "seconds", + "time_coverage_resolution": "3600.0", + "time_coverage_duration": "P0Y0M0DT12H0M0.000S" + } + }, + "NCISOMetadata": { + "attributes": { + "metadata_creation": "2025-07-01", + "nciso_version": "2.4.6" + } + } + }, + "variables": { + "rlon": { + "shape": [ + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "longitude in rotated pole grid", + "units": "degrees", + "eccc_grid_definition": "grtyp: E, ig1: 1430, ig2: 500, ig3: 56000, ig4: 44000", + "standard_name": "grid_longitude", + "axis": "X", + "_ChunkSizes": [ + 2540 + ], + "_CoordinateAxisType": "GeoX" + } + }, + "rlat": { + "shape": [ + "rlat" + ], + "type": "float", + "attributes": { + "long_name": "latitude in rotated pole grid", + "units": "degrees", + "eccc_grid_definition": "grtyp: E, ig1: 1430, ig2: 500, ig3: 56000, ig4: 44000", + "standard_name": "grid_latitude", + "axis": "Y", + "_ChunkSizes": [ + 1290 + ], + "_CoordinateAxisType": "GeoY" + } + }, + "time": { + "shape": [ + "time" + ], + "type": "int", + "attributes": { + "long_name": "time", + "units": "hours since 2024-01-01 00:00:00", + "calendar": "gregorian", + "standard_name": "time", + "axis": "T", + "_ChunkSizes": [ + 1024 + ], + "_CoordinateAxisType": "Time" + } + }, + "HRDPS_P_PR_SFC": { + "shape": [ + "time", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "Forecast: Quantity of precipitation", + "units": "m", + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1290, + 2540 + ] + } + }, + "rotated_pole": { + "shape": [ + "" + ], + "type": "float", + "attributes": { + "grid_north_pole_longitude": [ + 65.3051528930664 + ], + "north_pole_grid_longitude": [ + 0.0 + ], + "longitude_of_prime_meridian": [ + 0.0 + ], + "grid_north_pole_latitude": [ + 36.088523864746094 + ], + "earth_radius": [ + 6371220.0 + ], + "long_name": "coordinates of the rotated North Pole", + "grid_mapping_name": "rotated_latitude_longitude", + "_CoordinateTransformType": "Projection", + "_CoordinateAxisTypes": "GeoX GeoY" + } + }, + "lon": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "longitude", + "units": "degrees_east", + "eccc_grid_definition": "grtyp: Z, ig1: 76826, ig2: 74520, ig3: 1, ig4: 0", + "standard_name": "longitude", + "_ChunkSizes": [ + 1290, + 2540 + ], + "_CoordinateAxisType": "Lon" + } + }, + "lat": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "latitude", + "units": "degrees_north", + "eccc_grid_definition": "grtyp: Z, ig1: 76826, ig2: 74520, ig3: 1, ig4: 0", + "standard_name": "latitude", + "_ChunkSizes": [ + 1290, + 2540 + ], + "_CoordinateAxisType": "Lat" + } + } + }, + "access_urls": { + "HTTPServer": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/fileServer/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "OpenDAP": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "NCML": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncml/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "UDDC": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/uddc/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "ISO": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/iso/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "WCS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wcs/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "WMS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc", + "NetcdfSubset": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncss/point/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_PR_SFC/2024010100.nc" + } +} \ No newline at end of file diff --git a/tests/data/rdps.json b/tests/data/rdps.json new file mode 100644 index 0000000..7f44360 --- /dev/null +++ b/tests/data/rdps.json @@ -0,0 +1,489 @@ +{ + "@location": "Not provided because of security concerns.", + "@xmlns": { + "": "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" + }, + "attributes": { + "Conventions": "CF-1.6", + "_NCProperties": "version=2,netcdf=4.9.2,hdf5=1.14.2", + "_CoordSysBuilder": "ucar.nc2.internal.dataset.conv.CF1Convention" + }, + "dimensions": { + "pres": 4, + "rlat": 1076, + "rlon": 1102 + }, + "groups": { + "CFMetadata": { + "attributes": { + "geospatial_lon_min": [ + 0.0002899999963119626 + ], + "geospatial_lat_min": [ + -7.749110221862793 + ], + "geospatial_lon_max": [ + 359.99951171875 + ], + "geospatial_lat_max": [ + 89.96572875976562 + ], + "geospatial_lon_units": "degrees_east", + "geospatial_lat_units": "degrees_north", + "geospatial_lon_resolution": "3.036044005049318E-4", + "geospatial_lat_resolution": "8.240755029254131E-5", + "time_coverage_start": "2024-01-01T00:00:00Z", + "time_coverage_end": "2024-01-01T00:00:00Z", + "time_coverage_units": "seconds", + "time_coverage_resolution": "0.0", + "time_coverage_duration": "P0Y0M0DT0H0M0.000S" + } + }, + "NCISOMetadata": { + "attributes": { + "metadata_creation": "2025-07-01", + "nciso_version": "2.4.6" + } + } + }, + "variables": { + "time": { + "shape": [ + "time" + ], + "type": "double", + "attributes": { + "standard_name": "time", + "long_name": "Validity time", + "axis": "T", + "units": "hours since 2024-01-01 00:00:00", + "calendar": "gregorian", + "_ChunkSizes": [ + 512 + ], + "_CoordinateAxisType": "Time" + } + }, + "pres": { + "shape": [ + "pres" + ], + "type": "float", + "attributes": { + "axis": "Z", + "standard_name": "air_pressure", + "units": "hPa", + "positive": "down", + "_CoordinateAxisType": "Pressure", + "_CoordinateZisPositive": "down" + } + }, + "rlat": { + "shape": [ + "rlat" + ], + "type": "float", + "attributes": { + "long_name": "latitude in rotated pole grid", + "standard_name": "grid_latitude", + "units": "degrees", + "axis": "Y", + "_CoordinateAxisType": "GeoY" + } + }, + "rlon": { + "shape": [ + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "longitude in rotated pole grid", + "standard_name": "grid_longitude", + "units": "degrees", + "axis": "X", + "_CoordinateAxisType": "GeoX" + } + }, + "level": { + "shape": [ + "level" + ], + "type": "float", + "attributes": { + "axis": "Z", + "positive": "down", + "standard_name": "atmosphere_hybrid_sigma_ln_pressure_coordinate", + "formula": "p = exp(a+b*log(ps/pref)) / 100.0", + "formula_terms": "a: a b: b ps: P0 pref: pref", + "coordinates": "a b", + "_CoordinateAxisType": "GeoZ", + "_CoordinateZisPositive": "down" + } + }, + "GZ": { + "shape": [ + "time", + "pres", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "rotated_pole": { + "shape": [ + "" + ], + "type": "char", + "attributes": { + "grid_mapping_name": "rotated_latitude_longitude", + "earth_radius": [ + 6370997.0 + ], + "grid_north_pole_latitude": [ + 31.758312454493154 + ], + "grid_north_pole_longitude": [ + 87.59703130293302 + ], + "north_pole_grid_longitude": [ + 0.0 + ], + "_CoordinateTransformType": "Projection", + "_CoordinateAxisTypes": "GeoX GeoY" + } + }, + "HU": { + "shape": [ + "time", + "pres", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "grid_mapping": "rotated_pole", + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "PC": { + "shape": [ + "time", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1076, + 1102 + ] + } + }, + "PN": { + "shape": [ + "time", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1076, + 1102 + ] + } + }, + "PR": { + "shape": [ + "time", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1076, + 1102 + ] + } + }, + "TD": { + "shape": [ + "time", + "level", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "pref": { + "shape": [ + "" + ], + "type": "double", + "attributes": { + "units": "hPa" + } + }, + "TT_pressure_levels": { + "shape": [ + "time", + "pres", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "TT_model_levels": { + "shape": [ + "time", + "level", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "UU_pressure_levels": { + "shape": [ + "time", + "pres", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "UU_model_levels": { + "shape": [ + "time", + "level", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "VV_pressure_levels": { + "shape": [ + "time", + "pres", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "VV_model_levels": { + "shape": [ + "time", + "level", + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "_FillValue": [ + 1.0000000150474662e+30 + ], + "coordinates": "lon lat", + "grid_mapping": "rotated_pole", + "_ChunkSizes": [ + 1, + 1, + 1076, + 1102 + ] + } + }, + "leadtime": { + "shape": [ + "time" + ], + "type": "float", + "attributes": { + "standard_name": "forecast_period", + "long_name": "Lead time (since forecast_reference_time)", + "units": "hours", + "_ChunkSizes": [ + 1024 + ], + "_CoordinateAxisType": "TimeOffset" + } + }, + "reftime": { + "shape": [ + "" + ], + "type": "double", + "attributes": { + "standard_name": "forecast_reference_time", + "units": "hours since 2024-01-01 00:00:00", + "calendar": "gregorian", + "_CoordinateAxisType": "RunTime" + } + }, + "lon": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "units": "degrees_east", + "standard_name": "longitude", + "long_name": "longitude", + "_CoordinateAxisType": "Lon" + } + }, + "lat": { + "shape": [ + "rlat", + "rlon" + ], + "type": "float", + "attributes": { + "long_name": "latitude", + "standard_name": "latitude", + "units": "degrees_north", + "_CoordinateAxisType": "Lat" + } + }, + "a": { + "shape": [ + "level" + ], + "type": "double" + }, + "b": { + "shape": [ + "level" + ], + "type": "double" + } + }, + "access_urls": { + "HTTPServer": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/fileServer/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "OpenDAP": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "NCML": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncml/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "UDDC": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/uddc/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "ISO": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/iso/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "WCS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wcs/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "WMS": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/wms/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc", + "NetcdfSubset": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/ncss/point/birdhouse/testdata/HRDPS/RDPS_sample/2024010100_000.nc" + } +} \ No newline at end of file diff --git a/tests/data/update_data.py b/tests/data/update_data.py new file mode 100644 index 0000000..fb4b2c7 --- /dev/null +++ b/tests/data/update_data.py @@ -0,0 +1,50 @@ +import json + +import requests +import xncml +from siphon.catalog import TDSCatalog + +from STACpopulator.stac_utils import np2py + +""" +Run this script to fetch metadata attributes from the PAVICS THREDDS catalog. +This avoids making requests to the server during tests. +The results are commited to the repository, so this script only needs to be run when attributes change +or new datasets are added. +""" + + +def get_first_item_attrs(url): + cat = TDSCatalog(url) + + if cat.datasets.items(): + for item_name, ds in cat.datasets.items(): + r = requests.get(ds.access_urls["NCML"]) + attrs = xncml.Dataset.from_text(r.text).to_cf_dict() + attrs["access_urls"] = ds.access_urls + return np2py(attrs) + + +def make_test_data(): + """Fetches attribute data from the PAVICS THREDDS catalog and stores it in the data/ directory as a json.""" + # Mapping between test data file names and URLs + urls = { + "cordex6_raw.json": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/disk2/ouranos/CORDEX/CMIP6/DD/NAM-12/OURANOS/MPI-ESM1-2-LR/ssp370/r1i1p1f1/CRCM5/v1-r1/day/tas/v20231208/catalog.html", + "cordex6_ncml.json": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations" + "/RCM-CMIP6/CORDEX/NAM-12/day/catalog.html", + "rdps.json": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/HRDPS" + "/RDPS_sample/catalog.html", + "hrdps_sfc.json": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/HRDPS" + "/HRDPS_sample/HRDPS_P_PR_SFC/catalog.html", + "hrdps_p_tt.json": "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/HRDPS/HRDPS_sample/HRDPS_P_TT_10000/catalog.html", + } + + for filename, url in urls.items(): + attrs = get_first_item_attrs(url) + with open(filename, "w") as f: + json.dump(attrs, f, indent=2) + + +if __name__ == "__main__": + make_test_data() + print("Test data files created in the current directory.") diff --git a/tests/test_cordex.py b/tests/test_cordex.py index 6d3058f..a3084e0 100644 --- a/tests/test_cordex.py +++ b/tests/test_cordex.py @@ -5,39 +5,6 @@ from STACpopulator.extensions.cordex6 import Cordex6DataModel, Cordex6DataModelNcML -def get_first_item_attrs(url): - import requests - import xncml - from siphon.catalog import TDSCatalog - - from STACpopulator.stac_utils import np2py - - cat = TDSCatalog(url) - - if cat.datasets.items(): - for item_name, ds in cat.datasets.items(): - r = requests.get(ds.access_urls["NCML"]) - attrs = xncml.Dataset.from_text(r.text).to_cf_dict() - attrs["access_urls"] = ds.access_urls - return np2py(attrs) - - -def make_test_data(): - """Fetches attribute data from the PAVICS THREDDS catalog and stores it in the data/ directory as a json.""" - # Raw CORDEX data - url = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/disk2/ouranos/CORDEX/CMIP6/DD/NAM-12/OURANOS/MPI-ESM1-2-LR/ssp370/r1i1p1f1/CRCM5/v1-r1/day/tas/v20231208/catalog.html" - attrs = get_first_item_attrs(url) - with open("data/cordex6_raw.json", "w") as f: - json.dump(attrs, f, indent=2) - - # NcML CORDEX data - url = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/RCM-CMIP6/CORDEX/NAM-12/day/catalog.html" - attrs = get_first_item_attrs(url) - with open("data/cordex6_ncml.json", "w") as f: - json.dump(attrs, f, indent=2) - - -@pytest.mark.vcr("test_item_raw.yaml") def test_item_raw(): attrs = json.load(open("tests/data/cordex6_raw.json")) model = Cordex6DataModel.from_data(attrs) diff --git a/tests/test_rdps.py b/tests/test_rdps.py new file mode 100644 index 0000000..eeca56f --- /dev/null +++ b/tests/test_rdps.py @@ -0,0 +1,28 @@ +"""Test RDPS and HRDPS netCDF files.""" + +import json + +from STACpopulator.extensions.thredds import THREDDSCatalogDataModel + + +def test_rdps(): + attrs = json.load(open("tests/data/rdps.json")) + model = THREDDSCatalogDataModel.from_data(attrs) + item = model.stac_item() + assert set(model._helpers) == {"thredds", "datacube"} + assert item["id"] == "birdhouse__testdata__HRDPS__RDPS_sample__2024010100_000.nc" + assert "TD" in item["properties"]["cube:variables"] + + +def test_hrdps(): + attrs = json.load(open("tests/data/hrdps_sfc.json")) + model = THREDDSCatalogDataModel.from_data(attrs) + item = model.stac_item() + assert "HRDPS_P_PR_SFC" in item["properties"]["cube:variables"] + assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} + + attrs = json.load(open("tests/data/hrdps_p_tt.json")) + model = THREDDSCatalogDataModel.from_data(attrs) + item = model.stac_item() + assert "HRDPS_P_TT_10000" in item["properties"]["cube:variables"] + assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} From 6bde69efd74330a83c52800d24aa1c5913386784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 10:49:32 -0400 Subject: [PATCH 02/76] feat: rdps extension added --- STACpopulator/extensions/rdps.py | 339 +++++++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 STACpopulator/extensions/rdps.py diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py new file mode 100644 index 0000000..7859c5f --- /dev/null +++ b/STACpopulator/extensions/rdps.py @@ -0,0 +1,339 @@ +import hashlib +import json +from datetime import datetime +from typing import ( + Any, + Generic, + Iterable, + Literal, + MutableMapping, + Optional, + Type, + TypeVar, + Union, + cast, + get_args, +) + +import numpy as np +import pystac +from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator +from pydantic.fields import FieldInfo +from pystac.extensions import item_assets +from pystac.extensions.base import ( + ExtensionManagementMixin, + PropertiesExtension, + S, # generic pystac.STACObject + SummariesExtension, +) + +from STACpopulator.models import AnyGeometry +from STACpopulator.stac_utils import ServiceType, ncattrs_to_bbox, ncattrs_to_geometry + +T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) + +SchemaName = Literal["rdps"] +SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/rdps-global-attrs-schema.json" # FIXME: To be defined +PREFIX = f"{get_args(SchemaName)[0]}:" + + +class NumpyJSONEncoder(json.JSONEncoder): + """Encoder for numpy values in JSON objects.""" + + def default(self, obj: Any) -> Any: + """Return encoded value.""" + if isinstance(obj, (np.integer,)): + return int(obj) + elif isinstance(obj, (np.floating,)): + return float(obj) + elif isinstance(obj, (np.ndarray,)): + return obj.tolist() + return super().default(obj) + + +def add_ext_prefix(name: str) -> str: + """Return the given name prefixed with this extension's prefix.""" + return PREFIX + name if "datetime" not in name else name + + +def hash_dict(d: dict) -> str: + """Return the unique sha256 of the input dictionnary.""" + dict_str = json.dumps(d, sort_keys=True, separators=(",", ":"), cls=NumpyJSONEncoder) + return hashlib.sha256(dict_str.encode("utf-8")).hexdigest() + + +class RDPSProperties(BaseModel, validate_assignment=True): + """Data model for RDPS.""" + + Conventions: str + + _prefix = "rdps" + + model_config = ConfigDict(alias_generator=add_ext_prefix, populate_by_name=True, extra="ignore") + + @field_validator("version", check_fields=False) + @classmethod + def validate_version(cls, v: str, __info: ValidationInfo) -> bool: + """Return True iff the given version is valid.""" + assert v[0] == "v", "Version string should begin with a lower case 'v'" + assert v[1:].isdigit(), "All characters in version string, except first, should be digits" + return v + + +class RDPSHelper: + """Helper for RDPS data.""" + + def __init__(self, attrs: MutableMapping[str, Any], geometry_model: Type[AnyGeometry]) -> None: + self.attrs = attrs + self.rdps_attrs = attrs["attributes"] + self.cfmeta = attrs["groups"]["CFMetadata"]["attributes"] + self.geometry_model = geometry_model + + @property + def uid(self) -> str: + """Return a unique ID for RDPS data item.""" + return hash_dict(self.cfmeta) + + @property + def geometry(self) -> AnyGeometry: + """Return the geometry.""" + return self.geometry_model(**ncattrs_to_geometry(self.attrs)) + + @property + def bbox(self) -> list[float]: + """Return the bounding box.""" + return ncattrs_to_bbox(self.attrs) + + @property + def start_datetime(self) -> datetime: + """Return the beginning of the temporal extent.""" + return self.cfmeta["time_coverage_start"] + + @property + def end_datetime(self) -> datetime: + """Return the end of the temporal extent.""" + return self.cfmeta["time_coverage_end"] + + @property + def properties(self) -> RDPSProperties: + """Return properties.""" + props = RDPSProperties(**self.rdps_attrs) + return props + + def stac_item(self) -> "pystac.Item": + """Return a pystac Item.""" + item = pystac.Item( + id=self.uid, + geometry=self.geometry.model_dump(), + bbox=self.bbox, + properties={ + "start_datetime": self.start_datetime, + "end_datetime": self.end_datetime, + }, + datetime=None, + ) + item_rdps = RDPSExtension.ext(item, add_if_missing=True) + item_rdps.apply(self.properties) + return item + + +class RDPSExtension( + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[Union[pystac.Asset, pystac.Item, pystac.Collection]], +): + """Extension for RDPS data.""" + + @property + def name(self) -> SchemaName: + """Return the schema name.""" + return get_args(SchemaName)[0] + + def apply( + self, + properties: Union[RDPSProperties, dict[str, Any]], + ) -> None: + """Apply RDPS Extension properties to the extended :class:`~pystac.Item` or :class:`~pystac.Asset`.""" + if isinstance(properties, dict): + properties = RDPSProperties(**properties) + data_json = json.loads(properties.model_dump_json(by_alias=True)) + for prop, val in data_json.items(): + self._set_property(prop, val) + + @classmethod + def get_schema_uri(cls) -> str: + """Return this extension's schema URI.""" + return SCHEMA_URI + + @classmethod + def has_extension(cls, obj: S) -> bool: + """Return True iff the object has an extension for that matches this class' schema URI.""" + # FIXME: this override should be removed once an official and versioned schema is released + # ignore the original implementation logic for a version regex + # since in our case, the VERSION_REGEX is not fulfilled (ie: using 'main' branch, no tag available...) + ext_uri = cls.get_schema_uri() + return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions) + + @classmethod + def ext(cls, obj: T, add_if_missing: bool = False) -> "RDPSExtension[T]": + """Extend the given STAC Object with properties from the :stac-ext:`RDPS Extension `. + + This extension can be applied to instances of :class:`~pystac.Item` or + :class:`~pystac.Asset`. + + Raises + ------ + pystac.ExtensionTypeError : If an invalid object type is passed. + """ + if isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(RDPSExtension[T], CollectionRDPSExtension(obj)) + elif isinstance(obj, pystac.Item): + cls.ensure_has_extension(obj, add_if_missing) + return cast(RDPSExtension[T], ItemRDPSExtension(obj)) + elif isinstance(obj, pystac.Asset): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(RDPSExtension[T], AssetRDPSExtension(obj)) + elif isinstance(obj, item_assets.AssetDefinition): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(RDPSExtension[T], ItemAssetsRDPSExtension(obj)) + else: + raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) + + @classmethod + def summaries(cls, obj: pystac.Collection, add_if_missing: bool = False) -> "SummariesRDPSExtension": + """Return the extended summaries object for the given collection.""" + cls.ensure_has_extension(obj, add_if_missing) + return SummariesRDPSExtension(obj) + + +class ItemRDPSExtension(RDPSExtension[pystac.Item]): + """ + A concrete implementation of :class:`RDPSExtension` on an :class:`~pystac.Item`. + + Extends the properties of the Item to include properties defined in the + :stac-ext:`RDPS Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`RDPSExtension.ext` on an :class:`~pystac.Item` to extend it. + """ + + def __init__(self, item: pystac.Item) -> None: + self.item = item + self.properties = item.properties + + def get_assets( + self, + service_type: Optional[ServiceType] = None, + ) -> dict[str, pystac.Asset]: + """Get the item's assets where eo:bands are defined. + + Args: + service_type: If set, filter the assets such that only those with a + matching :class:`~STACpopulator.stac_utils.ServiceType` are returned. + + Returns + ------- + Dict[str, Asset]: A dictionary of assets that match ``service_type`` + if set or else all of this item's assets were service types are defined. + """ + return { + key: asset + for key, asset in self.item.get_assets().items() + if (service_type is ServiceType and service_type.value in asset.extra_fields) + or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + } + + def __repr__(self) -> str: + """Return repr.""" + return f"" + + +class ItemAssetsRDPSExtension(RDPSExtension[item_assets.AssetDefinition]): + """Extention for RDPS item assets.""" + + properties: dict[str, Any] + asset_defn: item_assets.AssetDefinition + + def __init__(self, item_asset: item_assets.AssetDefinition) -> None: + self.asset_defn = item_asset + self.properties = item_asset.properties + + +class AssetRDPSExtension(RDPSExtension[pystac.Asset]): + """ + A concrete implementation of :class:`RDPSExtension` on an :class:`~pystac.Asset`. + + Extends the Asset fields to include properties defined in the + :stac-ext:`RDPS Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`RDPSExtension.ext` on an :class:`~pystac.Asset` to extend it. + """ + + asset_href: str + """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Asset` fields, including extension properties.""" + + additional_read_properties: Optional[Iterable[dict[str, Any]]] = None + """If present, this will be a list containing 1 dictionary representing the + properties of the owning :class:`~pystac.Item`.""" + + def __init__(self, asset: pystac.Asset) -> None: + self.asset_href = asset.href + self.properties = asset.extra_fields + if asset.owner and isinstance(asset.owner, pystac.Item): + self.additional_read_properties = [asset.owner.properties] + + def __repr__(self) -> str: + """Return repr.""" + return f"" + + +class SummariesRDPSExtension(SummariesExtension): + """ + A concrete implementation of :class:`~SummariesExtension`. + + Extends the ``summaries`` field of a :class:`~pystac.Collection` to include properties + defined in the :stac-ext:`RDPS `. + """ + + def _check_rdps_property(self, prop: str) -> FieldInfo: + try: + return RDPSProperties.model_fields[prop] + except KeyError: + raise AttributeError(f"Name '{prop}' is not a valid RDPS property.") + + def _validate_rdps_property(self, prop: str, summaries: list[Any]) -> None: + model = RDPSProperties.model_construct() + validator = RDPSProperties.__pydantic_validator__ + for value in summaries: + validator.validate_assignment(model, prop, value) + + def get_rdps_property(self, prop: str) -> list[Any]: + """Set RDPS property.""" + self._check_rdps_property(prop) + return self.summaries.get_list(prop) + + def set_rdps_property(self, prop: str, summaries: list[Any]) -> None: + """Set RDPS property.""" + self._check_rdps_property(prop) + self._validate_rdps_property(prop, summaries) + self._set_summary(prop, summaries) + + def __getattr__(self, prop: str) -> list[Any]: + """Get RDPS property.""" + return self.get_rdps_property(prop) + + def __setattr__(self, prop: str, value: Any) -> None: + """Set RDPS property.""" + self.set_rdps_property(prop, value) + + +class CollectionRDPSExtension(RDPSExtension[pystac.Collection]): + """Extension for RDPS data.""" + + def __init__(self, collection: pystac.Collection) -> None: + self.collection = collection From 911cbe9108789ef054ee696044ef6234fa1cf566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 10:51:35 -0400 Subject: [PATCH 03/76] feat: rdps collection config added --- .../implementations/RDPS_CRIM/__init__.py | 3 +++ .../RDPS_CRIM/collection_config.yml | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 STACpopulator/implementations/RDPS_CRIM/__init__.py create mode 100644 STACpopulator/implementations/RDPS_CRIM/collection_config.yml diff --git a/STACpopulator/implementations/RDPS_CRIM/__init__.py b/STACpopulator/implementations/RDPS_CRIM/__init__.py new file mode 100644 index 0000000..7e1b1e1 --- /dev/null +++ b/STACpopulator/implementations/RDPS_CRIM/__init__.py @@ -0,0 +1,3 @@ +from .add_RDPS import add_parser_args, runner + +__all__ = ["add_parser_args", "runner"] diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml new file mode 100644 index 0000000..983a033 --- /dev/null +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -0,0 +1,17 @@ +title: RDPS +id: RDPS_CRIM +description: Regional Deterministic Prediction System +keywords: ['RDPS', 'Climate Change'] +license: "CC-BY-4.0" +spatialextent: [-180, -90, 180, 90] +temporalextent: ['1850-01-01', '2500-01-01'] + +links: + - rel: about + title : Project homepage + target : https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d + media_type: text/html + - rel: license + title : License + target : https://open.canada.ca/en/open-government-licence-canada + media_type: text/plain From b5f0de9b4707a886d29d8f0f534a1d31e964fb53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 10:51:54 -0400 Subject: [PATCH 04/76] feat: rdps implementation added --- .../implementations/RDPS_CRIM/add_RDPS.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 STACpopulator/implementations/RDPS_CRIM/add_RDPS.py diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py new file mode 100644 index 0000000..0570ecb --- /dev/null +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -0,0 +1,130 @@ +import argparse +import json +import logging +import os +from typing import Any, MutableMapping, Optional, Union + +from pystac.extensions.datacube import DatacubeExtension +from requests.sessions import Session + +from STACpopulator.extensions.datacube import DataCubeHelper +from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties +from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper +from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader +from STACpopulator.models import GeoJSONPolygon +from STACpopulator.populator_base import STACpopulatorBase + +LOGGER = logging.getLogger(__name__) + + +class RDPSpopulator(STACpopulatorBase): + """Populator that creates STAC objects representing RDPS data from a THREDDS catalog.""" + + item_properties_model = RDPSProperties + item_geometry_model = GeoJSONPolygon + + def __init__( + self, + stac_host: str, + data_loader: GenericLoader, + update: Optional[bool] = False, + session: Optional[Session] = None, + config_file: Optional[Union[os.PathLike[str], str]] = None, + ) -> None: + super().__init__( + stac_host, + data_loader, + update=update, + session=session, + config_file=config_file, + ) + + def create_stac_item( + self, item_name: str, item_data: MutableMapping[str, Any] + ) -> Union[None, MutableMapping[str, Any]]: + """Create a STAC item. + + :param item_name: name of the STAC item. Interpretation of name is left to the input loader implementation + :type item_name: str + :param item_data: dictionary like representation of all information on the item + :type item_data: MutableMapping[str, Any] + :return: _description_ + :rtype: MutableMapping[str, Any] + """ + # Add RDPS extension + try: + rdps_helper = RDPSHelper(item_data, self.item_geometry_model) + item = rdps_helper.stac_item() + except Exception as e: + raise Exception("Failed to add RDPS extension") from e + + # Add datacube extension + try: + datacube_helper = DataCubeHelper(item_data) + datacube_ext = DatacubeExtension.ext(item, add_if_missing=True) + datacube_ext.apply( + dimensions=datacube_helper.dimensions, + variables=datacube_helper.variables, + ) + except Exception as e: + raise Exception("Failed to add Datacube extension") from e + + try: + thredds_helper = THREDDSHelper(item_data["access_urls"]) + thredds_ext = THREDDSExtension.ext(item) + thredds_ext.apply(thredds_helper.services, thredds_helper.links) + except Exception as e: + raise Exception("Failed to add THREDDS extension") from e + + """ + try: + item.validate() + except STACValidationError as e: + raise Exception("Failed to validate STAC item") from e + """ + + # print(json.dumps(item.to_dict())) + return json.loads(json.dumps(item.to_dict())) + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + """Add additional CLI arguments to the argument parser.""" + parser.description = "RDPS STAC populator from a THREDDS catalog or NCML XML." + parser.add_argument("stac_host", help="STAC API URL") + parser.add_argument("href", help="URL to a THREDDS catalog or a NCML XML with RDPS metadata.") + parser.add_argument("--update", action="store_true", help="Update collection and its items") + parser.add_argument( + "--mode", + choices=["full", "single"], + default="full", + help="Operation mode, processing the full dataset or only the single reference.", + ) + parser.add_argument( + "--config", + type=str, + help=( + "Override configuration file for the populator. " + "By default, uses the adjacent configuration to the implementation class." + ), + ) + + +def runner(ns: argparse.Namespace, session: Session) -> int: + """Run the populator.""" + LOGGER.info(f"Arguments to call: {vars(ns)}") + + if ns.mode == "full": + data_loader = THREDDSLoader(ns.href, session=session) + else: + # To be implemented + data_loader = ErrorLoader() + + c = RDPSpopulator( + ns.stac_host, + data_loader, + update=ns.update, + session=session, + config_file=ns.config, + ) + c.ingest() + return 0 From 76b37a791b1cd8685a362900eec9b30467536d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 10:52:39 -0400 Subject: [PATCH 05/76] fix: export rdps implementation for cli access --- STACpopulator/implementations/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/implementations/__init__.py b/STACpopulator/implementations/__init__.py index fe16c15..bc400be 100644 --- a/STACpopulator/implementations/__init__.py +++ b/STACpopulator/implementations/__init__.py @@ -5,4 +5,4 @@ # - adds additional arguments to the given parser needed to run this implementation # - def runner(ns: argparse.Namespace) -> int: # - runs the implementation given a namespace constructed from the parser arguments supplied -__all__ = ["CMIP6_UofT", "DirectoryLoader", "CORDEXCMIP6_Ouranos"] +__all__ = ["CMIP6_UofT", "DirectoryLoader", "CORDEXCMIP6_Ouranos", "RDPS_CRIM"] From 5e5541ab21d2a19d03b5eb971dbe8234443db33e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 23 Sep 2025 11:13:00 -0400 Subject: [PATCH 06/76] fix: clean unused code comments --- STACpopulator/implementations/RDPS_CRIM/add_RDPS.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 0570ecb..2adf5b2 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -82,8 +82,6 @@ def create_stac_item( except STACValidationError as e: raise Exception("Failed to validate STAC item") from e """ - - # print(json.dumps(item.to_dict())) return json.loads(json.dumps(item.to_dict())) From 4d6aba00057496e6e6065172b04a240558950130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 24 Sep 2025 09:23:53 -0400 Subject: [PATCH 07/76] fix: clean unused code --- STACpopulator/extensions/rdps.py | 2 -- test.py | 0 2 files changed, 2 deletions(-) create mode 100644 test.py diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 7859c5f..72dd4f6 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -132,8 +132,6 @@ def stac_item(self) -> "pystac.Item": }, datetime=None, ) - item_rdps = RDPSExtension.ext(item, add_if_missing=True) - item_rdps.apply(self.properties) return item diff --git a/test.py b/test.py new file mode 100644 index 0000000..e69de29 From 38662463233597e6bc8f518a1606f28b600a04e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 24 Sep 2025 10:53:27 -0400 Subject: [PATCH 08/76] fix: filename for item id updated --- STACpopulator/extensions/rdps.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 72dd4f6..440bdfd 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,5 +1,7 @@ -import hashlib +"""RPDS Extension Module.""" + import json +import os from datetime import datetime from typing import ( Any, @@ -15,7 +17,6 @@ get_args, ) -import numpy as np import pystac from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator from pydantic.fields import FieldInfo @@ -37,31 +38,11 @@ PREFIX = f"{get_args(SchemaName)[0]}:" -class NumpyJSONEncoder(json.JSONEncoder): - """Encoder for numpy values in JSON objects.""" - - def default(self, obj: Any) -> Any: - """Return encoded value.""" - if isinstance(obj, (np.integer,)): - return int(obj) - elif isinstance(obj, (np.floating,)): - return float(obj) - elif isinstance(obj, (np.ndarray,)): - return obj.tolist() - return super().default(obj) - - def add_ext_prefix(name: str) -> str: """Return the given name prefixed with this extension's prefix.""" return PREFIX + name if "datetime" not in name else name -def hash_dict(d: dict) -> str: - """Return the unique sha256 of the input dictionnary.""" - dict_str = json.dumps(d, sort_keys=True, separators=(",", ":"), cls=NumpyJSONEncoder) - return hashlib.sha256(dict_str.encode("utf-8")).hexdigest() - - class RDPSProperties(BaseModel, validate_assignment=True): """Data model for RDPS.""" @@ -92,7 +73,10 @@ def __init__(self, attrs: MutableMapping[str, Any], geometry_model: Type[AnyGeom @property def uid(self) -> str: """Return a unique ID for RDPS data item.""" - return hash_dict(self.cfmeta) + item_url = self.attrs["access_urls"]["HTTPServer"] + item_filename = os.path.basename(item_url) + _uid, _ = os.path.splitext(item_filename) + return _uid # filename without extension @property def geometry(self) -> AnyGeometry: From 138f0dd80de50cc64985cf7ab4dbbcbe54b9bcbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 24 Sep 2025 11:17:24 -0400 Subject: [PATCH 09/76] fix: item validation added --- STACpopulator/implementations/RDPS_CRIM/add_RDPS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 2adf5b2..0be7455 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -4,6 +4,7 @@ import os from typing import Any, MutableMapping, Optional, Union +from pystac import STACValidationError from pystac.extensions.datacube import DatacubeExtension from requests.sessions import Session @@ -76,12 +77,11 @@ def create_stac_item( except Exception as e: raise Exception("Failed to add THREDDS extension") from e - """ try: item.validate() except STACValidationError as e: raise Exception("Failed to validate STAC item") from e - """ + return json.loads(json.dumps(item.to_dict())) From cd5e80163da910a3653cde71b7b2acce031f3486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 24 Sep 2025 11:19:53 -0400 Subject: [PATCH 10/76] fix: item validation added --- STACpopulator/extensions/rdps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 440bdfd..c10322d 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -34,7 +34,7 @@ T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) SchemaName = Literal["rdps"] -SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/rdps-global-attrs-schema.json" # FIXME: To be defined +SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/6dd71326241e29773f24e0329dc0bf9906d96f22/rdps-global-attrs-schema.json" # FIXME: To be defined PREFIX = f"{get_args(SchemaName)[0]}:" From eb86c504d29f0180f10d188a5dfb869d5687ee64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 26 Sep 2025 09:30:46 -0400 Subject: [PATCH 11/76] fix: add properties to item --- STACpopulator/extensions/rdps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index c10322d..c7e71a4 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -113,7 +113,8 @@ def stac_item(self) -> "pystac.Item": properties={ "start_datetime": self.start_datetime, "end_datetime": self.end_datetime, - }, + } + | self.properties.model_dump(), datetime=None, ) return item From bfdd8251a6f5ffbede5acdef120f15b914b1b58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 26 Sep 2025 09:44:29 -0400 Subject: [PATCH 12/76] feat: hrdps ext. and impl. added --- STACpopulator/extensions/hrdps.py | 48 +++++++ .../implementations/HRDPS_CRIM/__init__.py | 3 + .../implementations/HRDPS_CRIM/add_HRDPS.py | 128 ++++++++++++++++++ .../HRDPS_CRIM/collection_config.yml | 17 +++ STACpopulator/implementations/__init__.py | 2 +- 5 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 STACpopulator/extensions/hrdps.py create mode 100644 STACpopulator/implementations/HRDPS_CRIM/__init__.py create mode 100644 STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py create mode 100644 STACpopulator/implementations/HRDPS_CRIM/collection_config.yml diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py new file mode 100644 index 0000000..fb63c2c --- /dev/null +++ b/STACpopulator/extensions/hrdps.py @@ -0,0 +1,48 @@ +"""HRPDS Extension Module.""" + +from typing import ( + Literal, + TypeVar, + get_args, +) + +import pystac +from pydantic import ConfigDict +from pystac.extensions import item_assets + +from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties + +T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) + +SchemaName = Literal["hrdps"] +SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/hrdps-global-attrs-schema.json" # FIXME: To be defined +PREFIX = f"{get_args(SchemaName)[0]}:" + + +def add_ext_prefix(name: str) -> str: + """Return the given name prefixed with this extension's prefix.""" + return PREFIX + name if "datetime" not in name else name + + +class HRDPSProperties(RDPSProperties, validate_assignment=True): + """Data model for HRDPS.""" + + product: str + + Remarks: str + + License: str + + _prefix = "hrdps" + + model_config = ConfigDict(alias_generator=add_ext_prefix, populate_by_name=True, extra="ignore") + + +class HRDPSHelper(RDPSHelper): + """Helper for HRDPS data.""" + + @property + def properties(self) -> HRDPSProperties: + """Return properties.""" + props = HRDPSProperties(**self.rdps_attrs) + return props diff --git a/STACpopulator/implementations/HRDPS_CRIM/__init__.py b/STACpopulator/implementations/HRDPS_CRIM/__init__.py new file mode 100644 index 0000000..37a1759 --- /dev/null +++ b/STACpopulator/implementations/HRDPS_CRIM/__init__.py @@ -0,0 +1,3 @@ +from .add_HRDPS import add_parser_args, runner + +__all__ = ["add_parser_args", "runner"] diff --git a/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py new file mode 100644 index 0000000..800f7c3 --- /dev/null +++ b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py @@ -0,0 +1,128 @@ +import argparse +import json +import logging +import os +from typing import Any, MutableMapping, Optional, Union + +from pystac import STACValidationError +from pystac.extensions.datacube import DatacubeExtension +from requests.sessions import Session + +from STACpopulator.extensions.datacube import DataCubeHelper +from STACpopulator.extensions.hrdps import HRDPSHelper, HRDPSProperties +from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper +from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader +from STACpopulator.models import GeoJSONPolygon +from STACpopulator.populator_base import STACpopulatorBase + +LOGGER = logging.getLogger(__name__) + + +class HRDPSpopulator(STACpopulatorBase): + """Populator that creates STAC objects representing HRDPS data from a THREDDS catalog.""" + + item_properties_model = HRDPSProperties + item_geometry_model = GeoJSONPolygon + + def __init__( + self, + stac_host: str, + data_loader: GenericLoader, + update: Optional[bool] = False, + session: Optional[Session] = None, + config_file: Optional[Union[os.PathLike[str], str]] = None, + ) -> None: + super().__init__( + stac_host, + data_loader, + update=update, + session=session, + config_file=config_file, + ) + + def create_stac_item( + self, item_name: str, item_data: MutableMapping[str, Any] + ) -> Union[None, MutableMapping[str, Any]]: + """Create a STAC item. + + :param item_name: name of the STAC item. Interpretation of name is left to the input loader implementation + :type item_name: str + :param item_data: dictionary like representation of all information on the item + :type item_data: MutableMapping[str, Any] + :return: _description_ + :rtype: MutableMapping[str, Any] + """ + # Add HRDPS extension + try: + hrdps_helper = HRDPSHelper(item_data, self.item_geometry_model) + item = hrdps_helper.stac_item() + except Exception as e: + raise Exception("Failed to add HRDPS extension") from e + + # Add datacube extension + try: + datacube_helper = DataCubeHelper(item_data) + datacube_ext = DatacubeExtension.ext(item, add_if_missing=True) + datacube_ext.apply( + dimensions=datacube_helper.dimensions, + variables=datacube_helper.variables, + ) + except Exception as e: + raise Exception("Failed to add Datacube extension") from e + + try: + thredds_helper = THREDDSHelper(item_data["access_urls"]) + thredds_ext = THREDDSExtension.ext(item) + thredds_ext.apply(thredds_helper.services, thredds_helper.links) + except Exception as e: + raise Exception("Failed to add THREDDS extension") from e + + try: + item.validate() + except STACValidationError as e: + raise Exception("Failed to validate STAC item") from e + + return json.loads(json.dumps(item.to_dict())) + + +def add_parser_args(parser: argparse.ArgumentParser) -> None: + """Add additional CLI arguments to the argument parser.""" + parser.description = "HRDPS STAC populator from a THREDDS catalog or NCML XML." + parser.add_argument("stac_host", help="STAC API URL") + parser.add_argument("href", help="URL to a THREDDS catalog or a NCML XML with HRDPS metadata.") + parser.add_argument("--update", action="store_true", help="Update collection and its items") + parser.add_argument( + "--mode", + choices=["full", "single"], + default="full", + help="Operation mode, processing the full dataset or only the single reference.", + ) + parser.add_argument( + "--config", + type=str, + help=( + "Override configuration file for the populator. " + "By default, uses the adjacent configuration to the implementation class." + ), + ) + + +def runner(ns: argparse.Namespace, session: Session) -> int: + """Run the populator.""" + LOGGER.info(f"Arguments to call: {vars(ns)}") + + if ns.mode == "full": + data_loader = THREDDSLoader(ns.href, session=session) + else: + # To be implemented + data_loader = ErrorLoader() + + c = HRDPSpopulator( + ns.stac_host, + data_loader, + update=ns.update, + session=session, + config_file=ns.config, + ) + c.ingest() + return 0 diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml new file mode 100644 index 0000000..f825d87 --- /dev/null +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -0,0 +1,17 @@ +title: HRDPS +id: HRDPS_CRIM +description: High Resolution Deterministic Prediction System +keywords: ['HRDPS', 'Climate Change'] +license: "CC-BY-4.0" +spatialextent: [-180, -90, 180, 90] +temporalextent: ['1850-01-01', '2500-01-01'] + +links: + - rel: about + title : Project homepage + target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d + media_type: text/html + - rel: license + title : License + target : https://open.canada.ca/en/open-government-licence-canada + media_type: text/plain diff --git a/STACpopulator/implementations/__init__.py b/STACpopulator/implementations/__init__.py index bc400be..bf97fb7 100644 --- a/STACpopulator/implementations/__init__.py +++ b/STACpopulator/implementations/__init__.py @@ -5,4 +5,4 @@ # - adds additional arguments to the given parser needed to run this implementation # - def runner(ns: argparse.Namespace) -> int: # - runs the implementation given a namespace constructed from the parser arguments supplied -__all__ = ["CMIP6_UofT", "DirectoryLoader", "CORDEXCMIP6_Ouranos", "RDPS_CRIM"] +__all__ = ["CMIP6_UofT", "DirectoryLoader", "CORDEXCMIP6_Ouranos", "RDPS_CRIM", "HRDPS_CRIM"] From 49804f513e4be793f6bd366825886c7e77092221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 30 Sep 2025 10:43:15 -0400 Subject: [PATCH 13/76] fix: integrate properties --- STACpopulator/extensions/hrdps.py | 2 +- STACpopulator/extensions/rdps.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index fb63c2c..40f4289 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -15,7 +15,7 @@ T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) SchemaName = Literal["hrdps"] -SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/hrdps-global-attrs-schema.json" # FIXME: To be defined +SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/ba3884140b5a85585b3afa385ea05996db6414bf/rdps-global-attrs-schema.json" # FIXME: To be defined PREFIX = f"{get_args(SchemaName)[0]}:" diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index c7e71a4..93f9dd8 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -34,7 +34,7 @@ T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) SchemaName = Literal["rdps"] -SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/6dd71326241e29773f24e0329dc0bf9906d96f22/rdps-global-attrs-schema.json" # FIXME: To be defined +SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/ba3884140b5a85585b3afa385ea05996db6414bf/rdps-global-attrs-schema.json" # FIXME: To be defined PREFIX = f"{get_args(SchemaName)[0]}:" @@ -106,15 +106,17 @@ def properties(self) -> RDPSProperties: def stac_item(self) -> "pystac.Item": """Return a pystac Item.""" + properties = { + "start_datetime": self.start_datetime, + "end_datetime": self.end_datetime, + } + properties = properties | self.properties.model_dump() + item = pystac.Item( id=self.uid, geometry=self.geometry.model_dump(), bbox=self.bbox, - properties={ - "start_datetime": self.start_datetime, - "end_datetime": self.end_datetime, - } - | self.properties.model_dump(), + properties=properties, datetime=None, ) return item From 99868fc180809eb1135223996d0cc7c222f4c515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 2 Oct 2025 23:27:32 -0400 Subject: [PATCH 14/76] fix: add providers handling --- .../implementations/RDPS_CRIM/add_RDPS.py | 4 ++- .../RDPS_CRIM/collection_config.yml | 32 ++++++++++++------- STACpopulator/populator_base.py | 4 +++ docs/RDPS.md | 8 +++++ 4 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 docs/RDPS.md diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 0be7455..02aaf62 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -9,7 +9,7 @@ from requests.sessions import Session from STACpopulator.extensions.datacube import DataCubeHelper -from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties +from STACpopulator.extensions.rdps import RDPSExtension, RDPSHelper, RDPSProperties from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader from STACpopulator.models import GeoJSONPolygon @@ -56,6 +56,8 @@ def create_stac_item( try: rdps_helper = RDPSHelper(item_data, self.item_geometry_model) item = rdps_helper.stac_item() + rdps_ext = RDPSExtension.ext(item, add_if_missing=True) + rdps_ext.apply(rdps_helper.properties) except Exception as e: raise Exception("Failed to add RDPS extension") from e diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 983a033..4a184a4 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -1,17 +1,25 @@ title: RDPS id: RDPS_CRIM description: Regional Deterministic Prediction System -keywords: ['RDPS', 'Climate Change'] -license: "CC-BY-4.0" +keywords: ["RDPS", "Climate Change"] +license: "other" spatialextent: [-180, -90, 180, 90] -temporalextent: ['1850-01-01', '2500-01-01'] - +temporalextent: ["1850-01-01", "2500-01-01"] +providers: + - name: NRCAN + description: Natural Resources Canada + roles: + - producer + url: https://github.com/julemai/CaSPAr/wiki/Available-products links: - - rel: about - title : Project homepage - target : https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d - media_type: text/html - - rel: license - title : License - target : https://open.canada.ca/en/open-government-licence-canada - media_type: text/plain + - rel: about + title: Project homepage + target: https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d + media_type: text/html + - rel: license + title: License + target: https://open.canada.ca/en/open-government-licence-canada + media_type: text/plain +stac_extensions: + - DatacubeExtension + - THREDDSExtension diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index e8efd19..0512b87 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -142,6 +142,10 @@ def create_stac_collection(self) -> dict[str, Any]: # Add any assets if provided in the config self._collection_info["assets"] = self.__make_collection_assets() + # Cast providers to pystac objects + providers = self._collection_info["providers"] + self._collection_info["providers"] = [pystac.Provider(**provider) for provider in providers] + # Construct links if provided in the config. This needs to be done before constructing a collection object. collection_links = self.__make_collection_links() diff --git a/docs/RDPS.md b/docs/RDPS.md new file mode 100644 index 0000000..81393df --- /dev/null +++ b/docs/RDPS.md @@ -0,0 +1,8 @@ +# Regional Deterministic Prediction System +Metadata Integration Flow + +## 1. Applied Extensions + +- RDPSHelper (item) +- DatacubeExtension +- THREDDSExtension \ No newline at end of file From 24066be2f5c3357822681c2c05320f735b420aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 3 Oct 2025 04:18:51 -0400 Subject: [PATCH 15/76] feat: cf extension added --- STACpopulator/extensions/cf.py | 126 ++++++++++++++++++ STACpopulator/extensions/rdps.py | 10 +- .../implementations/RDPS_CRIM/add_RDPS.py | 17 ++- 3 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 STACpopulator/extensions/cf.py diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py new file mode 100644 index 0000000..3c45bbf --- /dev/null +++ b/STACpopulator/extensions/cf.py @@ -0,0 +1,126 @@ +"""CF Extension Module.""" + +from __future__ import annotations + +import functools +from typing import Any, Dict, List, Optional, TypeVar + +import pystac +from pystac.extensions import item_assets +from pystac.extensions.base import ( + ExtensionManagementMixin, + PropertiesExtension, +) +from pystac.item import Item + +T = TypeVar("T", pystac.Item, pystac.Asset, item_assets.AssetDefinition) +SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json" + + +class CFParameter: + """CFParameter.""" + + def __init__(self, name: str, unit: Optional[str] = None) -> None: + if not name: + raise ValueError("CFParameter name must be non-empty") + self.name = name + self.unit = unit + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary.""" + data = {"name": self.name} + if self.unit: + data["unit"] = self.unit + return data + + @staticmethod + def from_dict(d: dict[str, Any]) -> CFParameter: + """Convert from dictionary.""" + return CFParameter(name=d["name"], unit=d["unit"]) + + def __repr__(self) -> str: + """Return string repr.""" + return f"CFParameter (name={self.name}, unit={self.unit})" + + +class CFHelper: + """CFHelper.""" + + def __init__(self, variables: dict[str, any]) -> None: + """Take a STAC item variables to identify CF parameters metadata.""" + self.variables = variables + + @functools.cached_property + def cf_parameters(self) -> List[CFParameter]: + """Extracts cf:parameter-like information from item_data.""" + parameters = [] + + for _, var in self.variables.items(): + attrs = var.get("attributes", {}) + name = attrs.get("standard_name") # Get the required standard name + if not name: + # Skip if no valid name + continue + + unit = attrs.get("units") + parameters.append(CFParameter(name=name, unit=unit)) + + return parameters + + +# ------------------------------- +# CF Extension for Items/Assets +# ------------------------------- +class CFItemExtension(PropertiesExtension, ExtensionManagementMixin): + """CFItemExtension.""" + + def __init__(self, item: Item) -> None: + self.item = item + self.properties = item.properties + + @property + def cf_parameters(self) -> Optional[List[CFParameter]]: + """Return the cf parameters list.""" + result = self._get_property("cf:parameter", List[Dict[str, Any]]) + return [CFParameter.from_dict(v) for v in result] + + @cf_parameters.setter + def cf_parameters(self, value: List[CFParameter]) -> None: + """Set the cf parameters list.""" + self._set_property("cf:parameter", [p.to_dict() for p in value]) + + def apply(self, parameters: List[CFParameter], apply_to_properties: bool = True) -> None: + """Add cf:parameter array to assets (requirement) and optionally to item properties (informative).""" + if not self.item.assets: + return # FIXME: Explicitly raise the issue that THREDSExtension should be called before to populate assets. + + for _, asset in self.item.assets.items(): + # Add CF extension URI to asset's stac_extensions + if "stac_extensions" not in asset.extra_fields: + asset.extra_fields["stac_extensions"] = [] + if SCHEMA_URI not in asset.extra_fields["stac_extensions"]: + asset.extra_fields["stac_extensions"].append(SCHEMA_URI) + # Add cf:parameter array to asset extra_fields + asset.extra_fields["cf:parameter"] = [p.to_dict() for p in parameters] + + if apply_to_properties: + self.cf_parameters = parameters + + @classmethod + def ext(cls, item: Item, add_if_missing: bool = False) -> CFItemExtension: # FIXME: Use a generic type + """Extend the given STAC Object with properties from the :stac-ext:`CF Extension `. + + This extension can be applied to instances of :class:`~pystac.Item` or + :class:`~pystac.Asset`. + + Raises + ------ + pystac.ExtensionTypeError : If an invalid object type is passed. + """ + cls.ensure_has_extension(item, add_if_missing) + return cls(item) + + @classmethod + def get_schema_uri(cls) -> str: + """Return this extension's schema URI.""" + return SCHEMA_URI diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 93f9dd8..90457a0 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,5 +1,7 @@ """RPDS Extension Module.""" +from __future__ import annotations + import json import os from datetime import datetime @@ -34,7 +36,7 @@ T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) SchemaName = Literal["rdps"] -SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/ba3884140b5a85585b3afa385ea05996db6414bf/rdps-global-attrs-schema.json" # FIXME: To be defined +SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/rdps-global-attrs-schema.json" # FIXME: To be defined PREFIX = f"{get_args(SchemaName)[0]}:" @@ -110,8 +112,6 @@ def stac_item(self) -> "pystac.Item": "start_datetime": self.start_datetime, "end_datetime": self.end_datetime, } - properties = properties | self.properties.model_dump() - item = pystac.Item( id=self.uid, geometry=self.geometry.model_dump(), @@ -119,6 +119,8 @@ def stac_item(self) -> "pystac.Item": properties=properties, datetime=None, ) + item_rdps = RDPSExtension.ext(item, add_if_missing=True) + item_rdps.apply(self.properties) return item @@ -160,7 +162,7 @@ def has_extension(cls, obj: S) -> bool: return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions) @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> "RDPSExtension[T]": + def ext(cls, obj: T, add_if_missing: bool = False) -> RDPSExtension[T]: """Extend the given STAC Object with properties from the :stac-ext:`RDPS Extension `. This extension can be applied to instances of :class:`~pystac.Item` or diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 02aaf62..90be6b2 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -8,8 +8,9 @@ from pystac.extensions.datacube import DatacubeExtension from requests.sessions import Session +from STACpopulator.extensions.cf import CFHelper, CFItemExtension from STACpopulator.extensions.datacube import DataCubeHelper -from STACpopulator.extensions.rdps import RDPSExtension, RDPSHelper, RDPSProperties +from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader from STACpopulator.models import GeoJSONPolygon @@ -56,12 +57,10 @@ def create_stac_item( try: rdps_helper = RDPSHelper(item_data, self.item_geometry_model) item = rdps_helper.stac_item() - rdps_ext = RDPSExtension.ext(item, add_if_missing=True) - rdps_ext.apply(rdps_helper.properties) except Exception as e: raise Exception("Failed to add RDPS extension") from e - # Add datacube extension + # Add DataCube Extension try: datacube_helper = DataCubeHelper(item_data) datacube_ext = DatacubeExtension.ext(item, add_if_missing=True) @@ -72,6 +71,7 @@ def create_stac_item( except Exception as e: raise Exception("Failed to add Datacube extension") from e + # Add THREDDS Extension try: thredds_helper = THREDDSHelper(item_data["access_urls"]) thredds_ext = THREDDSExtension.ext(item) @@ -79,9 +79,18 @@ def create_stac_item( except Exception as e: raise Exception("Failed to add THREDDS extension") from e + # Add CF Extension + try: + cf_helper = CFHelper(item_data["variables"]) + cf_extension = CFItemExtension.ext(item, add_if_missing=True) + cf_extension.apply(cf_helper.cf_parameters) + except Exception as e: + raise Exception("Failed to add CF extension") from e + try: item.validate() except STACValidationError as e: + print(json.dumps(item.to_dict())) raise Exception("Failed to validate STAC item") from e return json.loads(json.dumps(item.to_dict())) From 90c30c1cf7833054d27f69a60333e8d5652ef12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 3 Oct 2025 13:10:07 -0400 Subject: [PATCH 16/76] fix: make providers optional --- STACpopulator/populator_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 0512b87..5652afb 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -143,8 +143,9 @@ def create_stac_collection(self) -> dict[str, Any]: self._collection_info["assets"] = self.__make_collection_assets() # Cast providers to pystac objects - providers = self._collection_info["providers"] - self._collection_info["providers"] = [pystac.Provider(**provider) for provider in providers] + if "providers" in self._collection_info: + providers = self._collection_info["providers"] + self._collection_info["providers"] = [pystac.Provider(**provider) for provider in providers] # Construct links if provided in the config. This needs to be done before constructing a collection object. collection_links = self.__make_collection_links() From 9ea89d7d828d0d12c5765b840fa032f693c3b50c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 3 Oct 2025 15:31:27 -0400 Subject: [PATCH 17/76] feat: file info extension added --- STACpopulator/extensions/fileinfo.py | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 STACpopulator/extensions/fileinfo.py diff --git a/STACpopulator/extensions/fileinfo.py b/STACpopulator/extensions/fileinfo.py new file mode 100644 index 0000000..f6d43db --- /dev/null +++ b/STACpopulator/extensions/fileinfo.py @@ -0,0 +1,55 @@ +import json +from dataclasses import dataclass +from typing import Generic, Optional, TypeVar, Union, cast + +import pystac +from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension + +T = TypeVar("T", pystac.Asset, pystac.Link) +SCHEMA_URI = "https://stac-extensions.github.io/file/v2.1.0/schema.json" + + +@dataclass +class FileInfo: + """File Info Properties.""" + + size: Optional[int] = None + checksum: Optional[str] = None + header_size: Optional[int] = None + byte_order: Optional[str] = None + local_path: Optional[str] = None + + +class FileInfoExtension( + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[Union[pystac.Asset, pystac.Link]], +): + """FileInfoExtension class.""" + + def apply(self, properties: Union[FileInfo, dict[str, any]]) -> None: + """Apply File Info Extension to the extended STAC Item or Asset.""" + if isinstance(properties, dict): + properties = FileInfo(**properties) + data_json = json.loads(properties.model_dump_json(by_alias=True)) + for prop, val in data_json.items(): + self._set_property(prop, val) + + @classmethod + def ext(cls, obj: T, add_if_missing: bool = False) -> "FileInfoExtension[T]": + """Extend the given STAC Object with properties from the :stac-ext:`FileInfo Extension `. + + This extension can be applied to instances of :class:`~pystac.Asset` or + :class:`~pystac.Link`. + + Raises + ------ + pystac.ExtensionTypeError : If an invalid object type is passed. + """ + cls.ensure_has_extension(obj, add_if_missing) + return cast(FileInfoExtension[T], cls(obj)) + + @classmethod + def get_schema_uri(cls) -> str: + """Return this extension's schema URI.""" + return SCHEMA_URI From fc8806b6821ad8ae88394f55f54b5b04ba68a092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 7 Oct 2025 11:58:44 -0400 Subject: [PATCH 18/76] feat: dataclasses-json library added --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index afe7d6e..2f9368c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ dependencies = [ "pyessv~=0.9", "requests~=2.32", "lxml~=6.0", + "dataclasses-json~=0.6.7" ] readme = "README.md" license = { file = "LICENSE" } From ad2bba9256e1216a4404830c205b45cd3ed4761e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 7 Oct 2025 23:28:07 -0400 Subject: [PATCH 19/76] fix: contacts extension added --- .../RDPS_CRIM/collection_config.yml | 20 ++++- STACpopulator/populator_base.py | 34 +++++++- STACpopulator/utils/__init__.py | 0 STACpopulator/utils/contact.py | 84 +++++++++++++++++++ 4 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 STACpopulator/utils/__init__.py create mode 100644 STACpopulator/utils/contact.py diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 4a184a4..3a50789 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -1,16 +1,18 @@ -title: RDPS id: RDPS_CRIM +title: RDPS description: Regional Deterministic Prediction System keywords: ["RDPS", "Climate Change"] license: "other" spatialextent: [-180, -90, 180, 90] temporalextent: ["1850-01-01", "2500-01-01"] + providers: - name: NRCAN description: Natural Resources Canada roles: - producer url: https://github.com/julemai/CaSPAr/wiki/Available-products + links: - rel: about title: Project homepage @@ -20,6 +22,22 @@ links: title: License target: https://open.canada.ca/en/open-government-licence-canada media_type: text/plain + +contacts: + - organization: Computer Research Institute of Montreal (CRIM) + identifier: CRIM + position: Support + emails: + - value: support-geo@crim.ca + roles: ["support"] + logo: + href: https://www.crim.ca/wp-content/uploads/2020/10/logo-color.png + rel: icon + type: image/png + title: Logo CRIM + stac_extensions: - DatacubeExtension - THREDDSExtension + - https://stac-extensions.github.io/cf/v0.2.0/schema.json + - https://stac-extensions.github.io/contacts/v0.1.1/schema.json \ No newline at end of file diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 5652afb..805c80e 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -19,6 +19,7 @@ from STACpopulator.input import GenericLoader from STACpopulator.models import AnyGeometry from STACpopulator.stac_utils import load_config +from STACpopulator.utils.contact import Contact LOGGER = logging.getLogger(__name__) @@ -143,9 +144,11 @@ def create_stac_collection(self) -> dict[str, Any]: self._collection_info["assets"] = self.__make_collection_assets() # Cast providers to pystac objects - if "providers" in self._collection_info: - providers = self._collection_info["providers"] - self._collection_info["providers"] = [pystac.Provider(**provider) for provider in providers] + self._collection_info["providers"] = self.__make_collection_providers() + + # Add contacts as extra_field + self._collection_info["extra_fields"] = {} + self._collection_info["extra_fields"]["contacts"] = self.__make_collection_contacts() # Construct links if provided in the config. This needs to be done before constructing a collection object. collection_links = self.__make_collection_links() @@ -183,6 +186,31 @@ def __make_collection_assets(self) -> Dict[str, pystac.Asset]: pystac_assets[asset_name] = pystac.Asset(**asset_info) return pystac_assets + def __make_collection_providers(self) -> List[pystac.Provider]: + """Create collection level providers based on data read in from the configuration file. + + :return: List of pystac Provider objects + :rtype: List[pystac.Provider] + """ + pystac_providers = [] + if "providers" in self._collection_info: + providers = self._collection_info.pop("providers") + pystac_providers = [pystac.Provider(**provider) for provider in providers] + return pystac_providers + + def __make_collection_contacts(self) -> List[Dict]: + """Create collection level contacts based on data read in from the configuration file. + + :return: List of Contact objects + :rtype: List[Contact] + """ + contacts = [] + if "contacts" in self._collection_info: + contacts = self._collection_info.pop("contacts") + # FIXME: Collection should support Contact or implement to_dict of Objects in extra_fields + contacts = [Contact(**contact).to_dict() for contact in contacts] + return contacts + def publish_stac_collection(self, collection_data: dict[str, Any]) -> None: """Publish this collection by uploading it to the STAC catalog at self.stac_host.""" post_stac_collection(self.stac_host, collection_data, self.update, session=self._session) diff --git a/STACpopulator/utils/__init__.py b/STACpopulator/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/STACpopulator/utils/contact.py b/STACpopulator/utils/contact.py new file mode 100644 index 0000000..482c125 --- /dev/null +++ b/STACpopulator/utils/contact.py @@ -0,0 +1,84 @@ +"""Contact data model.""" + +from dataclasses import dataclass, field +from typing import List, Optional + +import pystac +from dataclasses_json import LetterCase, config, dataclass_json + +SCHEMA_URI = "https://stac-extensions.github.io/contacts/v0.1.1/schema.json" + + +@dataclass_json +@dataclass +class Info: + """Gives contact information for and their "roles".""" + + value: str + """The actual contact information, e.g. the phone number or the email address.""" + + roles: Optional[List[str]] = None + """The type(s) of this contact information, e.g. whether it's at work or at home.""" + + +@dataclass_json +@dataclass +class Address: + """Physical location at which contact can be made.""" + + deliveryPoint: Optional[List[str]] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) + """Address lines for the location, for example a street name and a house number.""" + + city: Optional[str] = None + """City for the location.""" + + administrativeArea: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) + """State or province of the location.""" + + postalCode: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) + """ ZIP or other postal code.""" + + country: Optional[str] = None + """Country of the physical address.""" + + +@dataclass_json +@dataclass +class Contact: + """Provides information about a contact.""" + + name: Optional[str] = None + """The name of the responsible person. Required if organization is missing.""" + + organization: Optional[str] = None + """Organization or affiliation of the contact. Required if name is missing""" + + identifier: Optional[str] = None + """A value uniquely identifying the contact.""" + + position: Optional[str] = None + """The name of the role or position of the responsible person.""" + + description: Optional[str] = None + """Detailed multi-line description to fully explain the STAC entity.""" + + logo: Optional[pystac.Link] = None + """Graphic identifying the contact.""" + + phones: Optional[List[Info]] = None + """Telephone numbers at which contact can be made.""" + + emails: Optional[List[Info]] = None + """Email address at which contact can be made.""" + + addresses: Optional[List[Address]] = None + """Physical location at which contact can be made.""" + + links: Optional[List[pystac.Link]] = None + """Links related to the contact.""" + + contactInstructions: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) + """Supplemental instructions on how or when to contact the responsible party.""" + + roles: Optional[List[str]] = None + """The set of named duties, job functions and/or permissions associated with this contact.""" From 6c45ef2a02c5328cd25096a521518c25b62b35b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 8 Oct 2025 09:51:09 -0400 Subject: [PATCH 20/76] fix: file info extension added --- .gitignore | 3 + STACpopulator/extensions/cf.py | 20 ++--- STACpopulator/extensions/file.py | 73 +++++++++++++++++++ STACpopulator/extensions/fileinfo.py | 55 -------------- .../implementations/RDPS_CRIM/add_RDPS.py | 21 +++++- .../RDPS_CRIM/collection_config.yml | 2 +- 6 files changed, 106 insertions(+), 68 deletions(-) create mode 100644 STACpopulator/extensions/file.py delete mode 100644 STACpopulator/extensions/fileinfo.py diff --git a/.gitignore b/.gitignore index 603532e..3ad3e86 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ build # Old Submodule Path # Could be used locally pyessv-archive/ + +# Temp files downloaded +temp_files/ \ No newline at end of file diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 3c45bbf..3acda03 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -92,16 +92,16 @@ def cf_parameters(self, value: List[CFParameter]) -> None: def apply(self, parameters: List[CFParameter], apply_to_properties: bool = True) -> None: """Add cf:parameter array to assets (requirement) and optionally to item properties (informative).""" if not self.item.assets: - return # FIXME: Explicitly raise the issue that THREDSExtension should be called before to populate assets. - - for _, asset in self.item.assets.items(): - # Add CF extension URI to asset's stac_extensions - if "stac_extensions" not in asset.extra_fields: - asset.extra_fields["stac_extensions"] = [] - if SCHEMA_URI not in asset.extra_fields["stac_extensions"]: - asset.extra_fields["stac_extensions"].append(SCHEMA_URI) - # Add cf:parameter array to asset extra_fields - asset.extra_fields["cf:parameter"] = [p.to_dict() for p in parameters] + return # FIXME: Raise that THREDDSExtension should be called before to populate assets. + + # FIXME: This is temporary fix to validate the item. Normally, add cf:parameter to properties should be enough. + asset = self.item.assets["HTTPServer"] + if "stac_extensions" not in asset.extra_fields: + asset.extra_fields["stac_extensions"] = [] + if SCHEMA_URI not in asset.extra_fields["stac_extensions"]: + asset.extra_fields["stac_extensions"].append(SCHEMA_URI) + # Add cf:parameter array to asset extra_fields + asset.extra_fields["cf:parameter"] = [p.to_dict() for p in parameters] if apply_to_properties: self.cf_parameters = parameters diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py new file mode 100644 index 0000000..66087c8 --- /dev/null +++ b/STACpopulator/extensions/file.py @@ -0,0 +1,73 @@ +import hashlib +from pathlib import Path +from typing import Optional + +import pystac +import requests +from netCDF4 import Dataset + +from STACpopulator.extensions.base import Helper + + +class FileHelper(Helper): + """Helper to handle file info from elements of types Asset and Link.""" + + def __init__(self, href: str, download_dir: str = "temp_files") -> None: + """Initialize the FileHelper object.""" + self.href = href + self.download_dir = Path(download_dir) + self.download_dir.mkdir(exist_ok=True) + self.local_path = self.download_dir / Path(href).name + self.download_file() # Download the file + + # Compute properties + self.size = self.local_path.stat().st_size + self.checksum = self.calculate_checksum() + # self.header_size = self.get_netcdf_header_size(self.local_path) + self.byte_order = self.find_byte_order() + + def download_file(self) -> None: + """Download the current file into temporary directory.""" + if self.local_path.exists(): + return + r = requests.get(self.href, stream=True) + r.raise_for_status() + with open(self.local_path, "wb") as f: + for chunk in r.iter_content(4096): + f.write(chunk) + + def calculate_checksum(self) -> str: + """Return file md5 checksum value.""" + h = hashlib.md5() + with open(self.local_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + h.update(chunk) + return h.hexdigest() + + def find_byte_order(self) -> Optional[pystac.extensions.file.ByteOrder]: + """Detect byte order based on file header content.""" + # TODO: Check TIFF header. To be double-checked + with open(self.local_path, "rb") as f: + header = f.read(2) + if header == b"II": + return "little-endian" + elif header == b"MM": + return "big-endian" + else: + return None + + def get_netcdf_header_size(self, file_path: Path) -> int: + """Estimate the header size of a NetCDF (.nc) file in bytes.""" + with Dataset(file_path, "r") as ds: + variables = ds.variables + if not variables: + # No variables → header = full file? + return file_path.stat().st_size + # NetCDF-3 classic: header ends at the start of first variable + first_var = next(iter(variables.values())) + try: + offset = first_var._offset + except AttributeError: + # NetCDF-4/HDF5: header is less relevant, fallback to 0 + offset = 0 + return offset diff --git a/STACpopulator/extensions/fileinfo.py b/STACpopulator/extensions/fileinfo.py deleted file mode 100644 index f6d43db..0000000 --- a/STACpopulator/extensions/fileinfo.py +++ /dev/null @@ -1,55 +0,0 @@ -import json -from dataclasses import dataclass -from typing import Generic, Optional, TypeVar, Union, cast - -import pystac -from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension - -T = TypeVar("T", pystac.Asset, pystac.Link) -SCHEMA_URI = "https://stac-extensions.github.io/file/v2.1.0/schema.json" - - -@dataclass -class FileInfo: - """File Info Properties.""" - - size: Optional[int] = None - checksum: Optional[str] = None - header_size: Optional[int] = None - byte_order: Optional[str] = None - local_path: Optional[str] = None - - -class FileInfoExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[Union[pystac.Asset, pystac.Link]], -): - """FileInfoExtension class.""" - - def apply(self, properties: Union[FileInfo, dict[str, any]]) -> None: - """Apply File Info Extension to the extended STAC Item or Asset.""" - if isinstance(properties, dict): - properties = FileInfo(**properties) - data_json = json.loads(properties.model_dump_json(by_alias=True)) - for prop, val in data_json.items(): - self._set_property(prop, val) - - @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> "FileInfoExtension[T]": - """Extend the given STAC Object with properties from the :stac-ext:`FileInfo Extension `. - - This extension can be applied to instances of :class:`~pystac.Asset` or - :class:`~pystac.Link`. - - Raises - ------ - pystac.ExtensionTypeError : If an invalid object type is passed. - """ - cls.ensure_has_extension(obj, add_if_missing) - return cast(FileInfoExtension[T], cls(obj)) - - @classmethod - def get_schema_uri(cls) -> str: - """Return this extension's schema URI.""" - return SCHEMA_URI diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 90be6b2..9b8747f 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -6,10 +6,12 @@ from pystac import STACValidationError from pystac.extensions.datacube import DatacubeExtension +from pystac.extensions.file import AssetFileExtension from requests.sessions import Session from STACpopulator.extensions.cf import CFHelper, CFItemExtension from STACpopulator.extensions.datacube import DataCubeHelper +from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader @@ -82,11 +84,26 @@ def create_stac_item( # Add CF Extension try: cf_helper = CFHelper(item_data["variables"]) - cf_extension = CFItemExtension.ext(item, add_if_missing=True) - cf_extension.apply(cf_helper.cf_parameters) + cf_ext = CFItemExtension.ext(item, add_if_missing=True) + cf_ext.apply(cf_helper.cf_parameters) except Exception as e: raise Exception("Failed to add CF extension") from e + try: + asset = item.assets["HTTPServer"] + file_helper = FileHelper(asset.get_absolute_href()) + file_ext = AssetFileExtension.ext(asset, add_if_missing=True) + file_ext.apply( + byte_order=file_helper.byte_order, + checksum=file_helper.checksum, # FIXME: Displayed as n/a on browser + header_size=0, # FIXME: clarify expected value here + size=file_helper.size, + values=None, # NOTE: deprecated + local_path=None, # FIXME: clarify expected value here + ) + except Exception as e: + raise Exception("Failed to add Asset File extension") from e + try: item.validate() except STACValidationError as e: diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 3a50789..9c0d2ae 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -21,7 +21,7 @@ links: - rel: license title: License target: https://open.canada.ca/en/open-government-licence-canada - media_type: text/plain + media_type: text/html contacts: - organization: Computer Research Institute of Montreal (CRIM) From 730763ce6f388e684c3cb55c756fb6f4e899fad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 8 Oct 2025 11:06:37 -0400 Subject: [PATCH 21/76] fix clean cf extension code --- STACpopulator/extensions/cf.py | 250 +++++++++++++----- .../{utils => extensions}/contact.py | 0 .../implementations/RDPS_CRIM/add_RDPS.py | 15 +- STACpopulator/populator_base.py | 2 +- STACpopulator/utils/__init__.py | 0 5 files changed, 190 insertions(+), 77 deletions(-) rename STACpopulator/{utils => extensions}/contact.py (100%) delete mode 100644 STACpopulator/utils/__init__.py diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 3acda03..e174084 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -3,44 +3,50 @@ from __future__ import annotations import functools -from typing import Any, Dict, List, Optional, TypeVar +from dataclasses import dataclass +from typing import ( + Any, + Generic, + Iterable, + List, + Literal, + Optional, + TypeVar, + Union, + cast, + get_args, +) import pystac +from dataclasses_json import dataclass_json from pystac.extensions import item_assets -from pystac.extensions.base import ( - ExtensionManagementMixin, - PropertiesExtension, -) -from pystac.item import Item +from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension, S -T = TypeVar("T", pystac.Item, pystac.Asset, item_assets.AssetDefinition) +from STACpopulator.stac_utils import ServiceType + +T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) +SchemaName = Literal["cf"] SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json" +PREFIX = f"{get_args(SchemaName)[0]}:" +PARAMETER_PROP = PREFIX + "parameter" -class CFParameter: - """CFParameter.""" +def add_ext_prefix(name: str) -> str: + """Return the given name prefixed with this extension's prefix.""" + return PREFIX + name if "datetime" not in name else name - def __init__(self, name: str, unit: Optional[str] = None) -> None: - if not name: - raise ValueError("CFParameter name must be non-empty") - self.name = name - self.unit = unit - def to_dict(self) -> Dict[str, Any]: - """Convert to dictionary.""" - data = {"name": self.name} - if self.unit: - data["unit"] = self.unit - return data +@dataclass_json +@dataclass +class CFParameter: + """CFParameter.""" - @staticmethod - def from_dict(d: dict[str, Any]) -> CFParameter: - """Convert from dictionary.""" - return CFParameter(name=d["name"], unit=d["unit"]) + name: str + unit: Optional[str] def __repr__(self) -> str: """Return string repr.""" - return f"CFParameter (name={self.name}, unit={self.unit})" + return f"" class CFHelper: @@ -51,7 +57,7 @@ def __init__(self, variables: dict[str, any]) -> None: self.variables = variables @functools.cached_property - def cf_parameters(self) -> List[CFParameter]: + def parameters(self) -> List[CFParameter]: """Extracts cf:parameter-like information from item_data.""" parameters = [] @@ -62,65 +68,169 @@ def cf_parameters(self) -> List[CFParameter]: # Skip if no valid name continue - unit = attrs.get("units") + unit = attrs.get("units") or "" parameters.append(CFParameter(name=name, unit=unit)) return parameters -# ------------------------------- -# CF Extension for Items/Assets -# ------------------------------- -class CFItemExtension(PropertiesExtension, ExtensionManagementMixin): - """CFItemExtension.""" +class CFExtension( + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[Union[pystac.Asset, pystac.Item, pystac.Collection]], +): + """CF Metadata Extension.""" - def __init__(self, item: Item) -> None: - self.item = item - self.properties = item.properties + @property + def name(self) -> SchemaName: + """Return the schema name.""" + return get_args(SchemaName)[0] @property - def cf_parameters(self) -> Optional[List[CFParameter]]: - """Return the cf parameters list.""" - result = self._get_property("cf:parameter", List[Dict[str, Any]]) - return [CFParameter.from_dict(v) for v in result] - - @cf_parameters.setter - def cf_parameters(self, value: List[CFParameter]) -> None: - """Set the cf parameters list.""" - self._set_property("cf:parameter", [p.to_dict() for p in value]) - - def apply(self, parameters: List[CFParameter], apply_to_properties: bool = True) -> None: - """Add cf:parameter array to assets (requirement) and optionally to item properties (informative).""" - if not self.item.assets: - return # FIXME: Raise that THREDDSExtension should be called before to populate assets. - - # FIXME: This is temporary fix to validate the item. Normally, add cf:parameter to properties should be enough. - asset = self.item.assets["HTTPServer"] - if "stac_extensions" not in asset.extra_fields: - asset.extra_fields["stac_extensions"] = [] - if SCHEMA_URI not in asset.extra_fields["stac_extensions"]: - asset.extra_fields["stac_extensions"].append(SCHEMA_URI) - # Add cf:parameter array to asset extra_fields - asset.extra_fields["cf:parameter"] = [p.to_dict() for p in parameters] - - if apply_to_properties: - self.cf_parameters = parameters + def parameter(self) -> List[dict[str, Any]] | None: + """Get or set the CF parameter(s).""" + return self._get_property(PARAMETER_PROP, int) + + @parameter.setter + def parameter(self, v: List[dict[str, Any]] | None) -> None: + self._set_property(PARAMETER_PROP, v) + + def apply( + self, + parameters: Union[List[CFParameter], List[dict[str, Any]]], + ) -> None: + """Apply CF Extension properties to the extended :class:`~pystac.Item` or :class:`~pystac.Asset`.""" + if not isinstance(parameters[0], dict): + parameters = [p.to_dict() for p in parameters] + self.parameter = parameters @classmethod - def ext(cls, item: Item, add_if_missing: bool = False) -> CFItemExtension: # FIXME: Use a generic type + def get_schema_uri(cls) -> str: + """Return this extension's schema URI.""" + return SCHEMA_URI + + @classmethod + def has_extension(cls, obj: S) -> bool: + """Return True iff the object has an extension for that matches this class' schema URI.""" + # FIXME: this override should be removed once an official and versioned schema is released + # ignore the original implementation logic for a version regex + # since in our case, the VERSION_REGEX is not fulfilled (ie: using 'main' branch, no tag available...) + ext_uri = cls.get_schema_uri() + return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions) + + @classmethod + def ext(cls, obj: T, add_if_missing: bool = False) -> CFExtension[T]: """Extend the given STAC Object with properties from the :stac-ext:`CF Extension `. - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + This extension can be applied to instances of :class:`~pystac.Item`, :class:`~pystac.Asset`, or :class:`~pystac.Collection`. Raises ------ pystac.ExtensionTypeError : If an invalid object type is passed. """ - cls.ensure_has_extension(item, add_if_missing) - return cls(item) + if isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(CFExtension[T], CollectionCFExtension(obj)) + elif isinstance(obj, pystac.Item): + cls.ensure_has_extension(obj, add_if_missing) + return cast(CFExtension[T], ItemCFExtension(obj)) + elif isinstance(obj, pystac.Asset): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(CFExtension[T], AssetCFExtension(obj)) + elif isinstance(obj, item_assets.AssetDefinition): + cls.ensure_owner_has_extension(obj, add_if_missing) + return cast(CFExtension[T], ItemAssetsCFExtension(obj)) + else: + raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) + + +class ItemCFExtension(CFExtension[pystac.Item]): + """ + A concrete implementation of :class:`CFExtension` on an :class:`~pystac.Item`. + + Extends the properties of the Item to include properties defined in the + :stac-ext:`CF Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`CFExtension.ext` on an :class:`~pystac.Item` to extend it. + """ + + def __init__(self, item: pystac.Item) -> None: + self.item = item + self.properties = item.properties - @classmethod - def get_schema_uri(cls) -> str: - """Return this extension's schema URI.""" - return SCHEMA_URI + def get_assets( + self, + service_type: Optional[ServiceType] = None, + ) -> dict[str, pystac.Asset]: + """Get the item's assets where eo:bands are defined. + + Args: + service_type: If set, filter the assets such that only those with a + matching :class:`~STACpopulator.stac_utils.ServiceType` are returned. + + Returns + ------- + Dict[str, Asset]: A dictionary of assets that match ``service_type`` + if set or else all of this item's assets were service types are defined. + """ + return { + key: asset + for key, asset in self.item.get_assets().items() + if (service_type is ServiceType and service_type.value in asset.extra_fields) + or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + } + + def __repr__(self) -> str: + """Return repr.""" + return f"" + + +class ItemAssetsCFExtension(CFExtension[item_assets.AssetDefinition]): + """Extention for CF item assets.""" + + properties: dict[str, Any] + asset_defn: item_assets.AssetDefinition + + def __init__(self, item_asset: item_assets.AssetDefinition) -> None: + self.asset_defn = item_asset + self.properties = item_asset.properties + + +class AssetCFExtension(CFExtension[pystac.Asset]): + """ + A concrete implementation of :class:`CFExtension` on an :class:`~pystac.Asset`. + + Extends the Asset fields to include properties defined in the + :stac-ext:`CF Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`CFExtension.ext` on an :class:`~pystac.Asset` to extend it. + """ + + asset_href: str + """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Asset` fields, including extension properties.""" + + additional_read_properties: Optional[Iterable[dict[str, Any]]] = None + """If present, this will be a list containing 1 dictionary representing the + properties of the owning :class:`~pystac.Item`.""" + + def __init__(self, asset: pystac.Asset) -> None: + self.asset_href = asset.href + self.properties = asset.extra_fields + if asset.owner and isinstance(asset.owner, pystac.Item): + self.additional_read_properties = [asset.owner.properties] + + def __repr__(self) -> str: + """Return repr.""" + return f"" + + +class CollectionCFExtension(CFExtension[pystac.Collection]): + """Extension for CF data.""" + + def __init__(self, collection: pystac.Collection) -> None: + self.collection = collection diff --git a/STACpopulator/utils/contact.py b/STACpopulator/extensions/contact.py similarity index 100% rename from STACpopulator/utils/contact.py rename to STACpopulator/extensions/contact.py diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 9b8747f..1f218e8 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -6,10 +6,10 @@ from pystac import STACValidationError from pystac.extensions.datacube import DatacubeExtension -from pystac.extensions.file import AssetFileExtension +from pystac.extensions.file import FileExtension from requests.sessions import Session -from STACpopulator.extensions.cf import CFHelper, CFItemExtension +from STACpopulator.extensions.cf import CFExtension, CFHelper from STACpopulator.extensions.datacube import DataCubeHelper from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties @@ -81,18 +81,21 @@ def create_stac_item( except Exception as e: raise Exception("Failed to add THREDDS extension") from e + asset = item.assets["HTTPServer"] # Add CF Extension try: cf_helper = CFHelper(item_data["variables"]) - cf_ext = CFItemExtension.ext(item, add_if_missing=True) - cf_ext.apply(cf_helper.cf_parameters) + cf_item_ext = CFExtension.ext(item, add_if_missing=True) + cf_item_ext.apply(cf_helper.parameters) + + cf_asset_ext = CFExtension.ext(asset, add_if_missing=True) + cf_asset_ext.apply(cf_helper.parameters) except Exception as e: raise Exception("Failed to add CF extension") from e try: - asset = item.assets["HTTPServer"] file_helper = FileHelper(asset.get_absolute_href()) - file_ext = AssetFileExtension.ext(asset, add_if_missing=True) + file_ext = FileExtension.ext(asset, add_if_missing=True) file_ext.apply( byte_order=file_helper.byte_order, checksum=file_helper.checksum, # FIXME: Displayed as n/a on browser diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 805c80e..18184ee 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -16,10 +16,10 @@ stac_host_reachable, stac_version_match, ) +from STACpopulator.extensions.contact import Contact from STACpopulator.input import GenericLoader from STACpopulator.models import AnyGeometry from STACpopulator.stac_utils import load_config -from STACpopulator.utils.contact import Contact LOGGER = logging.getLogger(__name__) diff --git a/STACpopulator/utils/__init__.py b/STACpopulator/utils/__init__.py deleted file mode 100644 index e69de29..0000000 From 0ca52115b65ba6d92faf58af1b22faebc85b8a94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 8 Oct 2025 11:18:54 -0400 Subject: [PATCH 22/76] fix: make contacts optional --- STACpopulator/populator_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 18184ee..dbfa00a 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -147,8 +147,9 @@ def create_stac_collection(self) -> dict[str, Any]: self._collection_info["providers"] = self.__make_collection_providers() # Add contacts as extra_field - self._collection_info["extra_fields"] = {} - self._collection_info["extra_fields"]["contacts"] = self.__make_collection_contacts() + if "contacts" in self._collection_info: + self._collection_info["extra_fields"] = {} + self._collection_info["extra_fields"]["contacts"] = self.__make_collection_contacts() # Construct links if provided in the config. This needs to be done before constructing a collection object. collection_links = self.__make_collection_links() From a8c299d0e6146fa563ecabc30519b29e3b62dada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 13:31:24 -0400 Subject: [PATCH 23/76] fix: cordex test file fix --- tests/test_cordex.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_cordex.py b/tests/test_cordex.py index a3084e0..6d3058f 100644 --- a/tests/test_cordex.py +++ b/tests/test_cordex.py @@ -5,6 +5,39 @@ from STACpopulator.extensions.cordex6 import Cordex6DataModel, Cordex6DataModelNcML +def get_first_item_attrs(url): + import requests + import xncml + from siphon.catalog import TDSCatalog + + from STACpopulator.stac_utils import np2py + + cat = TDSCatalog(url) + + if cat.datasets.items(): + for item_name, ds in cat.datasets.items(): + r = requests.get(ds.access_urls["NCML"]) + attrs = xncml.Dataset.from_text(r.text).to_cf_dict() + attrs["access_urls"] = ds.access_urls + return np2py(attrs) + + +def make_test_data(): + """Fetches attribute data from the PAVICS THREDDS catalog and stores it in the data/ directory as a json.""" + # Raw CORDEX data + url = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/disk2/ouranos/CORDEX/CMIP6/DD/NAM-12/OURANOS/MPI-ESM1-2-LR/ssp370/r1i1p1f1/CRCM5/v1-r1/day/tas/v20231208/catalog.html" + attrs = get_first_item_attrs(url) + with open("data/cordex6_raw.json", "w") as f: + json.dump(attrs, f, indent=2) + + # NcML CORDEX data + url = "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/RCM-CMIP6/CORDEX/NAM-12/day/catalog.html" + attrs = get_first_item_attrs(url) + with open("data/cordex6_ncml.json", "w") as f: + json.dump(attrs, f, indent=2) + + +@pytest.mark.vcr("test_item_raw.yaml") def test_item_raw(): attrs = json.load(open("tests/data/cordex6_raw.json")) model = Cordex6DataModel.from_data(attrs) From ec29535bd8ee28d21561bf6b5f3b40b6b7d11a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 13:32:15 -0400 Subject: [PATCH 24/76] fix: get assets check service_type type --- STACpopulator/extensions/cmip6.py | 4 ++-- STACpopulator/extensions/rdps.py | 4 ++-- test.py | 0 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 test.py diff --git a/STACpopulator/extensions/cmip6.py b/STACpopulator/extensions/cmip6.py index b7596c8..af9e736 100644 --- a/STACpopulator/extensions/cmip6.py +++ b/STACpopulator/extensions/cmip6.py @@ -308,8 +308,8 @@ def get_assets( return { key: asset for key, asset in self.item.get_assets().items() - if (service_type is ServiceType and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) + or any(ServiceType.from_value(field, default=None) for field in asset.extra_fields) } def __repr__(self) -> str: diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 90457a0..e04f739 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -227,8 +227,8 @@ def get_assets( return { key: asset for key, asset in self.item.get_assets().items() - if (service_type is ServiceType and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) + or any(ServiceType.from_value(field, default=None) for field in asset.extra_fields) } def __repr__(self) -> str: diff --git a/test.py b/test.py deleted file mode 100644 index e69de29..0000000 From 27e2a40b9ba2f88c6685be65702f21624b97771d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 17:58:43 -0400 Subject: [PATCH 25/76] fix: get assets check service_type type --- STACpopulator/extensions/rdps.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index e04f739..eebe241 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import os from datetime import datetime from typing import ( Any, @@ -74,11 +73,19 @@ def __init__(self, attrs: MutableMapping[str, Any], geometry_model: Type[AnyGeom @property def uid(self) -> str: - """Return a unique ID for RDPS data item.""" - item_url = self.attrs["access_urls"]["HTTPServer"] - item_filename = os.path.basename(item_url) - _uid, _ = os.path.splitext(item_filename) - return _uid # filename without extension + """Return a unique ID from the server location. + + For datasets with a DRS, it might might more sense to use the dataset's metadata instead. + """ + if "HTTPServer" in self.attrs["access_urls"]: + location = self.attrs["access_urls"]["HTTPServer"].split("/fileServer/")[1] + elif "OpenDAP" in self.attrs["access_urls"]: + location = self.attrs["access_urls"]["OPENDAP"].split("/dodsC/")[1] + elif "NCML" in self.attrs["access_urls"]: + location = self.attrs["access_urls"]["NCML"].split("/ncml/")[1] + else: + raise ValueError("No valid access URL found in data.") + return location.replace("/", "__") @property def geometry(self) -> AnyGeometry: From 92bc6762d8b964772f6af777b87c44425f01fe0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 18:04:45 -0400 Subject: [PATCH 26/76] fix: contact data model removed --- STACpopulator/extensions/contact.py | 84 ----------------------------- STACpopulator/populator_base.py | 3 -- 2 files changed, 87 deletions(-) delete mode 100644 STACpopulator/extensions/contact.py diff --git a/STACpopulator/extensions/contact.py b/STACpopulator/extensions/contact.py deleted file mode 100644 index 482c125..0000000 --- a/STACpopulator/extensions/contact.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Contact data model.""" - -from dataclasses import dataclass, field -from typing import List, Optional - -import pystac -from dataclasses_json import LetterCase, config, dataclass_json - -SCHEMA_URI = "https://stac-extensions.github.io/contacts/v0.1.1/schema.json" - - -@dataclass_json -@dataclass -class Info: - """Gives contact information for and their "roles".""" - - value: str - """The actual contact information, e.g. the phone number or the email address.""" - - roles: Optional[List[str]] = None - """The type(s) of this contact information, e.g. whether it's at work or at home.""" - - -@dataclass_json -@dataclass -class Address: - """Physical location at which contact can be made.""" - - deliveryPoint: Optional[List[str]] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) - """Address lines for the location, for example a street name and a house number.""" - - city: Optional[str] = None - """City for the location.""" - - administrativeArea: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) - """State or province of the location.""" - - postalCode: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) - """ ZIP or other postal code.""" - - country: Optional[str] = None - """Country of the physical address.""" - - -@dataclass_json -@dataclass -class Contact: - """Provides information about a contact.""" - - name: Optional[str] = None - """The name of the responsible person. Required if organization is missing.""" - - organization: Optional[str] = None - """Organization or affiliation of the contact. Required if name is missing""" - - identifier: Optional[str] = None - """A value uniquely identifying the contact.""" - - position: Optional[str] = None - """The name of the role or position of the responsible person.""" - - description: Optional[str] = None - """Detailed multi-line description to fully explain the STAC entity.""" - - logo: Optional[pystac.Link] = None - """Graphic identifying the contact.""" - - phones: Optional[List[Info]] = None - """Telephone numbers at which contact can be made.""" - - emails: Optional[List[Info]] = None - """Email address at which contact can be made.""" - - addresses: Optional[List[Address]] = None - """Physical location at which contact can be made.""" - - links: Optional[List[pystac.Link]] = None - """Links related to the contact.""" - - contactInstructions: Optional[str] = field(metadata=config(letter_case=LetterCase.CAMEL), default=None) - """Supplemental instructions on how or when to contact the responsible party.""" - - roles: Optional[List[str]] = None - """The set of named duties, job functions and/or permissions associated with this contact.""" diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index dbfa00a..a0c6853 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -16,7 +16,6 @@ stac_host_reachable, stac_version_match, ) -from STACpopulator.extensions.contact import Contact from STACpopulator.input import GenericLoader from STACpopulator.models import AnyGeometry from STACpopulator.stac_utils import load_config @@ -208,8 +207,6 @@ def __make_collection_contacts(self) -> List[Dict]: contacts = [] if "contacts" in self._collection_info: contacts = self._collection_info.pop("contacts") - # FIXME: Collection should support Contact or implement to_dict of Objects in extra_fields - contacts = [Contact(**contact).to_dict() for contact in contacts] return contacts def publish_stac_collection(self, collection_data: dict[str, Any]) -> None: From 57a683695c230423cb556fbea6d7ecb675b45ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 23:44:24 -0400 Subject: [PATCH 27/76] fix: cf extension added using subclasses --- STACpopulator/extensions/cf.py | 45 +-- STACpopulator/extensions/file.py | 35 +- STACpopulator/extensions/rdps.py | 343 +----------------- .../implementations/RDPS_CRIM/add_RDPS.py | 73 +--- .../RDPS_CRIM/collection_config.yml | 12 +- docs/RDPS.md | 8 - 6 files changed, 76 insertions(+), 440 deletions(-) delete mode 100644 docs/RDPS.md diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index e174084..9026140 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -3,7 +3,6 @@ from __future__ import annotations import functools -from dataclasses import dataclass from typing import ( Any, Generic, @@ -18,13 +17,14 @@ ) import pystac -from dataclasses_json import dataclass_json +from pydantic import BaseModel from pystac.extensions import item_assets -from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension, S +from pystac.extensions.base import ExtensionManagementMixin, PropertiesExtension +from STACpopulator.extensions.base import ExtensionHelper from STACpopulator.stac_utils import ServiceType -T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) +T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset) SchemaName = Literal["cf"] SCHEMA_URI = "https://stac-extensions.github.io/cf/v0.2.0/schema.json" PREFIX = f"{get_args(SchemaName)[0]}:" @@ -36,20 +36,20 @@ def add_ext_prefix(name: str) -> str: return PREFIX + name if "datetime" not in name else name -@dataclass_json -@dataclass -class CFParameter: +class CFParameter(BaseModel): """CFParameter.""" - name: str - unit: Optional[str] + def __init__(self, name: str, unit: Optional[str]) -> None: + """CFParameter object init.""" + self.name = name + self.unit = unit def __repr__(self) -> str: """Return string repr.""" return f"" -class CFHelper: +class CFHelper(ExtensionHelper): """CFHelper.""" def __init__(self, variables: dict[str, any]) -> None: @@ -65,14 +65,24 @@ def parameters(self) -> List[CFParameter]: attrs = var.get("attributes", {}) name = attrs.get("standard_name") # Get the required standard name if not name: - # Skip if no valid name - continue - + continue # Skip if no valid name unit = attrs.get("units") or "" parameters.append(CFParameter(name=name, unit=unit)) return parameters + def apply(self, item: T, add_if_missing: bool = True) -> T: + """Apply the Datacube extension to an item.""" + ext = CFExtension.ext(item, add_if_missing=add_if_missing) + ext.apply(parameters=self.parameters) + + # FIXME: This temporary workaround has been added to comply with the (most certainly buggy) validation schema for CF extension + # It should be remove once the PR is integrated since applying on the item should be enough + asset = item.assets["HTTPServer"] + cf_asset_ext = CFExtension.ext(asset, add_if_missing=True) + cf_asset_ext.apply(self.parameters) + return item + class CFExtension( Generic[T], @@ -109,15 +119,6 @@ def get_schema_uri(cls) -> str: """Return this extension's schema URI.""" return SCHEMA_URI - @classmethod - def has_extension(cls, obj: S) -> bool: - """Return True iff the object has an extension for that matches this class' schema URI.""" - # FIXME: this override should be removed once an official and versioned schema is released - # ignore the original implementation logic for a version regex - # since in our case, the VERSION_REGEX is not fulfilled (ie: using 'main' branch, no tag available...) - ext_uri = cls.get_schema_uri() - return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions) - @classmethod def ext(cls, obj: T, add_if_missing: bool = False) -> CFExtension[T]: """Extend the given STAC Object with properties from the :stac-ext:`CF Extension `. diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 66087c8..3517bc3 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,32 +1,39 @@ import hashlib from pathlib import Path -from typing import Optional +from typing import Optional, TypeVar import pystac import requests from netCDF4 import Dataset +from pystac.extensions.file import FileExtension -from STACpopulator.extensions.base import Helper +from STACpopulator.extensions.base import ExtensionHelper +T = TypeVar("T", pystac.Asset, pystac.Link) -class FileHelper(Helper): + +class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" - def __init__(self, href: str, download_dir: str = "temp_files") -> None: + def __init__(self, href: str) -> None: """Initialize the FileHelper object.""" self.href = href - self.download_dir = Path(download_dir) - self.download_dir.mkdir(exist_ok=True) - self.local_path = self.download_dir / Path(href).name - self.download_file() # Download the file + self.local_file = None - # Compute properties - self.size = self.local_path.stat().st_size - self.checksum = self.calculate_checksum() - # self.header_size = self.get_netcdf_header_size(self.local_path) - self.byte_order = self.find_byte_order() + def apply(self, asset: T, add_if_missing: bool = True) -> T: + """Apply the FileExtension to an asset.""" + self.load_file() + file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) + file_ext.apply( + byte_order=self.byte_order, + checksum=self.checksum, # FIXME: Displayed as n/a on browser + header_size=0, # FIXME: clarify expected value here + size=self.size, + values=None, # NOTE: deprecated + local_path=None, # FIXME: clarify expected value here + ) - def download_file(self) -> None: + def load_file(self) -> None: """Download the current file into temporary directory.""" if self.local_path.exists(): return diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index eebe241..508709f 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,333 +1,30 @@ -"""RPDS Extension Module.""" - from __future__ import annotations -import json -from datetime import datetime -from typing import ( - Any, - Generic, - Iterable, - Literal, - MutableMapping, - Optional, - Type, - TypeVar, - Union, - cast, - get_args, -) - -import pystac -from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator -from pydantic.fields import FieldInfo -from pystac.extensions import item_assets -from pystac.extensions.base import ( - ExtensionManagementMixin, - PropertiesExtension, - S, # generic pystac.STACObject - SummariesExtension, -) - -from STACpopulator.models import AnyGeometry -from STACpopulator.stac_utils import ServiceType, ncattrs_to_bbox, ncattrs_to_geometry - -T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) - -SchemaName = Literal["rdps"] -SCHEMA_URI: str = "STACpopulator/extensions/schemas/rdps/rdps-global-attrs-schema.json" # FIXME: To be defined -PREFIX = f"{get_args(SchemaName)[0]}:" +from pydantic import model_validator +from STACpopulator.extensions.cf import CFHelper +from STACpopulator.extensions.file import FileHelper +from STACpopulator.extensions.thredds import THREDDSCatalogDataModel -def add_ext_prefix(name: str) -> str: - """Return the given name prefixed with this extension's prefix.""" - return PREFIX + name if "datetime" not in name else name +# Customize the THREDDSCatalogDataModel +class RDPSDataModel(THREDDSCatalogDataModel): + """Data model for RDPS NetCDF datasets.""" -class RDPSProperties(BaseModel, validate_assignment=True): - """Data model for RDPS.""" + # Extension classes + cf: CFHelper + file: FileHelper - Conventions: str - - _prefix = "rdps" - - model_config = ConfigDict(alias_generator=add_ext_prefix, populate_by_name=True, extra="ignore") - - @field_validator("version", check_fields=False) + @model_validator(mode="before") @classmethod - def validate_version(cls, v: str, __info: ValidationInfo) -> bool: - """Return True iff the given version is valid.""" - assert v[0] == "v", "Version string should begin with a lower case 'v'" - assert v[1:].isdigit(), "All characters in version string, except first, should be digits" - return v - - -class RDPSHelper: - """Helper for RDPS data.""" - - def __init__(self, attrs: MutableMapping[str, Any], geometry_model: Type[AnyGeometry]) -> None: - self.attrs = attrs - self.rdps_attrs = attrs["attributes"] - self.cfmeta = attrs["groups"]["CFMetadata"]["attributes"] - self.geometry_model = geometry_model - - @property - def uid(self) -> str: - """Return a unique ID from the server location. - - For datasets with a DRS, it might might more sense to use the dataset's metadata instead. - """ - if "HTTPServer" in self.attrs["access_urls"]: - location = self.attrs["access_urls"]["HTTPServer"].split("/fileServer/")[1] - elif "OpenDAP" in self.attrs["access_urls"]: - location = self.attrs["access_urls"]["OPENDAP"].split("/dodsC/")[1] - elif "NCML" in self.attrs["access_urls"]: - location = self.attrs["access_urls"]["NCML"].split("/ncml/")[1] - else: - raise ValueError("No valid access URL found in data.") - return location.replace("/", "__") - - @property - def geometry(self) -> AnyGeometry: - """Return the geometry.""" - return self.geometry_model(**ncattrs_to_geometry(self.attrs)) - - @property - def bbox(self) -> list[float]: - """Return the bounding box.""" - return ncattrs_to_bbox(self.attrs) - - @property - def start_datetime(self) -> datetime: - """Return the beginning of the temporal extent.""" - return self.cfmeta["time_coverage_start"] - - @property - def end_datetime(self) -> datetime: - """Return the end of the temporal extent.""" - return self.cfmeta["time_coverage_end"] - - @property - def properties(self) -> RDPSProperties: - """Return properties.""" - props = RDPSProperties(**self.rdps_attrs) - return props - - def stac_item(self) -> "pystac.Item": - """Return a pystac Item.""" - properties = { - "start_datetime": self.start_datetime, - "end_datetime": self.end_datetime, - } - item = pystac.Item( - id=self.uid, - geometry=self.geometry.model_dump(), - bbox=self.bbox, - properties=properties, - datetime=None, - ) - item_rdps = RDPSExtension.ext(item, add_if_missing=True) - item_rdps.apply(self.properties) - return item - - -class RDPSExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[Union[pystac.Asset, pystac.Item, pystac.Collection]], -): - """Extension for RDPS data.""" - - @property - def name(self) -> SchemaName: - """Return the schema name.""" - return get_args(SchemaName)[0] - - def apply( - self, - properties: Union[RDPSProperties, dict[str, Any]], - ) -> None: - """Apply RDPS Extension properties to the extended :class:`~pystac.Item` or :class:`~pystac.Asset`.""" - if isinstance(properties, dict): - properties = RDPSProperties(**properties) - data_json = json.loads(properties.model_dump_json(by_alias=True)) - for prop, val in data_json.items(): - self._set_property(prop, val) - - @classmethod - def get_schema_uri(cls) -> str: - """Return this extension's schema URI.""" - return SCHEMA_URI + def cf_helper(cls, data: dict) -> dict: + """Instantiate the cf helper.""" + data["cf"] = CFHelper(data["data"]["variables"]) + return data + @model_validator(mode="before") @classmethod - def has_extension(cls, obj: S) -> bool: - """Return True iff the object has an extension for that matches this class' schema URI.""" - # FIXME: this override should be removed once an official and versioned schema is released - # ignore the original implementation logic for a version regex - # since in our case, the VERSION_REGEX is not fulfilled (ie: using 'main' branch, no tag available...) - ext_uri = cls.get_schema_uri() - return obj.stac_extensions is not None and any(uri == ext_uri for uri in obj.stac_extensions) - - @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> RDPSExtension[T]: - """Extend the given STAC Object with properties from the :stac-ext:`RDPS Extension `. - - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. - - Raises - ------ - pystac.ExtensionTypeError : If an invalid object type is passed. - """ - if isinstance(obj, pystac.Collection): - cls.ensure_has_extension(obj, add_if_missing) - return cast(RDPSExtension[T], CollectionRDPSExtension(obj)) - elif isinstance(obj, pystac.Item): - cls.ensure_has_extension(obj, add_if_missing) - return cast(RDPSExtension[T], ItemRDPSExtension(obj)) - elif isinstance(obj, pystac.Asset): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(RDPSExtension[T], AssetRDPSExtension(obj)) - elif isinstance(obj, item_assets.AssetDefinition): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(RDPSExtension[T], ItemAssetsRDPSExtension(obj)) - else: - raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) - - @classmethod - def summaries(cls, obj: pystac.Collection, add_if_missing: bool = False) -> "SummariesRDPSExtension": - """Return the extended summaries object for the given collection.""" - cls.ensure_has_extension(obj, add_if_missing) - return SummariesRDPSExtension(obj) - - -class ItemRDPSExtension(RDPSExtension[pystac.Item]): - """ - A concrete implementation of :class:`RDPSExtension` on an :class:`~pystac.Item`. - - Extends the properties of the Item to include properties defined in the - :stac-ext:`RDPS Extension `. - - This class should generally not be instantiated directly. Instead, call - :meth:`RDPSExtension.ext` on an :class:`~pystac.Item` to extend it. - """ - - def __init__(self, item: pystac.Item) -> None: - self.item = item - self.properties = item.properties - - def get_assets( - self, - service_type: Optional[ServiceType] = None, - ) -> dict[str, pystac.Asset]: - """Get the item's assets where eo:bands are defined. - - Args: - service_type: If set, filter the assets such that only those with a - matching :class:`~STACpopulator.stac_utils.ServiceType` are returned. - - Returns - ------- - Dict[str, Asset]: A dictionary of assets that match ``service_type`` - if set or else all of this item's assets were service types are defined. - """ - return { - key: asset - for key, asset in self.item.get_assets().items() - if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) for field in asset.extra_fields) - } - - def __repr__(self) -> str: - """Return repr.""" - return f"" - - -class ItemAssetsRDPSExtension(RDPSExtension[item_assets.AssetDefinition]): - """Extention for RDPS item assets.""" - - properties: dict[str, Any] - asset_defn: item_assets.AssetDefinition - - def __init__(self, item_asset: item_assets.AssetDefinition) -> None: - self.asset_defn = item_asset - self.properties = item_asset.properties - - -class AssetRDPSExtension(RDPSExtension[pystac.Asset]): - """ - A concrete implementation of :class:`RDPSExtension` on an :class:`~pystac.Asset`. - - Extends the Asset fields to include properties defined in the - :stac-ext:`RDPS Extension `. - - This class should generally not be instantiated directly. Instead, call - :meth:`RDPSExtension.ext` on an :class:`~pystac.Asset` to extend it. - """ - - asset_href: str - """The ``href`` value of the :class:`~pystac.Asset` being extended.""" - - properties: dict[str, Any] - """The :class:`~pystac.Asset` fields, including extension properties.""" - - additional_read_properties: Optional[Iterable[dict[str, Any]]] = None - """If present, this will be a list containing 1 dictionary representing the - properties of the owning :class:`~pystac.Item`.""" - - def __init__(self, asset: pystac.Asset) -> None: - self.asset_href = asset.href - self.properties = asset.extra_fields - if asset.owner and isinstance(asset.owner, pystac.Item): - self.additional_read_properties = [asset.owner.properties] - - def __repr__(self) -> str: - """Return repr.""" - return f"" - - -class SummariesRDPSExtension(SummariesExtension): - """ - A concrete implementation of :class:`~SummariesExtension`. - - Extends the ``summaries`` field of a :class:`~pystac.Collection` to include properties - defined in the :stac-ext:`RDPS `. - """ - - def _check_rdps_property(self, prop: str) -> FieldInfo: - try: - return RDPSProperties.model_fields[prop] - except KeyError: - raise AttributeError(f"Name '{prop}' is not a valid RDPS property.") - - def _validate_rdps_property(self, prop: str, summaries: list[Any]) -> None: - model = RDPSProperties.model_construct() - validator = RDPSProperties.__pydantic_validator__ - for value in summaries: - validator.validate_assignment(model, prop, value) - - def get_rdps_property(self, prop: str) -> list[Any]: - """Set RDPS property.""" - self._check_rdps_property(prop) - return self.summaries.get_list(prop) - - def set_rdps_property(self, prop: str, summaries: list[Any]) -> None: - """Set RDPS property.""" - self._check_rdps_property(prop) - self._validate_rdps_property(prop, summaries) - self._set_summary(prop, summaries) - - def __getattr__(self, prop: str) -> list[Any]: - """Get RDPS property.""" - return self.get_rdps_property(prop) - - def __setattr__(self, prop: str, value: Any) -> None: - """Set RDPS property.""" - self.set_rdps_property(prop, value) - - -class CollectionRDPSExtension(RDPSExtension[pystac.Collection]): - """Extension for RDPS data.""" - - def __init__(self, collection: pystac.Collection) -> None: - self.collection = collection + def file_helper(cls, data: dict) -> dict: + """Instantiate the file helper.""" + data["file"] = FileHelper(data["data"]["variables"]) + return data diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 1f218e8..ddcfabf 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -1,21 +1,12 @@ import argparse -import json import logging import os from typing import Any, MutableMapping, Optional, Union -from pystac import STACValidationError -from pystac.extensions.datacube import DatacubeExtension -from pystac.extensions.file import FileExtension from requests.sessions import Session -from STACpopulator.extensions.cf import CFExtension, CFHelper -from STACpopulator.extensions.datacube import DataCubeHelper -from STACpopulator.extensions.file import FileHelper -from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties -from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper +from STACpopulator.extensions.rdps import RDPSDataModel from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader -from STACpopulator.models import GeoJSONPolygon from STACpopulator.populator_base import STACpopulatorBase LOGGER = logging.getLogger(__name__) @@ -24,8 +15,8 @@ class RDPSpopulator(STACpopulatorBase): """Populator that creates STAC objects representing RDPS data from a THREDDS catalog.""" - item_properties_model = RDPSProperties - item_geometry_model = GeoJSONPolygon + data_model = RDPSDataModel + item_geometry_model = None # Unnecessary, but kept for consistency def __init__( self, @@ -55,66 +46,12 @@ def create_stac_item( :return: _description_ :rtype: MutableMapping[str, Any] """ - # Add RDPS extension try: - rdps_helper = RDPSHelper(item_data, self.item_geometry_model) - item = rdps_helper.stac_item() + self.data_model = RDPSDataModel.from_data(item_data) + return self.data_model.stac_item() except Exception as e: raise Exception("Failed to add RDPS extension") from e - # Add DataCube Extension - try: - datacube_helper = DataCubeHelper(item_data) - datacube_ext = DatacubeExtension.ext(item, add_if_missing=True) - datacube_ext.apply( - dimensions=datacube_helper.dimensions, - variables=datacube_helper.variables, - ) - except Exception as e: - raise Exception("Failed to add Datacube extension") from e - - # Add THREDDS Extension - try: - thredds_helper = THREDDSHelper(item_data["access_urls"]) - thredds_ext = THREDDSExtension.ext(item) - thredds_ext.apply(thredds_helper.services, thredds_helper.links) - except Exception as e: - raise Exception("Failed to add THREDDS extension") from e - - asset = item.assets["HTTPServer"] - # Add CF Extension - try: - cf_helper = CFHelper(item_data["variables"]) - cf_item_ext = CFExtension.ext(item, add_if_missing=True) - cf_item_ext.apply(cf_helper.parameters) - - cf_asset_ext = CFExtension.ext(asset, add_if_missing=True) - cf_asset_ext.apply(cf_helper.parameters) - except Exception as e: - raise Exception("Failed to add CF extension") from e - - try: - file_helper = FileHelper(asset.get_absolute_href()) - file_ext = FileExtension.ext(asset, add_if_missing=True) - file_ext.apply( - byte_order=file_helper.byte_order, - checksum=file_helper.checksum, # FIXME: Displayed as n/a on browser - header_size=0, # FIXME: clarify expected value here - size=file_helper.size, - values=None, # NOTE: deprecated - local_path=None, # FIXME: clarify expected value here - ) - except Exception as e: - raise Exception("Failed to add Asset File extension") from e - - try: - item.validate() - except STACValidationError as e: - print(json.dumps(item.to_dict())) - raise Exception("Failed to validate STAC item") from e - - return json.loads(json.dumps(item.to_dict())) - def add_parser_args(parser: argparse.ArgumentParser) -> None: """Add additional CLI arguments to the argument parser.""" diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 9c0d2ae..c97fbae 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -9,9 +9,14 @@ temporalextent: ["1850-01-01", "2500-01-01"] providers: - name: NRCAN description: Natural Resources Canada - roles: + roles: + - licensor - producer url: https://github.com/julemai/CaSPAr/wiki/Available-products + - name: Ouranos + description: Ouranos + roles: + - host links: - rel: about @@ -37,7 +42,4 @@ contacts: title: Logo CRIM stac_extensions: - - DatacubeExtension - - THREDDSExtension - - https://stac-extensions.github.io/cf/v0.2.0/schema.json - - https://stac-extensions.github.io/contacts/v0.1.1/schema.json \ No newline at end of file + - DatacubeExtension \ No newline at end of file diff --git a/docs/RDPS.md b/docs/RDPS.md deleted file mode 100644 index 81393df..0000000 --- a/docs/RDPS.md +++ /dev/null @@ -1,8 +0,0 @@ -# Regional Deterministic Prediction System -Metadata Integration Flow - -## 1. Applied Extensions - -- RDPSHelper (item) -- DatacubeExtension -- THREDDSExtension \ No newline at end of file From d8c7b97f01a120dbdf5c11ce22748164b9e5c8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 9 Oct 2025 23:49:10 -0400 Subject: [PATCH 28/76] fix: hrdps impl. updated using subclass --- STACpopulator/extensions/hrdps.py | 50 +++---------------- STACpopulator/extensions/rdps.py | 9 ---- .../implementations/HRDPS_CRIM/add_HRDPS.py | 39 ++------------- 3 files changed, 10 insertions(+), 88 deletions(-) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index 40f4289..dccf506 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -1,48 +1,10 @@ -"""HRPDS Extension Module.""" +from __future__ import annotations -from typing import ( - Literal, - TypeVar, - get_args, -) +from STACpopulator.extensions.rdps import RDPSDataModel -import pystac -from pydantic import ConfigDict -from pystac.extensions import item_assets -from STACpopulator.extensions.rdps import RDPSHelper, RDPSProperties +# Customize the THREDDSCatalogDataModel +class HRDPSDataModel(RDPSDataModel): + """Data model for RDPS NetCDF datasets.""" -T = TypeVar("T", pystac.Collection, pystac.Item, pystac.Asset, item_assets.AssetDefinition) - -SchemaName = Literal["hrdps"] -SCHEMA_URI: str = "https://gist.githubusercontent.com/henriaidasso/a8546268004d5d3a748556994cfaadb4/raw/ba3884140b5a85585b3afa385ea05996db6414bf/rdps-global-attrs-schema.json" # FIXME: To be defined -PREFIX = f"{get_args(SchemaName)[0]}:" - - -def add_ext_prefix(name: str) -> str: - """Return the given name prefixed with this extension's prefix.""" - return PREFIX + name if "datetime" not in name else name - - -class HRDPSProperties(RDPSProperties, validate_assignment=True): - """Data model for HRDPS.""" - - product: str - - Remarks: str - - License: str - - _prefix = "hrdps" - - model_config = ConfigDict(alias_generator=add_ext_prefix, populate_by_name=True, extra="ignore") - - -class HRDPSHelper(RDPSHelper): - """Helper for HRDPS data.""" - - @property - def properties(self) -> HRDPSProperties: - """Return properties.""" - props = HRDPSProperties(**self.rdps_attrs) - return props + pass diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 508709f..8388a0f 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -3,7 +3,6 @@ from pydantic import model_validator from STACpopulator.extensions.cf import CFHelper -from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.thredds import THREDDSCatalogDataModel @@ -13,7 +12,6 @@ class RDPSDataModel(THREDDSCatalogDataModel): # Extension classes cf: CFHelper - file: FileHelper @model_validator(mode="before") @classmethod @@ -21,10 +19,3 @@ def cf_helper(cls, data: dict) -> dict: """Instantiate the cf helper.""" data["cf"] = CFHelper(data["data"]["variables"]) return data - - @model_validator(mode="before") - @classmethod - def file_helper(cls, data: dict) -> dict: - """Instantiate the file helper.""" - data["file"] = FileHelper(data["data"]["variables"]) - return data diff --git a/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py index 800f7c3..2780e64 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py +++ b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py @@ -1,16 +1,11 @@ import argparse -import json import logging import os from typing import Any, MutableMapping, Optional, Union -from pystac import STACValidationError -from pystac.extensions.datacube import DatacubeExtension from requests.sessions import Session -from STACpopulator.extensions.datacube import DataCubeHelper -from STACpopulator.extensions.hrdps import HRDPSHelper, HRDPSProperties -from STACpopulator.extensions.thredds import THREDDSExtension, THREDDSHelper +from STACpopulator.extensions.hrdps import HRDPSDataModel from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader from STACpopulator.models import GeoJSONPolygon from STACpopulator.populator_base import STACpopulatorBase @@ -21,7 +16,7 @@ class HRDPSpopulator(STACpopulatorBase): """Populator that creates STAC objects representing HRDPS data from a THREDDS catalog.""" - item_properties_model = HRDPSProperties + data_model = HRDPSDataModel item_geometry_model = GeoJSONPolygon def __init__( @@ -52,38 +47,12 @@ def create_stac_item( :return: _description_ :rtype: MutableMapping[str, Any] """ - # Add HRDPS extension try: - hrdps_helper = HRDPSHelper(item_data, self.item_geometry_model) - item = hrdps_helper.stac_item() + self.data_model = HRDPSDataModel.from_data(item_data) + return self.data_model.stac_item() except Exception as e: raise Exception("Failed to add HRDPS extension") from e - # Add datacube extension - try: - datacube_helper = DataCubeHelper(item_data) - datacube_ext = DatacubeExtension.ext(item, add_if_missing=True) - datacube_ext.apply( - dimensions=datacube_helper.dimensions, - variables=datacube_helper.variables, - ) - except Exception as e: - raise Exception("Failed to add Datacube extension") from e - - try: - thredds_helper = THREDDSHelper(item_data["access_urls"]) - thredds_ext = THREDDSExtension.ext(item) - thredds_ext.apply(thredds_helper.services, thredds_helper.links) - except Exception as e: - raise Exception("Failed to add THREDDS extension") from e - - try: - item.validate() - except STACValidationError as e: - raise Exception("Failed to validate STAC item") from e - - return json.loads(json.dumps(item.to_dict())) - def add_parser_args(parser: argparse.ArgumentParser) -> None: """Add additional CLI arguments to the argument parser.""" From 2c207a74d0ba477356f3e1aaa994dbff69e35d69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 00:00:50 -0400 Subject: [PATCH 29/76] fix: cf item extension get assets filter --- STACpopulator/extensions/cf.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 9026140..a11f509 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -31,11 +31,6 @@ PARAMETER_PROP = PREFIX + "parameter" -def add_ext_prefix(name: str) -> str: - """Return the given name prefixed with this extension's prefix.""" - return PREFIX + name if "datetime" not in name else name - - class CFParameter(BaseModel): """CFParameter.""" @@ -52,6 +47,8 @@ def __repr__(self) -> str: class CFHelper(ExtensionHelper): """CFHelper.""" + _prefix = "cf" + def __init__(self, variables: dict[str, any]) -> None: """Take a STAC item variables to identify CF parameters metadata.""" self.variables = variables @@ -80,7 +77,7 @@ def apply(self, item: T, add_if_missing: bool = True) -> T: # It should be remove once the PR is integrated since applying on the item should be enough asset = item.assets["HTTPServer"] cf_asset_ext = CFExtension.ext(asset, add_if_missing=True) - cf_asset_ext.apply(self.parameters) + cf_asset_ext.apply(parameters=self.parameters) return item @@ -178,7 +175,7 @@ def get_assets( return { key: asset for key, asset in self.item.get_assets().items() - if (service_type is ServiceType and service_type.value in asset.extra_fields) + if (service_type in ServiceType and service_type.value in asset.extra_fields) or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) } From c03b5cff4fc25303991ed2716abf6b411f98352a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 00:34:32 -0400 Subject: [PATCH 30/76] fix: remove dataclass-json dependency --- STACpopulator/extensions/cf.py | 16 ++++++---------- STACpopulator/extensions/rdps.py | 2 +- pyproject.toml | 1 - 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index a11f509..b256957 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -5,6 +5,7 @@ import functools from typing import ( Any, + Dict, Generic, Iterable, List, @@ -34,10 +35,8 @@ class CFParameter(BaseModel): """CFParameter.""" - def __init__(self, name: str, unit: Optional[str]) -> None: - """CFParameter object init.""" - self.name = name - self.unit = unit + name: str + unit: Optional[str] def __repr__(self) -> str: """Return string repr.""" @@ -47,11 +46,8 @@ def __repr__(self) -> str: class CFHelper(ExtensionHelper): """CFHelper.""" - _prefix = "cf" - - def __init__(self, variables: dict[str, any]) -> None: - """Take a STAC item variables to identify CF parameters metadata.""" - self.variables = variables + _prefix: str = "cf" + variables: Dict[str, Any] @functools.cached_property def parameters(self) -> List[CFParameter]: @@ -108,7 +104,7 @@ def apply( ) -> None: """Apply CF Extension properties to the extended :class:`~pystac.Item` or :class:`~pystac.Asset`.""" if not isinstance(parameters[0], dict): - parameters = [p.to_dict() for p in parameters] + parameters = [p.model_dump() for p in parameters] self.parameter = parameters @classmethod diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 8388a0f..aa0f8d2 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -17,5 +17,5 @@ class RDPSDataModel(THREDDSCatalogDataModel): @classmethod def cf_helper(cls, data: dict) -> dict: """Instantiate the cf helper.""" - data["cf"] = CFHelper(data["data"]["variables"]) + data["cf"] = CFHelper(variables=data["data"]["variables"]) return data diff --git a/pyproject.toml b/pyproject.toml index 2f9368c..afe7d6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,7 +137,6 @@ dependencies = [ "pyessv~=0.9", "requests~=2.32", "lxml~=6.0", - "dataclasses-json~=0.6.7" ] readme = "README.md" license = { file = "LICENSE" } From 126aa4c0308d8d0592afc11a7028b6b7fe655e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 00:55:42 -0400 Subject: [PATCH 31/76] doc: readme updated --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bd2fea8..7345d8c 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,12 @@ Provided implementations of `STACpopulatorBase`: | Implementation | Description | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| +| [RDPS_CRIM][RDPS_CRIM] | Crawls a THREDDS Catalog for RDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [CMIP6_UofT][CMIP6_UofT] | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [DirectoryLoader][DirLoader] | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint. | | [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | +[RDPS_CRIM]: STACpopulator/implementations/RDPS_CRIM/add_RDPS.py [CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py [DirLoader]: STACpopulator/implementations/DirectoryLoader/crawl_directory.py [CORDEX-CMIP6_Ouranos]: STACpopulator/implementations/CORDEX-CMIP6_Ouranos/add_CORDEX-CMIP6.py From 40fda0731d8afe5a7fde9418c153aa1fd7bb13e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 02:46:14 -0400 Subject: [PATCH 32/76] fix: file extension added using subclass --- .gitignore | 5 +- STACpopulator/extensions/file.py | 105 +++++++++++++++---------------- STACpopulator/extensions/rdps.py | 9 +++ 3 files changed, 60 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index 3ad3e86..a313e9d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,4 @@ build # Old Submodule Path # Could be used locally -pyessv-archive/ - -# Temp files downloaded -temp_files/ \ No newline at end of file +pyessv-archive/ \ No newline at end of file diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 3517bc3..913d306 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,10 +1,10 @@ +import functools import hashlib -from pathlib import Path -from typing import Optional, TypeVar +import io +from typing import Dict, TypeVar import pystac import requests -from netCDF4 import Dataset from pystac.extensions.file import FileExtension from STACpopulator.extensions.base import ExtensionHelper @@ -15,66 +15,61 @@ class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" - def __init__(self, href: str) -> None: - """Initialize the FileHelper object.""" - self.href = href - self.local_file = None + access_urls: Dict[str, str] + _local_file = None + _header_size = None - def apply(self, asset: T, add_if_missing: bool = True) -> T: + def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: """Apply the FileExtension to an asset.""" - self.load_file() + if self._local_file is None: + self.load_file() + + # FIXME: This extension is applicable to Assets and Links. + # Right now it is only applied to the HTTPServer asset to avoid + # too much load/latence when running the populator + asset = item.assets["HTTPServer"] file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) file_ext.apply( - byte_order=self.byte_order, - checksum=self.checksum, # FIXME: Displayed as n/a on browser - header_size=0, # FIXME: clarify expected value here + byte_order=None, # NOTE: Appear to available for each variable. Not sure what would be the value for the whole file. + checksum=self.checksum, + header_size=self.header_size, size=self.size, values=None, # NOTE: deprecated - local_path=None, # FIXME: clarify expected value here + local_path=None, # NOTE: might not be relevant ) def load_file(self) -> None: """Download the current file into temporary directory.""" - if self.local_path.exists(): + if self._local_file is not None: return - r = requests.get(self.href, stream=True) - r.raise_for_status() - with open(self.local_path, "wb") as f: - for chunk in r.iter_content(4096): - f.write(chunk) - - def calculate_checksum(self) -> str: - """Return file md5 checksum value.""" - h = hashlib.md5() - with open(self.local_path, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - h.update(chunk) - return h.hexdigest() - - def find_byte_order(self) -> Optional[pystac.extensions.file.ByteOrder]: - """Detect byte order based on file header content.""" - # TODO: Check TIFF header. To be double-checked - with open(self.local_path, "rb") as f: - header = f.read(2) - if header == b"II": - return "little-endian" - elif header == b"MM": - return "big-endian" - else: - return None - - def get_netcdf_header_size(self, file_path: Path) -> int: - """Estimate the header size of a NetCDF (.nc) file in bytes.""" - with Dataset(file_path, "r") as ds: - variables = ds.variables - if not variables: - # No variables → header = full file? - return file_path.stat().st_size - # NetCDF-3 classic: header ends at the start of first variable - first_var = next(iter(variables.values())) - try: - offset = first_var._offset - except AttributeError: - # NetCDF-4/HDF5: header is less relevant, fallback to 0 - offset = 0 - return offset + response = requests.get(self.access_urls["HTTPServer"], stream=True) + response.raise_for_status() + + # Read the first 64 KB for header/metadata (NetCDF4/HDF5 header) + head_chunk = response.raw.read(64 * 1024) + self._header_size = len(head_chunk) + + # Initialize in-memory buffer with header + buffer = io.BytesIO(head_chunk) + + # Read the rest of the stream and append to buffer while counting + for chunk in response.iter_content(chunk_size=8192): + buffer.write(chunk) + + # Store all data in memory + self._local_file = buffer.getvalue() + + @functools.cached_property + def size(self) -> int: + """Return file size in bytes.""" + return len(self._local_file) + + @functools.cached_property + def checksum(self) -> str: + """Return file sha256 checksum.""" + return hashlib.sha256(self._local_file).hexdigest() + + @functools.cached_property + def header_size(self) -> int: + """Return the header size of the netCDF (.nc) file in bytes.""" + return self._header_size diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index aa0f8d2..132d486 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -3,6 +3,7 @@ from pydantic import model_validator from STACpopulator.extensions.cf import CFHelper +from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.thredds import THREDDSCatalogDataModel @@ -12,6 +13,7 @@ class RDPSDataModel(THREDDSCatalogDataModel): # Extension classes cf: CFHelper + file: FileHelper @model_validator(mode="before") @classmethod @@ -19,3 +21,10 @@ def cf_helper(cls, data: dict) -> dict: """Instantiate the cf helper.""" data["cf"] = CFHelper(variables=data["data"]["variables"]) return data + + @model_validator(mode="before") + @classmethod + def file_helper(cls, data: dict) -> dict: + """Instantiate the file helper.""" + data["file"] = FileHelper(access_urls=data["data"]["access_urls"]) + return data From c968b285f71941f4e717774d8757e3182d6e5129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 02:47:16 -0400 Subject: [PATCH 33/76] fix: hrdps class doctstring updated --- STACpopulator/extensions/hrdps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index dccf506..76e2242 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -5,6 +5,6 @@ # Customize the THREDDSCatalogDataModel class HRDPSDataModel(RDPSDataModel): - """Data model for RDPS NetCDF datasets.""" + """Data model for HRDPS NetCDF datasets.""" pass From b75bb545e737629ca052d87809cde4a2d5df5c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 10 Oct 2025 12:51:28 -0400 Subject: [PATCH 34/76] fix: contact rtype update --- STACpopulator/populator_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index a0c6853..bb23e51 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -198,11 +198,11 @@ def __make_collection_providers(self) -> List[pystac.Provider]: pystac_providers = [pystac.Provider(**provider) for provider in providers] return pystac_providers - def __make_collection_contacts(self) -> List[Dict]: + def __make_collection_contacts(self) -> List[dict]: """Create collection level contacts based on data read in from the configuration file. - :return: List of Contact objects - :rtype: List[Contact] + :return: List of dictionnary contact objects + :rtype: List[dict] """ contacts = [] if "contacts" in self._collection_info: From f22b369cdb7465741e20d7f8747e6f10062f6a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 15 Oct 2025 11:32:53 -0400 Subject: [PATCH 35/76] fix: providers updated --- .../implementations/RDPS_CRIM/collection_config.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index c97fbae..cc0a710 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -7,16 +7,23 @@ spatialextent: [-180, -90, 180, 90] temporalextent: ["1850-01-01", "2500-01-01"] providers: - - name: NRCAN - description: Natural Resources Canada + - name: ECCC + description: Environment and Climate Change Canada (ECCC) roles: - licensor + - processor - producer url: https://github.com/julemai/CaSPAr/wiki/Available-products + - name: MSC + description: Meteorological Service of Canada (MSC) + roles: + - processor - name: Ouranos description: Ouranos roles: + - processor - host + url: https://www.ouranos.ca links: - rel: about From 7e6a764e99c39638ff04ac06886a42ee73ceb384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 15 Oct 2025 13:14:08 -0400 Subject: [PATCH 36/76] fix: collection info updated --- .../implementations/RDPS_CRIM/collection_config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index cc0a710..95e18a8 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -1,7 +1,7 @@ id: RDPS_CRIM title: RDPS -description: Regional Deterministic Prediction System -keywords: ["RDPS", "Climate Change"] +description: Regional Deterministic Prediction System (RDPS) +keywords: ["RDPS", "Climate Change", "ECCC"] license: "other" spatialextent: [-180, -90, 180, 90] temporalextent: ["1850-01-01", "2500-01-01"] @@ -18,6 +18,7 @@ providers: description: Meteorological Service of Canada (MSC) roles: - processor + url: https://eccc-msc.github.io/open-data/ - name: Ouranos description: Ouranos roles: From c462ecd99a9fb74626db7140da386e320f22071b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 15 Oct 2025 14:47:30 -0400 Subject: [PATCH 37/76] fix: file helper updated --- STACpopulator/extensions/file.py | 76 ++++++++++++++------------------ 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 913d306..7046213 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,6 +1,6 @@ +"""FileHelper module.""" + import functools -import hashlib -import io from typing import Dict, TypeVar import pystac @@ -9,67 +9,57 @@ from STACpopulator.extensions.base import ExtensionHelper +# Constants T = TypeVar("T", pystac.Asset, pystac.Link) +HTTP_SERVER_ASSET_KEY = "HTTPServer" +OPEN_DAP_ASSET_KEY = "OpenDAP" class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" access_urls: Dict[str, str] - _local_file = None - _header_size = None + _session = requests.Session() def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: """Apply the FileExtension to an asset.""" - if self._local_file is None: - self.load_file() - # FIXME: This extension is applicable to Assets and Links. - # Right now it is only applied to the HTTPServer asset to avoid - # too much load/latence when running the populator - asset = item.assets["HTTPServer"] + # Currently only applied to the HTTPServer asset to avoid heavy load during populator run + asset = item.assets[HTTP_SERVER_ASSET_KEY] file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) file_ext.apply( - byte_order=None, # NOTE: Appear to available for each variable. Not sure what would be the value for the whole file. - checksum=self.checksum, + byte_order=None, # NOTE: Appears to be variable-related. Unclear what would be the value for the whole file. header_size=self.header_size, size=self.size, - values=None, # NOTE: deprecated - local_path=None, # NOTE: might not be relevant + checksum=None, # NOTE: Should be made available in the metadata on THREDDS catalog + values=None, # NOTE: Deprecated field + local_path=None, # NOTE: Seems to be irrelevant ) - def load_file(self) -> None: - """Download the current file into temporary directory.""" - if self._local_file is not None: - return - response = requests.get(self.access_urls["HTTPServer"], stream=True) - response.raise_for_status() - - # Read the first 64 KB for header/metadata (NetCDF4/HDF5 header) - head_chunk = response.raw.read(64 * 1024) - self._header_size = len(head_chunk) - - # Initialize in-memory buffer with header - buffer = io.BytesIO(head_chunk) - - # Read the rest of the stream and append to buffer while counting - for chunk in response.iter_content(chunk_size=8192): - buffer.write(chunk) - - # Store all data in memory - self._local_file = buffer.getvalue() - @functools.cached_property def size(self) -> int: """Return file size in bytes.""" - return len(self._local_file) - - @functools.cached_property - def checksum(self) -> str: - """Return file sha256 checksum.""" - return hashlib.sha256(self._local_file).hexdigest() + if HTTP_SERVER_ASSET_KEY not in self.access_urls: + return 0 + res = self._session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) + res.raise_for_status() + return int(res.headers.get("Content-Length", None)) @functools.cached_property def header_size(self) -> int: - """Return the header size of the netCDF (.nc) file in bytes.""" - return self._header_size + """Return the header size of the netCDF file in bytes.""" + # FIXME: Implementation to be discussed to confirm compliance with header size definition + if OPEN_DAP_ASSET_KEY not in self.access_urls: + return 0 + + header_size = 0 + # .dds returns variable and dimension structure + # .das returns attributes and metadata + for end in [".dds", ".das"]: + url = self.access_urls[OPEN_DAP_ASSET_KEY] + end + res = self._session.get(url, stream=True) + res.raise_for_status() + + bytes_count: int = sum(len(chunk) for chunk in res.iter_content(chunk_size=8192)) + header_size += bytes_count + return header_size From e38ad09f2e6b9e3a3b7f68f1637ed21c439e5f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 16 Oct 2025 17:03:51 -0400 Subject: [PATCH 38/76] fix: collection links updated --- .../RDPS_CRIM/collection_config.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 95e18a8..0da854c 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -31,10 +31,20 @@ links: title: Project homepage target: https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html + extra_fields: + hreflang: https://eccc-msc.github.io/open-data/readme_en + describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_en + - rel: about + title: Page d'acceuil du projet + target: https://open.canada.ca/data/fr/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d + media_type: text/html + extra_fields: + hreflang: https://eccc-msc.github.io/open-data/readme_fr + describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_fr - rel: license title: License target: https://open.canada.ca/en/open-government-licence-canada - media_type: text/html + media_type: text/plain contacts: - organization: Computer Research Institute of Montreal (CRIM) @@ -47,7 +57,4 @@ contacts: href: https://www.crim.ca/wp-content/uploads/2020/10/logo-color.png rel: icon type: image/png - title: Logo CRIM - -stac_extensions: - - DatacubeExtension \ No newline at end of file + title: CRIM From dd7e4807e469f8b1abb59a0969a22c436dcae766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 21 Oct 2025 17:38:41 -0400 Subject: [PATCH 39/76] fix: warning fixed --- STACpopulator/extensions/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/STACpopulator/extensions/base.py b/STACpopulator/extensions/base.py index 357ada2..2d31ead 100644 --- a/STACpopulator/extensions/base.py +++ b/STACpopulator/extensions/base.py @@ -190,7 +190,8 @@ def create_uid(self) -> str: @model_validator(mode="after") def find_helpers(self) -> "BaseSTAC": """Populate the list of extensions.""" - for key, field in self.model_fields.items(): + # Access model fields from class. From obj will be removed in pydantic v3 + for key, field in self.__class__.model_fields.items(): if isinstance(field.annotation, type) and issubclass(field.annotation, Helper): self._helpers.append(key) return self From 4597fe9023cc30188e1a5db6d6a7e7c3fe877d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 21 Oct 2025 17:38:57 -0400 Subject: [PATCH 40/76] fix: rdps tests updated --- tests/test_rdps.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_rdps.py b/tests/test_rdps.py index eeca56f..a1596a8 100644 --- a/tests/test_rdps.py +++ b/tests/test_rdps.py @@ -2,27 +2,28 @@ import json -from STACpopulator.extensions.thredds import THREDDSCatalogDataModel +from STACpopulator.extensions.hrdps import HRDPSDataModel +from STACpopulator.extensions.rdps import RDPSDataModel def test_rdps(): attrs = json.load(open("tests/data/rdps.json")) - model = THREDDSCatalogDataModel.from_data(attrs) + model = RDPSDataModel.from_data(attrs) item = model.stac_item() - assert set(model._helpers) == {"thredds", "datacube"} + assert set(model._helpers) == {"thredds", "datacube", "cf", "file"} assert item["id"] == "birdhouse__testdata__HRDPS__RDPS_sample__2024010100_000.nc" assert "TD" in item["properties"]["cube:variables"] def test_hrdps(): attrs = json.load(open("tests/data/hrdps_sfc.json")) - model = THREDDSCatalogDataModel.from_data(attrs) + model = HRDPSDataModel.from_data(attrs) item = model.stac_item() assert "HRDPS_P_PR_SFC" in item["properties"]["cube:variables"] assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} attrs = json.load(open("tests/data/hrdps_p_tt.json")) - model = THREDDSCatalogDataModel.from_data(attrs) + model = HRDPSDataModel.from_data(attrs) item = model.stac_item() assert "HRDPS_P_TT_10000" in item["properties"]["cube:variables"] assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} From 29dc7e2e66a83f5227c45070aeb9f2da49a0c9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 21 Oct 2025 17:58:31 -0400 Subject: [PATCH 41/76] fix: populators updated --- .../implementations/HRDPS_CRIM/add_HRDPS.py | 45 +++---------------- .../implementations/RDPS_CRIM/add_RDPS.py | 42 +++-------------- 2 files changed, 13 insertions(+), 74 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py index 2780e64..1fc02a2 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py +++ b/STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py @@ -1,13 +1,11 @@ import argparse import logging -import os -from typing import Any, MutableMapping, Optional, Union +from typing import Any from requests.sessions import Session from STACpopulator.extensions.hrdps import HRDPSDataModel -from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader -from STACpopulator.models import GeoJSONPolygon +from STACpopulator.input import ErrorLoader, THREDDSLoader from STACpopulator.populator_base import STACpopulatorBase LOGGER = logging.getLogger(__name__) @@ -17,41 +15,12 @@ class HRDPSpopulator(STACpopulatorBase): """Populator that creates STAC objects representing HRDPS data from a THREDDS catalog.""" data_model = HRDPSDataModel - item_geometry_model = GeoJSONPolygon + item_geometry_model = None # Unnecessary, but kept for consistency - def __init__( - self, - stac_host: str, - data_loader: GenericLoader, - update: Optional[bool] = False, - session: Optional[Session] = None, - config_file: Optional[Union[os.PathLike[str], str]] = None, - ) -> None: - super().__init__( - stac_host, - data_loader, - update=update, - session=session, - config_file=config_file, - ) - - def create_stac_item( - self, item_name: str, item_data: MutableMapping[str, Any] - ) -> Union[None, MutableMapping[str, Any]]: - """Create a STAC item. - - :param item_name: name of the STAC item. Interpretation of name is left to the input loader implementation - :type item_name: str - :param item_data: dictionary like representation of all information on the item - :type item_data: MutableMapping[str, Any] - :return: _description_ - :rtype: MutableMapping[str, Any] - """ - try: - self.data_model = HRDPSDataModel.from_data(item_data) - return self.data_model.stac_item() - except Exception as e: - raise Exception("Failed to add HRDPS extension") from e + def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]: + """Return a STAC item.""" + dm = self.data_model.from_data(item_data) + return dm.stac_item() def add_parser_args(parser: argparse.ArgumentParser) -> None: diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index ddcfabf..a168826 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -1,12 +1,11 @@ import argparse import logging -import os -from typing import Any, MutableMapping, Optional, Union +from typing import Any from requests.sessions import Session from STACpopulator.extensions.rdps import RDPSDataModel -from STACpopulator.input import ErrorLoader, GenericLoader, THREDDSLoader +from STACpopulator.input import ErrorLoader, THREDDSLoader from STACpopulator.populator_base import STACpopulatorBase LOGGER = logging.getLogger(__name__) @@ -18,39 +17,10 @@ class RDPSpopulator(STACpopulatorBase): data_model = RDPSDataModel item_geometry_model = None # Unnecessary, but kept for consistency - def __init__( - self, - stac_host: str, - data_loader: GenericLoader, - update: Optional[bool] = False, - session: Optional[Session] = None, - config_file: Optional[Union[os.PathLike[str], str]] = None, - ) -> None: - super().__init__( - stac_host, - data_loader, - update=update, - session=session, - config_file=config_file, - ) - - def create_stac_item( - self, item_name: str, item_data: MutableMapping[str, Any] - ) -> Union[None, MutableMapping[str, Any]]: - """Create a STAC item. - - :param item_name: name of the STAC item. Interpretation of name is left to the input loader implementation - :type item_name: str - :param item_data: dictionary like representation of all information on the item - :type item_data: MutableMapping[str, Any] - :return: _description_ - :rtype: MutableMapping[str, Any] - """ - try: - self.data_model = RDPSDataModel.from_data(item_data) - return self.data_model.stac_item() - except Exception as e: - raise Exception("Failed to add RDPS extension") from e + def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]: + """Return a STAC item.""" + dm = self.data_model.from_data(item_data) + return dm.stac_item() def add_parser_args(parser: argparse.ArgumentParser) -> None: From ea75a9b3d10524bb245e1eed76d306a9c381ea9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 21 Oct 2025 18:31:41 -0400 Subject: [PATCH 42/76] doc: comments added --- STACpopulator/extensions/hrdps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index 76e2242..3041907 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -7,4 +7,5 @@ class HRDPSDataModel(RDPSDataModel): """Data model for HRDPS NetCDF datasets.""" + # FIXME: No specific props beyond RPDS. Maybe delete and only keep the RDPS impl.? pass From 26d91358829d73af5a7bef977dce993a6100b807 Mon Sep 17 00:00:00 2001 From: henriaidasso Date: Wed, 22 Oct 2025 10:40:29 -0400 Subject: [PATCH 43/76] fix: update link title Co-authored-by: David Huard --- STACpopulator/implementations/RDPS_CRIM/collection_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 0da854c..832d040 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -35,7 +35,7 @@ links: hreflang: https://eccc-msc.github.io/open-data/readme_en describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_en - rel: about - title: Page d'acceuil du projet + title: Page d'accueil du projet target: https://open.canada.ca/data/fr/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html extra_fields: From acaeea8207d53a46a34e761bc53f9ab06af27eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 22 Oct 2025 14:38:14 -0400 Subject: [PATCH 44/76] fix: add smaller size logo --- STACpopulator/implementations/RDPS_CRIM/collection_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 0da854c..9d41aef 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -54,7 +54,7 @@ contacts: - value: support-geo@crim.ca roles: ["support"] logo: - href: https://www.crim.ca/wp-content/uploads/2020/10/logo-color.png + href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png rel: icon type: image/png title: CRIM From ffa1021ccdce1b68a60dd0ffe0b13824a5bd854f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 22 Oct 2025 22:43:47 -0400 Subject: [PATCH 45/76] doc: readme updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7345d8c..46d47f5 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Provided implementations of `STACpopulatorBase`: | Implementation | Description | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| -| [RDPS_CRIM][RDPS_CRIM] | Crawls a THREDDS Catalog for RDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | +| [RDPS_CRIM][RDPS_CRIM] | Crawls a THREDDS Catalog for RDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [CMIP6_UofT][CMIP6_UofT] | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [DirectoryLoader][DirLoader] | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint. | | [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | From 5b60a09eae1334031b775bdd12734e91b9d014ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 22 Oct 2025 23:27:27 -0400 Subject: [PATCH 46/76] fix: rdps collection info updated --- .../RDPS_CRIM/collection_config.yml | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index a23225c..33fbe6d 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -1,10 +1,23 @@ id: RDPS_CRIM title: RDPS description: Regional Deterministic Prediction System (RDPS) -keywords: ["RDPS", "Climate Change", "ECCC"] +keywords: + [ + "RDPS", + "ECCC", + "MSC", + "Climate Change", + "Prediction", + "Deterministic", + "Meteorological data", + "Weather and Climate", + "Meteorological Service of Canada", + "Environment and Climate Change Canada", + "Weather and Environmental Operations", + ] license: "other" -spatialextent: [-180, -90, 180, 90] -temporalextent: ["1850-01-01", "2500-01-01"] +spatialextent: [-152.7685268, 27.2840148, -40.6938032, 70.6164825] +temporalextent: ["2014-11-18", "2025-04-01"] providers: - name: ECCC @@ -32,19 +45,19 @@ links: target: https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_en + hreflang: https://eccc-msc.github.io/open-data/readme_en describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_en - rel: about title: Page d'accueil du projet target: https://open.canada.ca/data/fr/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_fr + hreflang: https://eccc-msc.github.io/open-data/readme_fr describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_fr - rel: license title: License target: https://open.canada.ca/en/open-government-licence-canada - media_type: text/plain + media_type: text/html contacts: - organization: Computer Research Institute of Montreal (CRIM) From af32a48669e8d08c2bc9cb8a3c293c19416015a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 10:01:54 -0400 Subject: [PATCH 47/76] fix: update model_fields access --- STACpopulator/extensions/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/base.py b/STACpopulator/extensions/base.py index 2d31ead..934cc99 100644 --- a/STACpopulator/extensions/base.py +++ b/STACpopulator/extensions/base.py @@ -191,7 +191,7 @@ def create_uid(self) -> str: def find_helpers(self) -> "BaseSTAC": """Populate the list of extensions.""" # Access model fields from class. From obj will be removed in pydantic v3 - for key, field in self.__class__.model_fields.items(): + for key, field in type(self).model_fields.items(): if isinstance(field.annotation, type) and issubclass(field.annotation, Helper): self._helpers.append(key) return self From e24834ac8090db1bb57fa9a5154f5b4e0b483392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 10:10:45 -0400 Subject: [PATCH 48/76] fix: service type check in get_assets func --- STACpopulator/extensions/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/base.py b/STACpopulator/extensions/base.py index 934cc99..363854b 100644 --- a/STACpopulator/extensions/base.py +++ b/STACpopulator/extensions/base.py @@ -329,7 +329,7 @@ def get_assets( return { key: asset for key, asset in self.item.get_assets().items() - if (service_type is ServiceType and service_type.value in asset.extra_fields) + if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) } From e44235983902e72071a3434302649610f7a18665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 10:55:16 -0400 Subject: [PATCH 49/76] fix: from data return type --- STACpopulator/extensions/thredds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/thredds.py b/STACpopulator/extensions/thredds.py index c352c2b..329315b 100644 --- a/STACpopulator/extensions/thredds.py +++ b/STACpopulator/extensions/thredds.py @@ -180,7 +180,7 @@ class THREDDSCatalogDataModel(BaseSTAC): model_config = ConfigDict(populate_by_name=True, extra="ignore", arbitrary_types_allowed=True) @classmethod - def from_data(cls, data: dict) -> None: + def from_data(cls, data: dict) -> "THREDDSCatalogDataModel": """ Instantiate class from data provided by THREDDS Loader. From 85f70f1517f5a65b5e14000d07175e809980c4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 13:38:35 -0400 Subject: [PATCH 50/76] fix: reuse populator session in file helper --- CHANGES.md | 12 +++++++ STACpopulator/extensions/file.py | 34 ++++++++++++++++--- STACpopulator/extensions/rdps.py | 5 +++ .../implementations/RDPS_CRIM/add_RDPS.py | 3 +- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4288753..f3fd4b8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,18 @@ +## [0.10.0](https://github.com/crim-ca/stac-populator/tree/0.10.0) (2025-10-23) + +* Add `RDPS_CRIM` and `HRDPS_CRIM` implementation +* Add `cf` extension adding CF Parameter metadata to stac asset and items. +* Add `cf` and `file` helpers. +* Fix deprecated access to `model_fields` in BaseSTAC +* Fix bug service type check in extensions' get_assets method +* Fix return type of from_data in THREDDSCatalogDataModel +* Update RDPS and HRDPS tests. + + + ## [0.9.0](https://github.com/crim-ca/stac-populator/tree/0.9.0) (2025-08-26) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 7046213..ab6db43 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,11 +1,11 @@ """FileHelper module.""" import functools -from typing import Dict, TypeVar +from typing import Dict, Optional, TypeVar import pystac -import requests from pystac.extensions.file import FileExtension +from requests import Session from STACpopulator.extensions.base import ExtensionHelper @@ -19,7 +19,19 @@ class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" access_urls: Dict[str, str] - _session = requests.Session() + _session: Optional[Session] = None + + def __init__(self, access_urls: dict[str, str], session: Optional[Session] = None) -> None: + """Initialize the file helper. + + Parameters + ---------- + access_urls : dict + Dictionary of catalog access URLs. + session : requests.Session, optional + Requests session object to use for HTTP requests. Can be specified post init. + """ + super().__init__(access_urls=access_urls, _session=session) def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: """Apply the FileExtension to an asset.""" @@ -36,12 +48,24 @@ def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: local_path=None, # NOTE: Seems to be irrelevant ) + @property + def session(self) -> Session: + """Get requests session.""" + if self._session is None: + raise ValueError("Please set a valid `session` or provide a default requests.Session().") + return self._session + + @session.setter + def session(self, value: Session) -> None: + """Set requests session.""" + self._session = value + @functools.cached_property def size(self) -> int: """Return file size in bytes.""" if HTTP_SERVER_ASSET_KEY not in self.access_urls: return 0 - res = self._session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) + res = self.session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) res.raise_for_status() return int(res.headers.get("Content-Length", None)) @@ -57,7 +81,7 @@ def header_size(self) -> int: # .das returns attributes and metadata for end in [".dds", ".das"]: url = self.access_urls[OPEN_DAP_ASSET_KEY] + end - res = self._session.get(url, stream=True) + res = self.session.get(url, stream=True) res.raise_for_status() bytes_count: int = sum(len(chunk) for chunk in res.iter_content(chunk_size=8192)) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 132d486..bc80436 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,6 +1,7 @@ from __future__ import annotations from pydantic import model_validator +from requests import Session from STACpopulator.extensions.cf import CFHelper from STACpopulator.extensions.file import FileHelper @@ -28,3 +29,7 @@ def file_helper(cls, data: dict) -> dict: """Instantiate the file helper.""" data["file"] = FileHelper(access_urls=data["data"]["access_urls"]) return data + + def set_session(self, session: Session) -> None: + """Set session parameter to helper(s).""" + self.file.session = session diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index a168826..87bb0b1 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -19,7 +19,8 @@ class RDPSpopulator(STACpopulatorBase): def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]: """Return a STAC item.""" - dm = self.data_model.from_data(item_data) + dm: RDPSDataModel = self.data_model.from_data(item_data) + dm.set_session(self._session) return dm.stac_item() From faae3a16d5085b015a4bcebc906122e8c7f754ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 13:52:25 -0400 Subject: [PATCH 51/76] fix: reuse populator session in file helper --- STACpopulator/extensions/file.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index ab6db43..eaa53a2 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -4,6 +4,7 @@ from typing import Dict, Optional, TypeVar import pystac +import requests from pystac.extensions.file import FileExtension from requests import Session @@ -29,7 +30,7 @@ def __init__(self, access_urls: dict[str, str], session: Optional[Session] = Non access_urls : dict Dictionary of catalog access URLs. session : requests.Session, optional - Requests session object to use for HTTP requests. Can be specified post init. + Requests session object to use for HTTP requests. Defaults to requests.Session(). """ super().__init__(access_urls=access_urls, _session=session) @@ -40,9 +41,9 @@ def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: asset = item.assets[HTTP_SERVER_ASSET_KEY] file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) file_ext.apply( - byte_order=None, # NOTE: Appears to be variable-related. Unclear what would be the value for the whole file. - header_size=self.header_size, size=self.size, + byte_order=None, # NOTE: Appears to be variable-related. Unclear what would be the value for the whole file. + header_size=None, # NOTE: No utility yet available. Might not be relevant. checksum=None, # NOTE: Should be made available in the metadata on THREDDS catalog values=None, # NOTE: Deprecated field local_path=None, # NOTE: Seems to be irrelevant @@ -52,7 +53,8 @@ def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: def session(self) -> Session: """Get requests session.""" if self._session is None: - raise ValueError("Please set a valid `session` or provide a default requests.Session().") + # Initialize only on first call to allow setting value post init. + self._session = requests.Session() return self._session @session.setter @@ -68,22 +70,3 @@ def size(self) -> int: res = self.session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) res.raise_for_status() return int(res.headers.get("Content-Length", None)) - - @functools.cached_property - def header_size(self) -> int: - """Return the header size of the netCDF file in bytes.""" - # FIXME: Implementation to be discussed to confirm compliance with header size definition - if OPEN_DAP_ASSET_KEY not in self.access_urls: - return 0 - - header_size = 0 - # .dds returns variable and dimension structure - # .das returns attributes and metadata - for end in [".dds", ".das"]: - url = self.access_urls[OPEN_DAP_ASSET_KEY] + end - res = self.session.get(url, stream=True) - res.raise_for_status() - - bytes_count: int = sum(len(chunk) for chunk in res.iter_content(chunk_size=8192)) - header_size += bytes_count - return header_size From 24a546d302d2bd1985b6dca92ad07e965c63ad75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 14:24:29 -0400 Subject: [PATCH 52/76] fix: rdps extension updated --- STACpopulator/extensions/rdps.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index bc80436..95b98d7 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -8,7 +8,6 @@ from STACpopulator.extensions.thredds import THREDDSCatalogDataModel -# Customize the THREDDSCatalogDataModel class RDPSDataModel(THREDDSCatalogDataModel): """Data model for RDPS NetCDF datasets.""" @@ -20,16 +19,24 @@ class RDPSDataModel(THREDDSCatalogDataModel): @classmethod def cf_helper(cls, data: dict) -> dict: """Instantiate the cf helper.""" - data["cf"] = CFHelper(variables=data["data"]["variables"]) + field = "cf" + data[field] = cls.helper_class(field)(variables=data["data"]["variables"]) return data @model_validator(mode="before") @classmethod def file_helper(cls, data: dict) -> dict: """Instantiate the file helper.""" - data["file"] = FileHelper(access_urls=data["data"]["access_urls"]) + field = "file" + data[field] = cls.helper_class(field)(access_urls=data["data"]["access_urls"]) return data + @classmethod + def helper_class(cls, field_name: str) -> type[any]: + """Return type annotation of a model field.""" + # TODO: Should maybe be moved to parent for reuse? + return cls.model_fields[field_name].annotation + def set_session(self, session: Session) -> None: """Set session parameter to helper(s).""" self.file.session = session From 0b10d2ac706421576f56b7fed1f8226c28276556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 15:27:13 -0400 Subject: [PATCH 53/76] fix: collection level metadata updated --- .../HRDPS_CRIM/collection_config.yml | 84 +++++++++++++++++-- .../RDPS_CRIM/collection_config.yml | 29 +++++-- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index f825d87..bd08484 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -1,17 +1,89 @@ title: HRDPS id: HRDPS_CRIM description: High Resolution Deterministic Prediction System -keywords: ['HRDPS', 'Climate Change'] -license: "CC-BY-4.0" -spatialextent: [-180, -90, 180, 90] -temporalextent: ['1850-01-01', '2500-01-01'] +keywords: + [ + "HRDPS", + "ECCC", + "MSC", + "Climate Change", + "Prediction", + "High resolution", + "Deterministic", + "Meteorological data", + "Weather and Climate", + "Meteorological Service of Canada", + "Environment and Climate Change Canada", + "Weather and Environmental Operations", + ] +license: other +spatialextent: [-152.7685268, 27.2840148, -40.6938032, 70.6164825] +temporalextent: ["2014-11-18", "2025-04-01"] + +providers: + - name: ECCC + description: Environment and Climate Change Canada (ECCC) + roles: + - licensor + - processor + - producer + url: https://github.com/julemai/CaSPAr/wiki/Available-products + - name: MSC + description: Meteorological Service of Canada (MSC) + roles: + - processor + url: https://eccc-msc.github.io/open-data + - name: Ouranos + description: Ouranos + roles: + - processor + - host + url: https://www.ouranos.ca links: - rel: about title : Project homepage target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d media_type: text/html + extra_fields: + hreflang: https://eccc-msc.github.io/open-data/readme_en + describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en + - rel: about + title : Page d'accueil du projet + target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d + media_type: text/html + extra_fields: + hreflang: https://eccc-msc.github.io/open-data/readme_fr + describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_fr - rel: license - title : License + title : Open Government Licence - Canada target : https://open.canada.ca/en/open-government-licence-canada - media_type: text/plain + media_type: text/html + +contacts: + - organization: Computer Research Institute of Montreal (CRIM) + identifier: CRIM + position: Support + emails: + - value: support-geo@crim.ca + roles: ["work"] + logo: + href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png + rel: icon + type: image/png + title: CRIM + - organization: Environment and Climate Change Canada (ECCC) + identifier: ECCC + position: Support + emails: + - value: ECWeather-Meteo@ec.gc.ca + roles: ["work"] + phones: + - value: +18199972800 + roles: ["work"] + addresses: + - deliveryPoint: 77 Westmorland Street, suite 260 + city: Fredericton + administrativeArea: New Brunswick + postalCode: E3B 6Z4 + country: Canada \ No newline at end of file diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 33fbe6d..b58fa28 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -8,16 +8,18 @@ keywords: "MSC", "Climate Change", "Prediction", + "Regional", "Deterministic", + "Weather forecasts", "Meteorological data", "Weather and Climate", "Meteorological Service of Canada", "Environment and Climate Change Canada", "Weather and Environmental Operations", ] -license: "other" -spatialextent: [-152.7685268, 27.2840148, -40.6938032, 70.6164825] -temporalextent: ["2014-11-18", "2025-04-01"] +license: other +spatialextent: [-166.3, 7.2, -14.4, 85.0] +temporalextent: ["1986-04-22", "2025-04-01"] providers: - name: ECCC @@ -31,7 +33,7 @@ providers: description: Meteorological Service of Canada (MSC) roles: - processor - url: https://eccc-msc.github.io/open-data/ + url: https://eccc-msc.github.io/open-data - name: Ouranos description: Ouranos roles: @@ -55,7 +57,7 @@ links: hreflang: https://eccc-msc.github.io/open-data/readme_fr describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_fr - rel: license - title: License + title: Open Government Licence - Canada target: https://open.canada.ca/en/open-government-licence-canada media_type: text/html @@ -65,9 +67,24 @@ contacts: position: Support emails: - value: support-geo@crim.ca - roles: ["support"] + roles: ["work"] logo: href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png rel: icon type: image/png title: CRIM + - organization: Environment and Climate Change Canada (ECCC) + identifier: ECCC + position: Support + emails: + - value: ECWeather-Meteo@ec.gc.ca + roles: ["work"] + phones: + - value: +18199972800 + roles: ["work"] + addresses: + - deliveryPoint: 77 Westmorland Street, suite 260 + city: Fredericton + administrativeArea: New Brunswick + postalCode: E3B 6Z4 + country: Canada \ No newline at end of file From 0443acd9b9994d4363885c7b7dee0e4d3bf1a9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 15:35:06 -0400 Subject: [PATCH 54/76] fix: update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a313e9d..603532e 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,4 @@ build # Old Submodule Path # Could be used locally -pyessv-archive/ \ No newline at end of file +pyessv-archive/ From dfe1437263b5f3a8446bb342076bc217c6c1f8f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 15:48:21 -0400 Subject: [PATCH 55/76] fix: remove blank lines --- CHANGES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f3fd4b8..9b0e8df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,8 +14,6 @@ * Fix return type of from_data in THREDDSCatalogDataModel * Update RDPS and HRDPS tests. - - ## [0.9.0](https://github.com/crim-ca/stac-populator/tree/0.9.0) (2025-08-26) From 387baa7d3b16d7da4d1ef8ad9a62c08a26a7eec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 16:24:12 -0400 Subject: [PATCH 56/76] fix: set file asset key in helper init --- STACpopulator/extensions/file.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index eaa53a2..ded651c 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -12,33 +12,39 @@ # Constants T = TypeVar("T", pystac.Asset, pystac.Link) -HTTP_SERVER_ASSET_KEY = "HTTPServer" -OPEN_DAP_ASSET_KEY = "OpenDAP" class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" access_urls: Dict[str, str] - _session: Optional[Session] = None + asset_key: str + _session: Optional[Session] - def __init__(self, access_urls: dict[str, str], session: Optional[Session] = None) -> None: + def __init__( + self, + access_urls: dict[str, str], + asset_key: Optional[str] = "HTTPServer", + session: Optional[Session] = None, + ) -> None: """Initialize the file helper. Parameters ---------- - access_urls : dict + access_urls : dict[str, str] Dictionary of catalog access URLs. + asset_key : str, optional. + Asset key matching main file in access_urls. Defaults to `HTTPServer`. session : requests.Session, optional - Requests session object to use for HTTP requests. Defaults to requests.Session(). + Requests session object to use for HTTP requests. Defaults to `requests.Session()`. """ - super().__init__(access_urls=access_urls, _session=session) + super().__init__(access_urls=access_urls, asset_key=asset_key, _session=session) def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: """Apply the FileExtension to an asset.""" # FIXME: This extension is applicable to Assets and Links. - # Currently only applied to the HTTPServer asset to avoid heavy load during populator run - asset = item.assets[HTTP_SERVER_ASSET_KEY] + # Currently applied to the HTTPServer asset by default to avoid heavy load during populator run + asset = item.assets[self.asset_key] file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) file_ext.apply( size=self.size, @@ -65,8 +71,8 @@ def session(self, value: Session) -> None: @functools.cached_property def size(self) -> int: """Return file size in bytes.""" - if HTTP_SERVER_ASSET_KEY not in self.access_urls: + if self.asset_key not in self.access_urls: return 0 - res = self.session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) + res = self.session.head(self.access_urls[self.asset_key]) res.raise_for_status() return int(res.headers.get("Content-Length", None)) From 11029e4acd89b85d2d193fdf257b2d68d379baa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 23 Oct 2025 16:40:36 -0400 Subject: [PATCH 57/76] fix: udpate default variable values --- STACpopulator/extensions/file.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index ded651c..985434e 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -18,13 +18,13 @@ class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" access_urls: Dict[str, str] - asset_key: str - _session: Optional[Session] + asset_key: str = "HTTPServer" + _session: Optional[Session] = None def __init__( self, access_urls: dict[str, str], - asset_key: Optional[str] = "HTTPServer", + asset_key: str = "HTTPServer", session: Optional[Session] = None, ) -> None: """Initialize the file helper. @@ -33,7 +33,7 @@ def __init__( ---------- access_urls : dict[str, str] Dictionary of catalog access URLs. - asset_key : str, optional. + asset_key : str. Asset key matching main file in access_urls. Defaults to `HTTPServer`. session : requests.Session, optional Requests session object to use for HTTP requests. Defaults to `requests.Session()`. From f904c4c3cf3e116a6cecfdf5b0874675d533d953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 24 Oct 2025 09:29:40 -0400 Subject: [PATCH 58/76] fix: optional file size if asset key absent --- STACpopulator/extensions/file.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 985434e..0b1c43c 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,6 +1,7 @@ """FileHelper module.""" import functools +import logging from typing import Dict, Optional, TypeVar import pystac @@ -12,6 +13,7 @@ # Constants T = TypeVar("T", pystac.Asset, pystac.Link) +logger = logging.getLogger(__name__) class FileHelper(ExtensionHelper): @@ -69,10 +71,11 @@ def session(self, value: Session) -> None: self._session = value @functools.cached_property - def size(self) -> int: - """Return file size in bytes.""" + def size(self) -> Optional[int]: + """Return file size in bytes, None if asset_key not in access URLs dictionnary.""" if self.asset_key not in self.access_urls: - return 0 + logger.warning("Asset key %s is not present in access URLs.", self.asset_key) + return None res = self.session.head(self.access_urls[self.asset_key]) res.raise_for_status() return int(res.headers.get("Content-Length", None)) From 01ac7dac745e957018c730f75026863a77a69595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 29 Oct 2025 17:15:05 -0400 Subject: [PATCH 59/76] fix: refactor helpers instantiation --- STACpopulator/extensions/base.py | 10 +++ STACpopulator/extensions/cf.py | 9 +++ STACpopulator/extensions/datacube.py | 9 +++ STACpopulator/extensions/file.py | 66 +++++++++++-------- STACpopulator/extensions/hrdps.py | 2 +- STACpopulator/extensions/rdps.py | 29 -------- STACpopulator/extensions/thredds.py | 57 ++++++++++++---- .../implementations/RDPS_CRIM/add_RDPS.py | 3 +- 8 files changed, 115 insertions(+), 70 deletions(-) diff --git a/STACpopulator/extensions/base.py b/STACpopulator/extensions/base.py index 363854b..6653ff4 100644 --- a/STACpopulator/extensions/base.py +++ b/STACpopulator/extensions/base.py @@ -64,6 +64,16 @@ class Helper: """Class to be subclassed by extension helpers.""" + @classmethod + @abstractmethod + def from_data( + cls, + data: dict[str, Any], + **kwargs, + ) -> "Helper": + """Create a Helper instance from raw data.""" + pass + class ExtensionHelper(BaseModel, Helper): """Base class for dataset properties going into the catalog. diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index b256957..019d19b 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -64,6 +64,15 @@ def parameters(self) -> List[CFParameter]: return parameters + @classmethod + def from_data( + cls, + data: dict[str, Any], + **kwargs, + ) -> "CFHelper": + """Create a CFHelper instance from raw data.""" + return cls(variables=data["data"]["variables"], **kwargs) + def apply(self, item: T, add_if_missing: bool = True) -> T: """Apply the Datacube extension to an item.""" ext = CFExtension.ext(item, add_if_missing=add_if_missing) diff --git a/STACpopulator/extensions/datacube.py b/STACpopulator/extensions/datacube.py index 6167acc..6b8e186 100644 --- a/STACpopulator/extensions/datacube.py +++ b/STACpopulator/extensions/datacube.py @@ -141,6 +141,15 @@ def __init__(self, attrs: MutableMapping[str, Any]) -> None: }, } + @classmethod + def from_data( + cls, + data: dict[str, Any], + **kwargs, + ) -> "DataCubeHelper": + """Create a DataCubeHelper instance from raw data.""" + return cls(attrs=data["data"]) + @property @functools.cache def dimensions(self) -> dict[str, Dimension]: diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index eaa53a2..7d9fb46 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -1,10 +1,12 @@ """FileHelper module.""" import functools +import logging from typing import Dict, Optional, TypeVar import pystac import requests +from pydantic import ConfigDict, Field from pystac.extensions.file import FileExtension from requests import Session @@ -12,33 +14,55 @@ # Constants T = TypeVar("T", pystac.Asset, pystac.Link) -HTTP_SERVER_ASSET_KEY = "HTTPServer" -OPEN_DAP_ASSET_KEY = "OpenDAP" +logger = logging.getLogger(__name__) class FileHelper(ExtensionHelper): """Helper to handle file info from elements of types Asset and Link.""" access_urls: Dict[str, str] - _session: Optional[Session] = None + asset_key: str = "HTTPServer" + session: Optional[Session] = Field(default=None, exclude=True) - def __init__(self, access_urls: dict[str, str], session: Optional[Session] = None) -> None: + model_config = ConfigDict(arbitrary_types_allowed=True) + + def __init__( + self, + access_urls: dict[str, str], + asset_key: str = "HTTPServer", + session: Optional[Session] = None, + ) -> None: """Initialize the file helper. Parameters ---------- - access_urls : dict + access_urls : dict[str, str] Dictionary of catalog access URLs. + asset_key : str. + Asset key matching main file in access_urls. Defaults to `HTTPServer`. session : requests.Session, optional - Requests session object to use for HTTP requests. Defaults to requests.Session(). + Requests session object to use for HTTP requests. Defaults to `requests.Session()`. """ - super().__init__(access_urls=access_urls, _session=session) + super().__init__( + access_urls=access_urls, + asset_key=asset_key, + session=session or requests.Session(), + ) + + @classmethod + def from_data( + cls, + data: dict[str, any], + **kwargs, + ) -> "FileHelper": + """Create a FileHelper instance from raw data.""" + return cls(access_urls=data["data"]["access_urls"], **kwargs) def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: """Apply the FileExtension to an asset.""" # FIXME: This extension is applicable to Assets and Links. - # Currently only applied to the HTTPServer asset to avoid heavy load during populator run - asset = item.assets[HTTP_SERVER_ASSET_KEY] + # Currently applied to the HTTPServer asset by default to avoid heavy load during populator run + asset = item.assets[self.asset_key] file_ext = FileExtension.ext(asset, add_if_missing=add_if_missing) file_ext.apply( size=self.size, @@ -49,24 +73,12 @@ def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: local_path=None, # NOTE: Seems to be irrelevant ) - @property - def session(self) -> Session: - """Get requests session.""" - if self._session is None: - # Initialize only on first call to allow setting value post init. - self._session = requests.Session() - return self._session - - @session.setter - def session(self, value: Session) -> None: - """Set requests session.""" - self._session = value - @functools.cached_property - def size(self) -> int: - """Return file size in bytes.""" - if HTTP_SERVER_ASSET_KEY not in self.access_urls: - return 0 - res = self.session.head(self.access_urls[HTTP_SERVER_ASSET_KEY]) + def size(self) -> Optional[int]: + """Return file size in bytes, None if asset_key not in access URLs dictionnary.""" + if self.asset_key not in self.access_urls: + logger.warning("Asset key %s is not present in access URLs.", self.asset_key) + return None + res = self.session.head(self.access_urls[self.asset_key]) res.raise_for_status() return int(res.headers.get("Content-Length", None)) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index 3041907..3204dfb 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -7,5 +7,5 @@ class HRDPSDataModel(RDPSDataModel): """Data model for HRDPS NetCDF datasets.""" - # FIXME: No specific props beyond RPDS. Maybe delete and only keep the RDPS impl.? + # FIXME: No specific props beyond RPDS. Kept to facilitate evolution. pass diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 95b98d7..3b480af 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,8 +1,5 @@ from __future__ import annotations -from pydantic import model_validator -from requests import Session - from STACpopulator.extensions.cf import CFHelper from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.thredds import THREDDSCatalogDataModel @@ -14,29 +11,3 @@ class RDPSDataModel(THREDDSCatalogDataModel): # Extension classes cf: CFHelper file: FileHelper - - @model_validator(mode="before") - @classmethod - def cf_helper(cls, data: dict) -> dict: - """Instantiate the cf helper.""" - field = "cf" - data[field] = cls.helper_class(field)(variables=data["data"]["variables"]) - return data - - @model_validator(mode="before") - @classmethod - def file_helper(cls, data: dict) -> dict: - """Instantiate the file helper.""" - field = "file" - data[field] = cls.helper_class(field)(access_urls=data["data"]["access_urls"]) - return data - - @classmethod - def helper_class(cls, field_name: str) -> type[any]: - """Return type annotation of a model field.""" - # TODO: Should maybe be moved to parent for reuse? - return cls.model_fields[field_name].annotation - - def set_session(self, session: Session) -> None: - """Set session parameter to helper(s).""" - self.file.session = session diff --git a/STACpopulator/extensions/thredds.py b/STACpopulator/extensions/thredds.py index 329315b..b0e835b 100644 --- a/STACpopulator/extensions/thredds.py +++ b/STACpopulator/extensions/thredds.py @@ -1,3 +1,4 @@ +import inspect from typing import Generic, TypeVar, Union, cast import pystac @@ -135,6 +136,15 @@ class THREDDSHelper(Helper): def __init__(self, access_urls: dict[str, str]) -> None: self.access_urls = {ServiceType.from_value(svc): url for svc, url in access_urls.items()} + @classmethod + def from_data( + cls, + data: dict[str, any], + **kwargs, + ) -> "THREDDSHelper": + """Create a THREDDSHelper instance from raw data.""" + return cls(access_urls=data["data"]["access_urls"]) + @property def services(self) -> list[THREDDSService]: """Return a list of THREDDS services including one for this helper.""" @@ -180,13 +190,16 @@ class THREDDSCatalogDataModel(BaseSTAC): model_config = ConfigDict(populate_by_name=True, extra="ignore", arbitrary_types_allowed=True) @classmethod - def from_data(cls, data: dict) -> "THREDDSCatalogDataModel": + def from_data(cls, data: dict, **kwargs) -> "THREDDSCatalogDataModel": """ Instantiate class from data provided by THREDDS Loader. This is where we match the Loader's output to the STAC item and extensions inputs. If we had multiple loaders, that's probably the only thing that would be different between them. """ + # Inject kwargs for helpers into data + data["_extra_kwargs"] = kwargs + return cls( data=data, start_datetime=data["groups"]["CFMetadata"]["attributes"]["time_coverage_start"], @@ -197,16 +210,38 @@ def from_data(cls, data: dict) -> "THREDDSCatalogDataModel": @model_validator(mode="before") @classmethod - def datacube_helper(cls, data: dict) -> dict: - """Instantiate the DataCubeHelper.""" - data["datacube"] = DataCubeHelper(data["data"]) - return data - - @model_validator(mode="before") - @classmethod - def thredds_helper(cls, data: dict) -> dict: - """Instantiate the THREDDSHelper.""" - data["thredds"] = THREDDSHelper(data["data"]["access_urls"]) + def instantiate_helpers(cls, data: dict[str, any]) -> dict[str, any]: + """Automatically instantiate helper fields before model initialization. + + This method detects all fields annotated as subclasses of `Helper` + and populates them by calling their respective `from_data()` constructors. + Any extra keyword arguments are forwarded to helpers that accept them. + + Parameters + ---------- + data : dict[str, Any] + The raw input dictionary of parameters used to construct this class. + + Returns + ------- + dict[str, Any] + The modified data dictionary with instantiated helper objects injected + into their corresponding fields. + """ + # Retrieve forwarded kwargs and remove from the data object + kwargs = data["data"].pop("_extra_kwargs", {}) + + # Iterate over model fields and find helpers + for field_name, field in cls.model_fields.items(): + field_type = field.annotation + if isinstance(field_type, type) and issubclass(field_type, Helper): + # if helper not provided in constructor + if field_name not in data: + # Filter kwargs to only include those accepted by the helper's constructor. + type_signature = inspect.signature(field_type.__init__) + accepted_kwargs = {k: v for k, v in kwargs.items() if k in type_signature.parameters} + # Instantiate helper and forward accepted kwargs + data[field_name] = field_type.from_data(data, **accepted_kwargs) return data def create_uid(self) -> str: diff --git a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py index 87bb0b1..e176659 100644 --- a/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +++ b/STACpopulator/implementations/RDPS_CRIM/add_RDPS.py @@ -19,8 +19,7 @@ class RDPSpopulator(STACpopulatorBase): def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]: """Return a STAC item.""" - dm: RDPSDataModel = self.data_model.from_data(item_data) - dm.set_session(self._session) + dm = self.data_model.from_data(item_data, session=self._session) return dm.stac_item() From 29b289078ec2355251d471b5dee472cbc701b4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 30 Oct 2025 18:27:27 -0400 Subject: [PATCH 60/76] doc: changes updated --- CHANGES.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9b0e8df..e6b2768 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,12 +6,13 @@ ## [0.10.0](https://github.com/crim-ca/stac-populator/tree/0.10.0) (2025-10-23) -* Add `RDPS_CRIM` and `HRDPS_CRIM` implementation -* Add `cf` extension adding CF Parameter metadata to stac asset and items. +* Add `RDPS_CRIM` and `HRDPS_CRIM` implementations. +* Add `cf` extension adding CF Parameter metadata to (H)RDPS stac asset and items. * Add `cf` and `file` helpers. -* Fix deprecated access to `model_fields` in BaseSTAC -* Fix bug service type check in extensions' get_assets method -* Fix return type of from_data in THREDDSCatalogDataModel +* Add `providers` and `contacts` extensions metdata to (H)RDPS stac collection. +* Fix deprecated access to `model_fields` in `BaseSTAC` data model class. +* Fix bug service type check in extensions' `get_assets` methods. +* Fix return type of `from_data` in `THREDDSCatalogDataModel`. * Update RDPS and HRDPS tests. ## [0.9.0](https://github.com/crim-ca/stac-populator/tree/0.9.0) (2025-08-26) From 293a37e70afca39afb295e40008cad0e0d0fa793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Thu, 30 Oct 2025 18:31:16 -0400 Subject: [PATCH 61/76] doc: readme updated --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 46d47f5..f09bfcc 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,13 @@ Provided implementations of `STACpopulatorBase`: | Implementation | Description | |----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------| | [RDPS_CRIM][RDPS_CRIM] | Crawls a THREDDS Catalog for RDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | +| [HRDPS_CRIM][HRDPS_CRIM] | Crawls a THREDDS Catalog for HRDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [CMIP6_UofT][CMIP6_UofT] | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [DirectoryLoader][DirLoader] | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint. | -| [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | +| [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | [RDPS_CRIM]: STACpopulator/implementations/RDPS_CRIM/add_RDPS.py +[HRDPS_CRIM]: STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py [CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py [DirLoader]: STACpopulator/implementations/DirectoryLoader/crawl_directory.py [CORDEX-CMIP6_Ouranos]: STACpopulator/implementations/CORDEX-CMIP6_Ouranos/add_CORDEX-CMIP6.py From 64e39de0776b752add5e54e5c7739ca4e6e8555b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 00:26:00 -0400 Subject: [PATCH 62/76] fix: collection links updated --- .../HRDPS_CRIM/collection_config.yml | 68 +++++++++++++------ .../RDPS_CRIM/collection_config.yml | 38 +++++++++-- 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index bd08484..365d4fb 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -41,25 +41,55 @@ providers: url: https://www.ouranos.ca links: - - rel: about - title : Project homepage - target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d - media_type: text/html - extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_en - describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en - - rel: about - title : Page d'accueil du projet - target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d - media_type: text/html - extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_fr - describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_fr - - rel: license - title : Open Government Licence - Canada - target : https://open.canada.ca/en/open-government-licence-canada - media_type: text/html - + - rel: about + title : Project homepage + target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: about + title : Page d'accueil du projet + target : https://open.canada.ca/data/en/dataset/5b401fa0-6c29-57f0-b3d5-749f301d829d + media_type: text/html + extra_fields: + hreflang: fr-CA + - rel: author + title: Environment and Climate Change Canada (ECCC) + target: https://www.canada.ca/en/environment-climate-change.html + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: author + title: Environnement et Changement Climatique Canada (ECCC) + target: https://www.canada.ca/fr/environnement-changement-climatique.html + media_type: text/html + extra_fields: + hreflang: fr-CA + - rel: describedby + title: Data and Products of the High Resolution Deterministic Prediction System (HRDPS) + target: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: describedby + title: Données et Produits du Système à haute résolution de prévision déterministe (SHRPD) + target: https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_fr + media_type: text/html + extra_fields: + hreflang: fr-CA + - rel: license + title : Open Government Licence - Canada + target : https://open.canada.ca/en/open-government-licence-canada + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: license + title: Licence du gouvernement ouvert – Canada + target: https://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada + media_type: text/html + extra_fields: + hreflang: fr-CA + contacts: - organization: Computer Research Institute of Montreal (CRIM) identifier: CRIM diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index b58fa28..94b475a 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -47,19 +47,49 @@ links: target: https://open.canada.ca/data/en/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_en - describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_en + hreflang: en-CA - rel: about title: Page d'accueil du projet target: https://open.canada.ca/data/fr/dataset/a9f2828c-0d78-5eb6-a4c7-1fc1219f1e3d media_type: text/html extra_fields: - hreflang: https://eccc-msc.github.io/open-data/readme_fr - describedby: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_fr + hreflang: fr-CA + - rel: author + title: Environment and Climate Change Canada (ECCC) + target: https://www.canada.ca/en/environment-climate-change.html + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: author + title: Environnement et Changement Climatique Canada (ECCC) + target: https://www.canada.ca/fr/environnement-changement-climatique.html + media_type: text/html + extra_fields: + hreflang: fr-CA + - rel: describedby + title: Data and products of the Regional Deterministic Prediction System (RDPS) + target: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_en + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: describedby + title: Données et produits du Système régional de prévision déterministe (SRPD) + target: https://eccc-msc.github.io/open-data/msc-data/nwp_rdps/readme_rdps_fr/ + media_type: text/html + extra_fields: + hreflang: fr-CA - rel: license title: Open Government Licence - Canada target: https://open.canada.ca/en/open-government-licence-canada media_type: text/html + extra_fields: + hreflang: en-CA + - rel: license + title: Licence du gouvernement ouvert – Canada + target: https://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada + media_type: text/html + extra_fields: + hreflang: fr-CA contacts: - organization: Computer Research Institute of Montreal (CRIM) From 660265df2a73c73a08484680e3edb855ea806d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 00:53:46 -0400 Subject: [PATCH 63/76] fix: collection contacts updated --- .../HRDPS_CRIM/collection_config.yml | 37 +++++++++++++++++-- .../RDPS_CRIM/collection_config.yml | 37 +++++++++++++++++-- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index 365d4fb..e7e43a2 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -94,26 +94,55 @@ contacts: - organization: Computer Research Institute of Montreal (CRIM) identifier: CRIM position: Support + roles: + - processor emails: - value: support-geo@crim.ca - roles: ["work"] + roles: + - work logo: href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png rel: icon type: image/png title: CRIM + links: + - rel: about + title: Site web + target: https://www.crim.ca/fr/ + media_type: text/html + extra_fields: + hreflang: fr-CA - organization: Environment and Climate Change Canada (ECCC) identifier: ECCC position: Support + roles: + - licensor + - processor + - producer emails: - value: ECWeather-Meteo@ec.gc.ca - roles: ["work"] + roles: + - work phones: - value: +18199972800 - roles: ["work"] + roles: + - work addresses: - deliveryPoint: 77 Westmorland Street, suite 260 city: Fredericton administrativeArea: New Brunswick postalCode: E3B 6Z4 - country: Canada \ No newline at end of file + country: Canada + links: + - rel: about + title: Web page + target: https://www.canada.ca/en/environment-climate-change.html + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: about + title: Site web + target: https://www.canada.ca/fr/environnement-changement-climatique.html + media_type: text/html + extra_fields: + hreflang: fr-CA \ No newline at end of file diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 94b475a..8b1f2bc 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -95,26 +95,55 @@ contacts: - organization: Computer Research Institute of Montreal (CRIM) identifier: CRIM position: Support + roles: + - processor emails: - value: support-geo@crim.ca - roles: ["work"] + roles: + - work logo: href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png rel: icon type: image/png title: CRIM + links: + - rel: about + title: Site web + target: https://www.crim.ca/fr/ + media_type: text/html + extra_fields: + hreflang: fr-CA - organization: Environment and Climate Change Canada (ECCC) identifier: ECCC position: Support + roles: + - licensor + - processor + - producer emails: - value: ECWeather-Meteo@ec.gc.ca - roles: ["work"] + roles: + - work phones: - value: +18199972800 - roles: ["work"] + roles: + - work addresses: - deliveryPoint: 77 Westmorland Street, suite 260 city: Fredericton administrativeArea: New Brunswick postalCode: E3B 6Z4 - country: Canada \ No newline at end of file + country: Canada + links: + - rel: about + title: Web page + target: https://www.canada.ca/en/environment-climate-change.html + media_type: text/html + extra_fields: + hreflang: en-CA + - rel: about + title: Site web + target: https://www.canada.ca/fr/environnement-changement-climatique.html + media_type: text/html + extra_fields: + hreflang: fr-CA \ No newline at end of file From afbe940ea63c1d2091eb5bbe819805da4cec0024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 01:37:56 -0400 Subject: [PATCH 64/76] fix: rdps tests updated to check added fields --- tests/test_rdps.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_rdps.py b/tests/test_rdps.py index a1596a8..7d99a90 100644 --- a/tests/test_rdps.py +++ b/tests/test_rdps.py @@ -10,20 +10,54 @@ def test_rdps(): attrs = json.load(open("tests/data/rdps.json")) model = RDPSDataModel.from_data(attrs) item = model.stac_item() + assets = item["assets"] + cf_parameters = item["properties"].get("cf:parameter", []) + parameter_names = [param["name"] for param in cf_parameters] + assert set(model._helpers) == {"thredds", "datacube", "cf", "file"} assert item["id"] == "birdhouse__testdata__HRDPS__RDPS_sample__2024010100_000.nc" + # DataCubeExtension + assert "cube:variables" in item["properties"] assert "TD" in item["properties"]["cube:variables"] + # CFExtension + assert any("cf:parameter" in asset for asset in assets.values()) + assert "cf:parameter" in item["properties"] + assert set(["time", "latitude", "longitude"]).issubset(set(parameter_names)) + # FileExtension + assert any("file:size" in asset for asset in assets.values()) -def test_hrdps(): +def test_hrdps_sfc(): attrs = json.load(open("tests/data/hrdps_sfc.json")) model = HRDPSDataModel.from_data(attrs) item = model.stac_item() + assets = item["assets"] + cf_parameters = item["properties"].get("cf:parameter", []) + parameter_names = [param["name"] for param in cf_parameters] + # DataCubeExtension assert "HRDPS_P_PR_SFC" in item["properties"]["cube:variables"] assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} + # CFExtension + assert any("cf:parameter" in asset for asset in assets.values()) + assert "cf:parameter" in item["properties"] + assert set(["time", "latitude", "longitude"]).issubset(set(parameter_names)) + # FileExtension + assert any("file:size" in asset for asset in assets.values()) + +def test_hrdps_p_tt(): attrs = json.load(open("tests/data/hrdps_p_tt.json")) model = HRDPSDataModel.from_data(attrs) item = model.stac_item() + assets = item["assets"] + cf_parameters = item["properties"].get("cf:parameter", []) + parameter_names = [param["name"] for param in cf_parameters] + # DataCubeExtension assert "HRDPS_P_TT_10000" in item["properties"]["cube:variables"] assert item["properties"]["cube:dimensions"].keys() == {"time", "rlat", "rlon"} + # CFExtension + assert any("cf:parameter" in asset for asset in assets.values()) + assert "cf:parameter" in item["properties"] + assert set(["time", "latitude", "longitude"]).issubset(set(parameter_names)) + # FileExtension + assert any("file:size" in asset for asset in assets.values()) From 64c33b65e683ea2994ca719ee17642c49d894ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 14:29:06 -0400 Subject: [PATCH 65/76] fix: add crim as indexer in providers --- .../implementations/HRDPS_CRIM/collection_config.yml | 7 ++++++- .../implementations/RDPS_CRIM/collection_config.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index e7e43a2..e483496 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -39,6 +39,11 @@ providers: - processor - host url: https://www.ouranos.ca + - name: CRIM + description: Computer Research Institute of Montreal + roles: + - indexer + url: https://crim.ca links: - rel: about @@ -95,7 +100,7 @@ contacts: identifier: CRIM position: Support roles: - - processor + - indexer emails: - value: support-geo@crim.ca roles: diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index 8b1f2bc..f82eca8 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -40,6 +40,11 @@ providers: - processor - host url: https://www.ouranos.ca + - name: CRIM + description: Computer Research Institute of Montreal + roles: + - indexer + url: https://crim.ca links: - rel: about @@ -96,7 +101,7 @@ contacts: identifier: CRIM position: Support roles: - - processor + - indexer emails: - value: support-geo@crim.ca roles: From b73c216ecd0993b7c44648b416485d98dfc88e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 14:39:44 -0400 Subject: [PATCH 66/76] fix: add crim as indexer in contacts only --- .../implementations/HRDPS_CRIM/collection_config.yml | 5 ----- .../implementations/RDPS_CRIM/collection_config.yml | 5 ----- 2 files changed, 10 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index e483496..184cbb6 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -39,11 +39,6 @@ providers: - processor - host url: https://www.ouranos.ca - - name: CRIM - description: Computer Research Institute of Montreal - roles: - - indexer - url: https://crim.ca links: - rel: about diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index f82eca8..f26b628 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -40,11 +40,6 @@ providers: - processor - host url: https://www.ouranos.ca - - name: CRIM - description: Computer Research Institute of Montreal - roles: - - indexer - url: https://crim.ca links: - rel: about From 5b668e9490a0944ba988545ac3655e34710adebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Fri, 31 Oct 2025 14:43:39 -0400 Subject: [PATCH 67/76] fix: license links updated --- .../implementations/HRDPS_CRIM/collection_config.yml | 8 ++++---- .../implementations/RDPS_CRIM/collection_config.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index 184cbb6..4d99cbf 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -78,14 +78,14 @@ links: extra_fields: hreflang: fr-CA - rel: license - title : Open Government Licence - Canada - target : https://open.canada.ca/en/open-government-licence-canada + title: Environment and Climate Change Canada Data Servers End-use License + target: https://eccc-msc.github.io/open-data/licence/readme_en/ media_type: text/html extra_fields: hreflang: en-CA - rel: license - title: Licence du gouvernement ouvert – Canada - target: https://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada + title: Licence d’utilisation finale pour les serveurs de données d’Environnement et Changement climatique Canada + target: https://eccc-msc.github.io/open-data/licence/readme_fr/ media_type: text/html extra_fields: hreflang: fr-CA diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index f26b628..be82c7b 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -79,14 +79,14 @@ links: extra_fields: hreflang: fr-CA - rel: license - title: Open Government Licence - Canada - target: https://open.canada.ca/en/open-government-licence-canada + title: Environment and Climate Change Canada Data Servers End-use License + target: https://eccc-msc.github.io/open-data/licence/readme_en/ media_type: text/html extra_fields: hreflang: en-CA - rel: license - title: Licence du gouvernement ouvert – Canada - target: https://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada + title: Licence d’utilisation finale pour les serveurs de données d’Environnement et Changement climatique Canada + target: https://eccc-msc.github.io/open-data/licence/readme_fr/ media_type: text/html extra_fields: hreflang: fr-CA From 84ab682fd7dfd80b822fb8ccd45d282b3e48dc8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 11 Nov 2025 17:11:33 -0500 Subject: [PATCH 68/76] fix: dimensions and variables updated --- STACpopulator/extensions/datacube.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/STACpopulator/extensions/datacube.py b/STACpopulator/extensions/datacube.py index 6b8e186..c4e8e84 100644 --- a/STACpopulator/extensions/datacube.py +++ b/STACpopulator/extensions/datacube.py @@ -174,15 +174,18 @@ def dimensions(self) -> dict[str, Dimension]: extent = bbox[1], bbox[3] elif key in ["T", "time"]: extent = self.temporal_extent() + elif key == "Z": + extent is None else: extent = [None, None] properties = dict( type=type_.value, - extent=extent, description=v.get("description", v.get("long_name", criteria["standard_name"][0])) or "", ) + if extent is not None: # omit extent for Z axis + properties["extent"] = extent if type_ == DimensionType.SPATIAL: properties["axis"] = axis @@ -222,9 +225,11 @@ def variables(self) -> dict[str, Variable]: else: dtype = VariableType.DATA.value + dimensions = meta.get("shape", []) + variables[name] = Variable( properties=dict( - dimensions=meta["shape"], + dimensions=[] if dimensions == [""] else dimensions, type=dtype, description=attrs.get("description", attrs.get("long_name", "")), unit=attrs.get("units", ""), From 1a0b8aab29358f09d277776a39d11072feaac017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Tue, 11 Nov 2025 17:18:42 -0500 Subject: [PATCH 69/76] fix: dimensions and variables updated --- STACpopulator/extensions/datacube.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/STACpopulator/extensions/datacube.py b/STACpopulator/extensions/datacube.py index c4e8e84..19cad36 100644 --- a/STACpopulator/extensions/datacube.py +++ b/STACpopulator/extensions/datacube.py @@ -174,18 +174,15 @@ def dimensions(self) -> dict[str, Dimension]: extent = bbox[1], bbox[3] elif key in ["T", "time"]: extent = self.temporal_extent() - elif key == "Z": - extent is None else: extent = [None, None] properties = dict( type=type_.value, + extent=extent, description=v.get("description", v.get("long_name", criteria["standard_name"][0])) or "", ) - if extent is not None: # omit extent for Z axis - properties["extent"] = extent if type_ == DimensionType.SPATIAL: properties["axis"] = axis From 227a7d54982986a5918a54630cc6262ddfc97f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Wed, 12 Nov 2025 10:45:28 -0500 Subject: [PATCH 70/76] fix: precommit run --- CHANGES.md | 4 ---- README.md | 2 +- .../implementations/HRDPS_CRIM/collection_config.yml | 4 ++-- STACpopulator/implementations/RDPS_CRIM/collection_config.yml | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5a77ca8..e38e1d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,10 +17,6 @@ ## [0.9.0](https://github.com/crim-ca/stac-populator/tree/0.9.0) (2025-08-26) - - -## [0.9.0](https://github.com/crim-ca/stac-populator/tree/0.9.0) (2025-08-26) - * Include `PYESSV_ARCHIVE_HOME` environment variable in Dockerfile. * Remove redundant CLI arguments. * Fix bug in `THREDDSLoader` iterator introduced in 0.8.0. Simple iteration over `catalog_refs` returns the diff --git a/README.md b/README.md index 5dc1e51..b416067 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Provided implementations of `STACpopulatorBase`: | [HRDPS_CRIM][HRDPS_CRIM] | Crawls a THREDDS Catalog for HRDPS NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [CMIP6_UofT][CMIP6_UofT] | Crawls a THREDDS Catalog for CMIP6 NCML-annotated NetCDF references to publish corresponding STAC Collection and Items. | | [DirectoryLoader][DirLoader] | Crawls a subdirectory hierarchy of pre-generated STAC Collections and Items to publish to a STAC API endpoint. | -| [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | +| [CORDEX-CMIP6_Ouranos][CORDEX-CMIP6_Ouranos] | Crawls a THREDDS Catalog for CORDEX-CMIP6 NetCDF references to publish corresponding STAC Collection and Items. | [RDPS_CRIM]: STACpopulator/implementations/RDPS_CRIM/add_RDPS.py [HRDPS_CRIM]: STACpopulator/implementations/HRDPS_CRIM/add_HRDPS.py diff --git a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml index 4d99cbf..62216d6 100644 --- a/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/HRDPS_CRIM/collection_config.yml @@ -89,7 +89,7 @@ links: media_type: text/html extra_fields: hreflang: fr-CA - + contacts: - organization: Computer Research Institute of Montreal (CRIM) identifier: CRIM @@ -145,4 +145,4 @@ contacts: target: https://www.canada.ca/fr/environnement-changement-climatique.html media_type: text/html extra_fields: - hreflang: fr-CA \ No newline at end of file + hreflang: fr-CA diff --git a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml index be82c7b..85180be 100644 --- a/STACpopulator/implementations/RDPS_CRIM/collection_config.yml +++ b/STACpopulator/implementations/RDPS_CRIM/collection_config.yml @@ -99,7 +99,7 @@ contacts: - indexer emails: - value: support-geo@crim.ca - roles: + roles: - work logo: href: https://raw.githubusercontent.com/henriaidasso/image-store/refs/heads/main/img/crim.png @@ -146,4 +146,4 @@ contacts: target: https://www.canada.ca/fr/environnement-changement-climatique.html media_type: text/html extra_fields: - hreflang: fr-CA \ No newline at end of file + hreflang: fr-CA From 0dadae765e7b3aa7bec60824180667788c7b8503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 11:32:07 -0500 Subject: [PATCH 71/76] fix: updated get assets methods for uniformity --- STACpopulator/extensions/base.py | 2 +- STACpopulator/extensions/cf.py | 4 ++-- STACpopulator/extensions/cmip6.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/STACpopulator/extensions/base.py b/STACpopulator/extensions/base.py index 6653ff4..cce94b5 100644 --- a/STACpopulator/extensions/base.py +++ b/STACpopulator/extensions/base.py @@ -340,7 +340,7 @@ def get_assets( key: asset for key, asset in self.item.get_assets().items() if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + or any(ServiceType.from_value(field, default=False) for field in asset.extra_fields) } def __repr__(self) -> str: diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 019d19b..056e78d 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -180,8 +180,8 @@ def get_assets( return { key: asset for key, asset in self.item.get_assets().items() - if (service_type in ServiceType and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) is ServiceType for field in asset.extra_fields) + if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) + or any(ServiceType.from_value(field, default=False) for field in asset.extra_fields) } def __repr__(self) -> str: diff --git a/STACpopulator/extensions/cmip6.py b/STACpopulator/extensions/cmip6.py index af9e736..e4fa172 100644 --- a/STACpopulator/extensions/cmip6.py +++ b/STACpopulator/extensions/cmip6.py @@ -309,7 +309,7 @@ def get_assets( key: asset for key, asset in self.item.get_assets().items() if (isinstance(service_type, ServiceType) and service_type.value in asset.extra_fields) - or any(ServiceType.from_value(field, default=None) for field in asset.extra_fields) + or any(ServiceType.from_value(field, default=False) for field in asset.extra_fields) } def __repr__(self) -> str: From 49f0d4a8510330a559583e345273d632a4084d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 11:33:20 -0500 Subject: [PATCH 72/76] fix: cf iterate on values updated --- STACpopulator/extensions/cf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 056e78d..1adbbc2 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -54,7 +54,7 @@ def parameters(self) -> List[CFParameter]: """Extracts cf:parameter-like information from item_data.""" parameters = [] - for _, var in self.variables.items(): + for var in self.variables.values(): attrs = var.get("attributes", {}) name = attrs.get("standard_name") # Get the required standard name if not name: From 185ddac65e7f822bb8155247e1ee246439443b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 11:42:08 -0500 Subject: [PATCH 73/76] fix: unit str to match validation schema --- STACpopulator/extensions/cf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/STACpopulator/extensions/cf.py b/STACpopulator/extensions/cf.py index 1adbbc2..ff15cf2 100644 --- a/STACpopulator/extensions/cf.py +++ b/STACpopulator/extensions/cf.py @@ -36,7 +36,7 @@ class CFParameter(BaseModel): """CFParameter.""" name: str - unit: Optional[str] + unit: str def __repr__(self) -> str: """Return string repr.""" @@ -59,7 +59,7 @@ def parameters(self) -> List[CFParameter]: name = attrs.get("standard_name") # Get the required standard name if not name: continue # Skip if no valid name - unit = attrs.get("units") or "" + unit = attrs.get("units", "") parameters.append(CFParameter(name=name, unit=unit)) return parameters From 8cdbc593571cfa69c2531a33b43b45036341d1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 11:56:48 -0500 Subject: [PATCH 74/76] fix: return none for file content-length issues --- STACpopulator/extensions/file.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/STACpopulator/extensions/file.py b/STACpopulator/extensions/file.py index 7d9fb46..a026b01 100644 --- a/STACpopulator/extensions/file.py +++ b/STACpopulator/extensions/file.py @@ -75,10 +75,29 @@ def apply(self, item: pystac.Item, add_if_missing: bool = True) -> T: @functools.cached_property def size(self) -> Optional[int]: - """Return file size in bytes, None if asset_key not in access URLs dictionnary.""" + """Get file size in bytes. + + Returns + ------- + (int): File size in bytes or None if: + - the `asset_key` is not in `access_urls` dict + - the server does include a Content-Length header + - the Content-Length value is non-numeric. + """ if self.asset_key not in self.access_urls: logger.warning("Asset key %s is not present in access URLs.", self.asset_key) return None res = self.session.head(self.access_urls[self.asset_key]) res.raise_for_status() - return int(res.headers.get("Content-Length", None)) + + content_length = res.headers.get("Content-Length") + try: + return int(content_length) + except (TypeError, ValueError) as e: + logger.exception( + "Failed to parse Content-Length header (%r) for asset key %s", + content_length, + self.asset_key, + exc_info=e, + ) + return None From 3f033c2e15eeb9792984333bec0c86a9a4e4b5d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 12:00:37 -0500 Subject: [PATCH 75/76] fix: remove unused future imports and adds in thredds --- STACpopulator/extensions/hrdps.py | 2 -- STACpopulator/extensions/rdps.py | 2 -- STACpopulator/extensions/thredds.py | 8 +++++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/STACpopulator/extensions/hrdps.py b/STACpopulator/extensions/hrdps.py index 3204dfb..3d3ba2c 100644 --- a/STACpopulator/extensions/hrdps.py +++ b/STACpopulator/extensions/hrdps.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from STACpopulator.extensions.rdps import RDPSDataModel diff --git a/STACpopulator/extensions/rdps.py b/STACpopulator/extensions/rdps.py index 3b480af..64cc09b 100644 --- a/STACpopulator/extensions/rdps.py +++ b/STACpopulator/extensions/rdps.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from STACpopulator.extensions.cf import CFHelper from STACpopulator.extensions.file import FileHelper from STACpopulator.extensions.thredds import THREDDSCatalogDataModel diff --git a/STACpopulator/extensions/thredds.py b/STACpopulator/extensions/thredds.py index b0e835b..ae49dca 100644 --- a/STACpopulator/extensions/thredds.py +++ b/STACpopulator/extensions/thredds.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import inspect from typing import Generic, TypeVar, Union, cast @@ -69,7 +71,7 @@ def get_schema_uri(cls) -> str: return "" @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> "THREDDSExtension[T]": + def ext(cls, obj: T, add_if_missing: bool = False) -> THREDDSExtension[T]: """Extend the given STAC Object with properties from the :stac-ext:`THREDDS Extension `. This extension can be applied to instances of :class:`~pystac.Item` or @@ -141,7 +143,7 @@ def from_data( cls, data: dict[str, any], **kwargs, - ) -> "THREDDSHelper": + ) -> THREDDSHelper: """Create a THREDDSHelper instance from raw data.""" return cls(access_urls=data["data"]["access_urls"]) @@ -190,7 +192,7 @@ class THREDDSCatalogDataModel(BaseSTAC): model_config = ConfigDict(populate_by_name=True, extra="ignore", arbitrary_types_allowed=True) @classmethod - def from_data(cls, data: dict, **kwargs) -> "THREDDSCatalogDataModel": + def from_data(cls, data: dict, **kwargs) -> THREDDSCatalogDataModel: """ Instantiate class from data provided by THREDDS Loader. From 07696c4956fe07d3ad3cf8a2ce0f8f0b56564e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henri=20A=C3=AFdasso?= Date: Mon, 17 Nov 2025 14:43:16 -0500 Subject: [PATCH 76/76] fix: changes version fix --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 683ac24..db5bb28 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,11 +13,11 @@ * Fix return type of `from_data` in `THREDDSCatalogDataModel`. * Update RDPS and HRDPS tests. -## [1.11.0](https://github.com/crim-ca/stac-populator/tree/1.11.0) (2025-11-17) +## [0.11.0](https://github.com/crim-ca/stac-populator/tree/0.11.0) (2025-11-17) * Add option to automatically update collection extents and summaries based on ingested items. -## [1.10.0](https://github.com/crim-ca/stac-populator/tree/1.10.0) (2025-11-11) +## [0.10.0](https://github.com/crim-ca/stac-populator/tree/0.10.0) (2025-11-11) * Add `pre-commit` linting rules (code format + STAC field sorting in JSON). * Add initial RDPS and HRDPS examples with minimal metadata from Ouranos THREDDS samples.