Skip to content

Update CMIP7 unit tests #884

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mauzey1 merged 10 commits into from
Nov 20, 2025
Merged

Update CMIP7 unit tests #884

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
062e591
Add cmip7-cmor-tables as submodule
mauzey1 Nov 5, 2025
f29da45
data_specs_version and mip_era can be added to a file if they are in …
mauzey1 Nov 5, 2025
6acd24c
Update test_cmor_CMIP7.py to use cmip7-cmor-tables
mauzey1 Nov 5, 2025
ced3418
Use assertIn to check if attribute is present in file
mauzey1 Nov 6, 2025
e075825
Use full checkout for CircleCI
mauzey1 Nov 7, 2025
5ec042e
Apply full checkout to all jobs in CircleCI
mauzey1 Nov 7, 2025
803838c
Merge branch 'main' into 879-update-cmip7-tests-to-use-cmip7-cmor-tables
mauzey1 Nov 10, 2025
66b8edb
Update CMIP7 tests to use cmip7-cmor-tables
mauzey1 Nov 11, 2025
49f3934
Use the CMIP7 CV file in TestTables instead of the one in cmip7-cmor-…
mauzey1 Nov 12, 2025
0625f5a
Add activity_id section to CMIP7 CV in TestTables
mauzey1 Nov 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
version: 2.1

checkout:
method: full
post:
- ./ci-support/checkout_merge_commit.sh

Expand Down Expand Up @@ -145,7 +146,8 @@ jobs:
PYTHON_VERSION: << parameters.python_version >>
CHANNELS: "-c conda-forge"
steps:
- checkout
- checkout:
method: full
- attach_workspace:
at: .
- run: *setup_env
Expand All @@ -168,7 +170,8 @@ jobs:
USER: "pcmdi"
LABEL: "nightly"
steps:
- checkout
- checkout:
method: full
- attach_workspace:
at: .
- run: *setup_env
Expand All @@ -184,7 +187,8 @@ jobs:
USER: "pcmdi"
LABEL: "testing"
steps:
- checkout
- checkout:
method: full
- attach_workspace:
at: .
- run: *setup_env
Expand Down
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@
[submodule "mip-cmor-tables"]
path = mip-cmor-tables
url = https://github.com/PCMDI/mip-cmor-tables.git
[submodule "cmip7-cmor-tables"]
path = cmip7-cmor-tables
url = https://github.com/WCRP-CMIP/cmip7-cmor-tables
27 changes: 14 additions & 13 deletions Src/cmor.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2931,28 +2931,29 @@ int cmor_setDefaultGblAttr(int ref_table_id)

/* -------------------------------------------------------------------- */
/* Set further_info_url template if required and not already set. */
/* Set data_specs_version and mip_era if they are in the CV. */
/* -------------------------------------------------------------------- */
for (k = 0; k < required_attrs->anElements; k++) {
if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_FURTHERINFOURL) == 0
&& cmor_current_dataset.furtherinfourl[0] == '\0')
{
ierr |= cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_FURTHERINFOURLTMPL, CMOR_DEFAULT_FURTHERURL_TEMPLATE, 0);
} else if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_DATASPECSVERSION) == 0) {
CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_DATASPECSVERSION);
if (CV_value != NULL && CV_value->szValue[0] != '\0') {
cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_DATASPECSVERSION,
CV_value->szValue, 0);
}
} else if(strcmp(required_attrs->aszValue[k], GLOBAL_ATT_MIP_ERA) == 0) {
CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_MIP_ERA);
if (CV_value != NULL && CV_value->szValue[0] != '\0') {
cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_MIP_ERA,
CV_value->szValue, 0);
}
}
}

/* -------------------------------------------------------------------- */
/* Set data_specs_version and mip_era if they are in the CV. */
/* -------------------------------------------------------------------- */
CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_DATASPECSVERSION);
if (CV_value != NULL && CV_value->szValue[0] != '\0') {
cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_DATASPECSVERSION,
CV_value->szValue, 0);
}
CV_value = cmor_CV_rootsearch(cmor_tables[ref_table_id].CV, CV_KEY_MIP_ERA);
if (CV_value != NULL && CV_value->szValue[0] != '\0') {
cmor_set_cur_dataset_attribute_internal(GLOBAL_ATT_MIP_ERA,
CV_value->szValue, 0);
}

/* -------------------------------------------------------------------- */
/* Set the path and file name template if they are in the CV and */
/* the user didn't define them. */
Expand Down
44 changes: 24 additions & 20 deletions Test/test_cmor_CMIP7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,22 @@

from netCDF4 import Dataset

DATASET_INFO = {
"_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json",
"_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json",
CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables"
CV_PATH = "cmip7-cmor-tables/test/CMIP7-CV_for-cmor.json"
Comment thread
mauzey1 marked this conversation as resolved.
Resolved

USER_INPUT = {
"_AXIS_ENTRY_FILE": "CMIP7_coordinate.json",
"_FORMULA_VAR_FILE": "CMIP7_formula_terms.json",
"_cmip7_option": 1,
"_controlled_vocabulary_file": "TestTables/CMIP7_CV.json",
"_controlled_vocabulary_file": CV_PATH,
"activity_id": "CMIP",
"branch_method": "standard",
"branch_time_in_child": 30.0,
"branch_time_in_parent": 10800.0,
"calendar": "360_day",
"cv_version": "6.2.19.0",
"experiment": "1 percent per year increase in CO2",
"experiment_id": "1pctCO2",
"experiment": "Simulation of the pre-industrial climate",
"experiment_id": "piControl",
"forcing_index": "f30",
"grid": "N96",
"grid_label": "gn",
Expand All @@ -41,6 +44,7 @@
"frequency": "mon",
"region": "glb",
"archive_id": "WCRP",
"drs_specs": "MIP-DRS7",
"output_path_template": "<activity_id><source_id><experiment_id><member_id><variable_id><branding_suffix><grid_label>",
"output_file_template": "<variable_id><branding_suffix><frequency><region><grid_label><source_id><experiment_id><variant_id>[<time_range>].nc",
}
Expand All @@ -52,11 +56,11 @@ def setUp(self):
Write out a simple file using CMOR
"""
# Set up CMOR
cmor.setup(inpath="TestTables", netcdf_file_action=cmor.CMOR_REPLACE)
cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE)

# Define dataset using DATASET_INFO
# Define dataset using USER_INPUT
with open("Test/input_cmip7.json", "w") as input_file_handle:
json.dump(DATASET_INFO, input_file_handle, sort_keys=True, indent=4)
json.dump(USER_INPUT, input_file_handle, sort_keys=True, indent=4)

# read dataset info
error_flag = cmor.dataset_json("Test/input_cmip7.json")
Expand All @@ -77,7 +81,7 @@ def test_cmip7(self):
])
time = numpy.array([15.5, 45])
time_bnds = numpy.array([0, 31, 60])
cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")
cmorlat = cmor.axis("latitude",
coord_vals=lat,
cell_bounds=lat_bnds,
Expand Down Expand Up @@ -108,14 +112,14 @@ def test_cmip7(self):
'frequency': 'mon',
'archive_id': 'WCRP',
'mip_era': 'CMIP7',
'data_specs_version': 'CMIP-7.0.0.0',
'data_specs_version': 'CMIP-7.0.0.0-alpha',
'host_collection': 'CMIP7',
}

for attr, val in test_attrs.items():
self.assertTrue(attr in attrs)
self.assertIn(attr, attrs)
self.assertEqual(val, ds.getncattr(attr))
institution_id = DATASET_INFO["institution_id"]
institution_id = USER_INPUT["institution_id"]
license_id = "CC BY 4.0"
license_type = "Creative Commons Attribution 4.0 International"
license_url = "https://creativecommons.org/licenses/by/4.0/"
Expand Down Expand Up @@ -153,7 +157,7 @@ def test_secondary_modeling_realm(self):
])
time = numpy.array([15.5, 45])
time_bnds = numpy.array([0, 31, 60])
cmor.load_table("CMIP7_atmos2d.json")
cmor.load_table("CMIP7_atmos.json")
cmorlat = cmor.axis("latitude",
coord_vals=lat,
cell_bounds=lat_bnds,
Expand All @@ -167,30 +171,30 @@ def test_secondary_modeling_realm(self):
cell_bounds=time_bnds,
units="days since 2018")
axes = [cmortime, cmorlat, cmorlon]
cmorpr = cmor.variable("pr_tavg-u-hxy-u", "kg m-2 s-1", axes)
cmorpr = cmor.variable("prra_tavg-u-hxy-is", "kg m-2 s-1", axes)
self.assertEqual(cmor.write(cmorpr, pr), 0)
filename = cmor.close(cmorpr, file_name=True)
self.assertEqual(cmor.close(), 0)

ds = Dataset(filename)
attrs = ds.ncattrs()
test_attrs = {
'branding_suffix': 'tavg-u-hxy-u',
'branding_suffix': 'tavg-u-hxy-is',
'temporal_label': 'tavg',
'vertical_label': 'u',
'horizontal_label': 'hxy',
'area_label': 'u',
'area_label': 'is',
'region': 'glb',
'frequency': 'mon',
'archive_id': 'WCRP',
'mip_era': 'CMIP7',
'data_specs_version': 'CMIP-7.0.0.0',
'data_specs_version': 'CMIP-7.0.0.0-alpha',
'host_collection': 'CMIP7',
'realm': 'atmos ocean',
'realm': 'atmos land landIce',
}

for attr, val in test_attrs.items():
self.assertTrue(attr in attrs)
self.assertIn(attr, attrs)
self.assertEqual(val, ds.getncattr(attr))

ds.close()
Expand Down
50 changes: 27 additions & 23 deletions Test/test_cmor_check_cv_structure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,26 @@
import unittest
from base_CMIP6_CV import BaseCVsTest

CMIP7_TABLES_PATH = "cmip7-cmor-tables/tables"
CV_PATH = "cmip7-cmor-tables/test/CMIP7-CV_for-cmor.json"

USER_INPUT = {
"_AXIS_ENTRY_FILE": "Tables/CMIP6_coordinate.json",
"_FORMULA_VAR_FILE": "Tables/CMIP6_formula_terms.json",
"_AXIS_ENTRY_FILE": "CMIP7_coordinate.json",
"_FORMULA_VAR_FILE": "CMIP7_formula_terms.json",
"_cmip7_option": 1,
"_controlled_vocabulary_file": "",
"_controlled_vocabulary_file": None,
"activity_id": "CMIP",
"branch_method": "standard",
"branch_time_in_child": 30.0,
"branch_time_in_parent": 10800.0,
"calendar": "360_day",
"cv_version": "6.2.19.0",
"experiment": "1 percent per year increase in CO2",
"experiment_id": "1pctCO2",
"forcing_index": "3",
"experiment": "Simulation of the pre-industrial climate",
"experiment_id": "piControl",
"forcing_index": "f30",
"grid": "N96",
"grid_label": "gn",
"initialization_index": "1",
"initialization_index": "i000001d",
"institution_id": "PCMDI",
"license_id": "CC BY 4.0",
"nominal_resolution": "250 km",
Expand All @@ -31,15 +34,16 @@
"parent_source_id": "PCMDI-test-1-0",
"parent_experiment_id": "piControl",
"parent_variant_label": "r1i1p1f3",
"physics_index": "1",
"realization_index": "9",
"physics_index": "p1",
"realization_index": "r009",
"source_id": "PCMDI-test-1-0",
"source_type": "AOGCM CHEM BGC",
"tracking_prefix": "hdl:21.14100",
"host_collection": "CMIP7",
"frequency": "mon",
"region": "glb",
"archive_id": "WCRP"
"archive_id": "WCRP",
"drs_specs": "MIP-DRS7"
}


Expand All @@ -51,11 +55,11 @@ def test_check_for_cv_json_key(self):
cv_path = f"TestTables/CMIP7_CV_{test_name}.json"
input_path = f"Test/input_{test_name}.json"

cmor.setup(inpath="TestTables",
cmor.setup(inpath=CMIP7_TABLES_PATH,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)

with open("TestTables/CMIP7_CV.json", "r") as cv_infile:
with open(CV_PATH, "r") as cv_infile:
cv = json.load(cv_infile)
no_cv_key = cv["CV"]
with open(cv_path, "w") as cv_outfile:
Expand All @@ -71,7 +75,7 @@ def test_check_for_cv_json_key(self):
raise RuntimeError("CMOR dataset_json call failed")

with self.assertRaises(cmor.CMORError):
cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")

self.assertCV("CV section was not found in table: ")

Expand All @@ -84,11 +88,11 @@ def test_check_cv_attribute_values(self):
cv_path = f"TestTables/CMIP7_CV_{test_name}.json"
input_path = f"Test/input_{test_name}.json"

cmor.setup(inpath="TestTables",
cmor.setup(inpath=CMIP7_TABLES_PATH,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)

with open("TestTables/CMIP7_CV.json", "r") as cv_infile:
with open(CV_PATH, "r") as cv_infile:
cv = json.load(cv_infile)
cv["CV"]["branding_suffix"] = ["template"]
cv["CV"]["mip_era"] = {"mip_era": "CMIP7"}
Expand All @@ -107,7 +111,7 @@ def test_check_cv_attribute_values(self):
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")

self.assertCV(
"must be a string",
Expand Down Expand Up @@ -143,11 +147,11 @@ def test_check_cv_single_value_pairs(self):
cv_path = f"TestTables/CMIP7_CV_{test_name}.json"
input_path = f"Test/input_{test_name}.json"

cmor.setup(inpath="TestTables",
cmor.setup(inpath=CMIP7_TABLES_PATH,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)

with open("TestTables/CMIP7_CV.json", "r") as cv_infile:
with open(CV_PATH, "r") as cv_infile:
cv = json.load(cv_infile)
cv["CV"]["institution_id"]["PCMDI"] = {"LLNL": "Livermore, CA"}
with open(cv_path, "w") as cv_outfile:
Expand All @@ -162,7 +166,7 @@ def test_check_cv_single_value_pairs(self):
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")

self.assertCV(
"in attribute \"institution_id\" cannot be an object",
Expand All @@ -178,7 +182,7 @@ def test_check_cv_array_values(self):
cv_path = f"TestTables/CMIP7_CV_{test_name}.json"
input_path = f"Test/input_{test_name}.json"

cmor.setup(inpath="TestTables",
cmor.setup(inpath=CMIP7_TABLES_PATH,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)

Expand All @@ -201,7 +205,7 @@ def test_check_cv_array_values(self):
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")

self.assertCV(
"has elements in its array that are not strings",
Expand All @@ -217,7 +221,7 @@ def test_check_cv_nested_objects(self):
cv_path = f"TestTables/CMIP7_CV_{test_name}.json"
input_path = f"Test/input_{test_name}.json"

cmor.setup(inpath="TestTables",
cmor.setup(inpath=CMIP7_TABLES_PATH,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=self.tmpfile)

Expand All @@ -244,7 +248,7 @@ def test_check_cv_nested_objects(self):
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

cmor.load_table("CMIP7_ocean2d.json")
cmor.load_table("CMIP7_ocean.json")

self.assertCV(
"has elements in its array that are not strings",
Expand Down
Loading
Loading