From a70259d9125ba0a9808da715ae3617d8f4b89f48 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:52:40 +0000 Subject: [PATCH 01/12] Implement inferred simboard www defaults --- tests/test_sections.py | 1 + tests/test_zppy_main.py | 93 +++++++++++++++++++++++++++++++++++++++ zppy/__main__.py | 14 ++++++ zppy/defaults/default.ini | 7 ++- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 tests/test_zppy_main.py diff --git a/tests/test_sections.py b/tests/test_sections.py index 8b34fd42..49a73b57 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -92,6 +92,7 @@ def test_sections(): "plugins": [], "reservation": "", "qos": "regular", + "simboard_type": "prod", "templateDir": "zppy/templates", "ts_atm_grid": "180x360_aave", "ts_atm_subsection": "", diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py new file mode 100644 index 00000000..91b6deda --- /dev/null +++ b/tests/test_zppy_main.py @@ -0,0 +1,93 @@ +import configparser +from pathlib import Path +from typing import Any, Dict +from unittest.mock import MagicMock + +import pytest +from configobj import ConfigObj +from validate import Validator + +from zppy.__main__ import _determine_parameters + + +def _fake_machine_info() -> MagicMock: + config = configparser.ConfigParser() + config["e3sm_unified"] = {"base_path": "/unified"} + config["diagnostics"] = {"base_path": "/diagnostics"} + config["web_portal"] = { + "base_path": "/global/cfs/cdirs/e3sm/www", + "base_url": "https://portal.nersc.gov/cfs/e3sm", + } + machine_info = MagicMock() + machine_info.machine = "pm-cpu" + machine_info.config = config + machine_info.get_account_defaults.return_value = ("e3sm", "regular", "cpu", None) + return machine_info + + +def _base_config() -> Dict[str, Dict[str, Any]]: + return { + "default": { + "machine": "", + "account": "", + "partition": "", + "constraint": "", + "environment_commands": "", + "infer_path_parameters": True, + "simboard_type": "prod", + "www": "", + } + } + + +@pytest.mark.parametrize( + ("simboard_type", "expected_www"), + [ + ("prod", "/global/cfs/cdirs/e3sm/www/simboard/prod/"), + ("dev", "/global/cfs/cdirs/e3sm/www/simboard/dev/"), + ], +) +def test_determine_parameters_infers_simboard_www( + simboard_type: str, expected_www: str +) -> None: + config = _base_config() + config["default"]["simboard_type"] = simboard_type + + updated = _determine_parameters(_fake_machine_info(), config) + + assert updated["default"]["www"] == expected_www + + +def test_determine_parameters_requires_www_without_path_inference() -> None: + config = _base_config() + config["default"]["infer_path_parameters"] = False + + with pytest.raises( + ValueError, match="www must be provided when infer_path_parameters is False." + ): + _determine_parameters(_fake_machine_info(), config) + + +def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: + config_path = tmp_path / "bad_simboard.cfg" + config_path.write_text( + "\n".join( + [ + "[default]", + "case = case_name", + "input = /input", + "output = /output", + "www = /www", + "simboard_type = invalid", + ] + ) + ) + config = ConfigObj( + str(config_path), + configspec="/home/runner/work/zppy/zppy/zppy/defaults/default.ini", + ) + + result = config.validate(Validator()) + + assert result is not True + assert result["default"]["simboard_type"] is False diff --git a/zppy/__main__.py b/zppy/__main__.py index 61d9ede1..3d89da65 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -259,9 +259,23 @@ def _determine_parameters(machine_info: MachineInfo, config: ConfigObj) -> Confi config["default"][ "environment_commands" ] = f"source {unified_base}/load_latest_e3sm_unified_{machine}.sh" + _set_default_www(machine_info, config) return config +def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: + if config["default"]["www"] != "": + return + if not config["default"]["infer_path_parameters"]: + raise ValueError("www must be provided when infer_path_parameters is False.") + + simboard_type = config["default"]["simboard_type"] + web_portal_base_path = machine_info.config.get("web_portal", "base_path") + config["default"]["www"] = ( + f"{web_portal_base_path}/simboard/{simboard_type}/" + ) + + def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> None: existing_bundles: List[Bundle] = [] diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 05800b22..44bf5f5c 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -70,6 +70,8 @@ plugins = force_list(default=list()) qos = string(default="regular") # Reservation -- if you have access to a node reservation, specify it with this parameter. reservation = string(default="") +# Which SimBoard namespace to use when inferring `www` +simboard_type = option("prod", "dev", default="prod") # Use for e3sm_to_cmip and/or ilamb tasks. # Name of the grid used by the relevant `[ts]` atm subtask ts_atm_grid = string(default="180x360_aave") @@ -106,8 +108,9 @@ walltime = string(default="02:00:00") # web_portal_base_path -- NOTE: this parameter is created internally # web_portal_base_url -- NOTE: this parameter is created internally # Where the post-processing visuals should go (to be viewed online) -# NOTE: no default, must be provided by user -www = string +# Leave blank to infer `/simboard//` +# when `infer_path_parameters = True` +www = string(default="") # The years to run; "1:100:20" would mean process years 1-100 in 20-year increments years = string_list(default=list("")) From d7d426fbb0865cd1335971bbba6a5eb59b9c2b66 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:53:41 +0000 Subject: [PATCH 02/12] Refine simboard default coverage --- tests/test_zppy_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index 91b6deda..6fa35d04 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -70,6 +70,7 @@ def test_determine_parameters_requires_www_without_path_inference() -> None: def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard.cfg" + default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" config_path.write_text( "\n".join( [ @@ -84,7 +85,7 @@ def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: ) config = ConfigObj( str(config_path), - configspec="/home/runner/work/zppy/zppy/zppy/defaults/default.ini", + configspec=str(default_ini), ) result = config.validate(Validator()) From 0c190f4162cd975878a9aecf122e4c9841d7d4d7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:44:27 +0000 Subject: [PATCH 03/12] Add explicit simboard publishing config --- tests/test_sections.py | 10 ++++- tests/test_zppy_main.py | 83 +++++++++++++++++++++++++++++++++------ zppy/__main__.py | 24 +++++++---- zppy/defaults/default.ini | 12 ++++-- zppy/simboard.py | 68 ++++++++++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 25 deletions(-) create mode 100644 zppy/simboard.py diff --git a/tests/test_sections.py b/tests/test_sections.py index 49a73b57..5c78d788 100644 --- a/tests/test_sections.py +++ b/tests/test_sections.py @@ -92,7 +92,6 @@ def test_sections(): "plugins": [], "reservation": "", "qos": "regular", - "simboard_type": "prod", "templateDir": "zppy/templates", "ts_atm_grid": "180x360_aave", "ts_atm_subsection": "", @@ -108,6 +107,15 @@ def test_sections(): } compare(actual_default, expected_default) + # simboard + section_name = "simboard" + actual_section = config[section_name] + expected_section = { + "enabled": False, + "simulation_type": "production", + } + compare(actual_section, expected_section) + # ts section_name = "ts" actual_section = config[section_name] diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index 6fa35d04..eeb6ba10 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -34,41 +34,96 @@ def _base_config() -> Dict[str, Dict[str, Any]]: "constraint": "", "environment_commands": "", "infer_path_parameters": True, - "simboard_type": "prod", "www": "", - } + }, + "simboard": { + "enabled": False, + "simulation_type": "production", + }, } @pytest.mark.parametrize( - ("simboard_type", "expected_www"), + ("simulation_type", "expected_www"), [ - ("prod", "/global/cfs/cdirs/e3sm/www/simboard/prod/"), - ("dev", "/global/cfs/cdirs/e3sm/www/simboard/dev/"), + ( + "production", + "/global/cfs/cdirs/e3sm/www/diagnostics_archive/production/", + ), + ( + "development", + "/global/cfs/cdirs/e3sm/www/diagnostics_archive/development/", + ), ], ) def test_determine_parameters_infers_simboard_www( - simboard_type: str, expected_www: str + simulation_type: str, expected_www: str ) -> None: config = _base_config() - config["default"]["simboard_type"] = simboard_type + config["simboard"]["enabled"] = True + config["simboard"]["simulation_type"] = simulation_type updated = _determine_parameters(_fake_machine_info(), config) assert updated["default"]["www"] == expected_www -def test_determine_parameters_requires_www_without_path_inference() -> None: +def test_determine_parameters_preserves_explicit_www_when_simboard_enabled() -> None: + config = _base_config() + config["default"]["www"] = "/custom/www" + config["simboard"]["enabled"] = True + + updated = _determine_parameters(_fake_machine_info(), config) + + assert updated["default"]["www"] == "/custom/www" + + +def test_determine_parameters_requires_www_without_simboard() -> None: config = _base_config() - config["default"]["infer_path_parameters"] = False with pytest.raises( - ValueError, match="www must be provided when infer_path_parameters is False." + ValueError, + match=( + r"www is empty\. Provide \[default\] www or set \[simboard\] " + r"enabled = True" + ), ): _determine_parameters(_fake_machine_info(), config) -def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: +def test_determine_parameters_rejects_none_simulation_type_when_enabled() -> None: + config = _base_config() + config["simboard"]["enabled"] = True + config["simboard"]["simulation_type"] = "none" + + with pytest.raises( + ValueError, + match=( + "simboard.simulation_type must be 'production' or 'development' " + "when simboard.enabled is True." + ), + ): + _determine_parameters(_fake_machine_info(), config) + + +def test_determine_parameters_requires_inferable_web_root() -> None: + config = _base_config() + config["simboard"]["enabled"] = True + machine_info = _fake_machine_info() + machine_info.config.remove_option("web_portal", "base_path") + + with pytest.raises( + ValueError, + match=( + "www is empty and simboard.enabled is True, but machine 'pm-cpu' " + "has no web_portal.base_path in mache; cannot infer a " + "diagnostics_archive path." + ), + ): + _determine_parameters(machine_info, config) + + +def test_default_ini_rejects_invalid_simulation_type(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard.cfg" default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" config_path.write_text( @@ -79,7 +134,9 @@ def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: "input = /input", "output = /output", "www = /www", - "simboard_type = invalid", + "", + "[simboard]", + "simulation_type = invalid", ] ) ) @@ -91,4 +148,4 @@ def test_default_ini_rejects_invalid_simboard_type(tmp_path: Path) -> None: result = config.validate(Validator()) assert result is not True - assert result["default"]["simboard_type"] is False + assert result["simboard"]["simulation_type"] is False diff --git a/zppy/__main__.py b/zppy/__main__.py index 3d89da65..bbd5d56b 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -23,6 +23,12 @@ from zppy.mpas_analysis import mpas_analysis from zppy.pcmdi_diags import pcmdi_diags from zppy.provenance import build_provenance_extras, write_provenance_settings +from zppy.simboard import ( + infer_simboard_www, + simboard, + simboard_enabled, + validate_simboard_config, +) from zppy.tc_analysis import tc_analysis from zppy.ts import ts from zppy.utils import check_status, submit_script @@ -264,16 +270,17 @@ def _determine_parameters(machine_info: MachineInfo, config: ConfigObj) -> Confi def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: + validate_simboard_config(config) if config["default"]["www"] != "": return - if not config["default"]["infer_path_parameters"]: - raise ValueError("www must be provided when infer_path_parameters is False.") - simboard_type = config["default"]["simboard_type"] - web_portal_base_path = machine_info.config.get("web_portal", "base_path") - config["default"]["www"] = ( - f"{web_portal_base_path}/simboard/{simboard_type}/" - ) + if not simboard_enabled(config): + raise ValueError( + "www is empty. Provide [default] www or set [simboard] enabled = " + "True to infer a SimBoard-compatible diagnostics_archive path." + ) + + config["default"]["www"] = infer_simboard_www(machine_info, config) def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> None: @@ -282,6 +289,9 @@ def _launch_scripts(config: ConfigObj, script_dir, job_ids_file, plugins) -> Non # predefined bundles existing_bundles = predefined_bundles(config, script_dir, existing_bundles) + # simboard configuration task + existing_bundles = simboard(config, script_dir, existing_bundles, job_ids_file) + # climo tasks existing_bundles = climo(config, script_dir, existing_bundles, job_ids_file) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 44bf5f5c..57d28d72 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -70,8 +70,6 @@ plugins = force_list(default=list()) qos = string(default="regular") # Reservation -- if you have access to a node reservation, specify it with this parameter. reservation = string(default="") -# Which SimBoard namespace to use when inferring `www` -simboard_type = option("prod", "dev", default="prod") # Use for e3sm_to_cmip and/or ilamb tasks. # Name of the grid used by the relevant `[ts]` atm subtask ts_atm_grid = string(default="180x360_aave") @@ -108,8 +106,8 @@ walltime = string(default="02:00:00") # web_portal_base_path -- NOTE: this parameter is created internally # web_portal_base_url -- NOTE: this parameter is created internally # Where the post-processing visuals should go (to be viewed online) -# Leave blank to infer `/simboard//` -# when `infer_path_parameters = True` +# Leave blank to infer `/diagnostics_archive//` +# when `[simboard] enabled = True` www = string(default="") # The years to run; "1:100:20" would mean process years 1-100 in 20-year increments years = string_list(default=list("")) @@ -121,6 +119,12 @@ active = boolean(default=True) [[__many__]] active = boolean(default=None) +[simboard] +# Opt in to SimBoard-compatible publishing behavior. +enabled = boolean(default=False) +# Use "none" only when SimBoard publishing is disabled. +simulation_type = option("production", "development", "none", default="production") + [climo] exclude = boolean(default=False) # NOTE: always overrides value in [default] diff --git a/zppy/simboard.py b/zppy/simboard.py new file mode 100644 index 00000000..c161d939 --- /dev/null +++ b/zppy/simboard.py @@ -0,0 +1,68 @@ +from configparser import NoOptionError, NoSectionError +from typing import List + +from configobj import ConfigObj +from mache import MachineInfo + +from zppy.bundle import Bundle +from zppy.logger import _setup_custom_logger + +logger = _setup_custom_logger(__name__) + + +def simboard( + config: ConfigObj, + script_dir: str, + existing_bundles: List[Bundle], + job_ids_file: str, +) -> List[Bundle]: + del script_dir, job_ids_file + if config["simboard"].sections: + raise ValueError("The [simboard] section does not support subsections.") + return existing_bundles + + +def simboard_enabled(config: ConfigObj) -> bool: + return bool(config["simboard"]["enabled"]) + + +def validate_simboard_config(config: ConfigObj) -> None: + if not simboard_enabled(config): + return + + simulation_type = str(config["simboard"]["simulation_type"]) + if simulation_type == "none": + raise ValueError( + "simboard.simulation_type must be 'production' or 'development' " + "when simboard.enabled is True." + ) + + +def infer_simboard_www(machine_info: MachineInfo, config: ConfigObj) -> str: + simulation_type = str(config["simboard"]["simulation_type"]) + try: + web_portal_base_path = machine_info.config.get("web_portal", "base_path") + except (NoSectionError, NoOptionError) as exc: + raise ValueError( + f"www is empty and simboard.enabled is True, but machine " + f"'{machine_info.machine}' has no web_portal.base_path in mache; " + "cannot infer a diagnostics_archive path." + ) from exc + + web_portal_base_path = web_portal_base_path.rstrip("/") + if web_portal_base_path == "": + raise ValueError( + f"www is empty and simboard.enabled is True, but machine " + f"'{machine_info.machine}' has an empty web_portal.base_path in " + "mache; cannot infer a diagnostics_archive path." + ) + + inferred_www = ( + f"{web_portal_base_path}/diagnostics_archive/{simulation_type}/" + ) + logger.info( + "Inferred www=%s from mache web_portal.base_path because " + "simboard.enabled is True.", + inferred_www, + ) + return inferred_www From 1d1921ad3c3c329fd0d5ebde147c99ea677e8628 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:45:41 +0000 Subject: [PATCH 04/12] Tighten simboard config messaging --- tests/test_zppy_main.py | 4 ++-- zppy/__main__.py | 5 +++-- zppy/defaults/default.ini | 1 + zppy/simboard.py | 5 ++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index eeb6ba10..16ddb6c0 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -84,8 +84,8 @@ def test_determine_parameters_requires_www_without_simboard() -> None: with pytest.raises( ValueError, match=( - r"www is empty\. Provide \[default\] www or set \[simboard\] " - r"enabled = True" + r"www is empty\. Provide \[default\] www or set `enabled = True` " + r"in the \[simboard\] section" ), ): _determine_parameters(_fake_machine_info(), config) diff --git a/zppy/__main__.py b/zppy/__main__.py index bbd5d56b..ea2d0fd4 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -276,8 +276,9 @@ def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: if not simboard_enabled(config): raise ValueError( - "www is empty. Provide [default] www or set [simboard] enabled = " - "True to infer a SimBoard-compatible diagnostics_archive path." + "www is empty. Provide [default] www or set `enabled = True` in " + "the [simboard] section to infer a SimBoard-compatible " + "diagnostics_archive path." ) config["default"]["www"] = infer_simboard_www(machine_info, config) diff --git a/zppy/defaults/default.ini b/zppy/defaults/default.ini index 57d28d72..6da1673f 100755 --- a/zppy/defaults/default.ini +++ b/zppy/defaults/default.ini @@ -123,6 +123,7 @@ active = boolean(default=True) # Opt in to SimBoard-compatible publishing behavior. enabled = boolean(default=False) # Use "none" only when SimBoard publishing is disabled. +# Default to "production" so enabled configs can opt in without overriding it. simulation_type = option("production", "development", "none", default="production") [climo] diff --git a/zppy/simboard.py b/zppy/simboard.py index c161d939..26811dd3 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -12,11 +12,10 @@ def simboard( config: ConfigObj, - script_dir: str, + _script_dir: str, existing_bundles: List[Bundle], - job_ids_file: str, + _job_ids_file: str, ) -> List[Bundle]: - del script_dir, job_ids_file if config["simboard"].sections: raise ValueError("The [simboard] section does not support subsections.") return existing_bundles From 3da1fee218ac9897ff0f48dfef187cf108a9b149 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:46:30 +0000 Subject: [PATCH 05/12] Clarify simboard task intent --- zppy/__main__.py | 2 ++ zppy/simboard.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/zppy/__main__.py b/zppy/__main__.py index ea2d0fd4..4827e6ba 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -270,6 +270,8 @@ def _determine_parameters(machine_info: MachineInfo, config: ConfigObj) -> Confi def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: + # Keep SimBoard-specific validation active even when `www` is already set, + # because `[simboard] enabled = True` still enables SimBoard validation. validate_simboard_config(config) if config["default"]["www"] != "": return diff --git a/zppy/simboard.py b/zppy/simboard.py index 26811dd3..55c9a97a 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -16,12 +16,23 @@ def simboard( existing_bundles: List[Bundle], _job_ids_file: str, ) -> List[Bundle]: + """Validate the configuration-only `[simboard]` task hook. + + This section is an explicit top-level SimBoard configuration entry point, + analogous to `[bundle]`: it influences how other tasks are configured, but + it does not launch an HPC job of its own. + """ if config["simboard"].sections: raise ValueError("The [simboard] section does not support subsections.") return existing_bundles def simboard_enabled(config: ConfigObj) -> bool: + """Return whether SimBoard publishing is enabled. + + Assumes `config` has already been validated against `default.ini`, which + provides the `[simboard]` section and its default values. + """ return bool(config["simboard"]["enabled"]) From 32e13ff0874c71b484f784a392feaf38bf6f9262 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:47:18 +0000 Subject: [PATCH 06/12] Clean up simboard validation helpers --- zppy/__main__.py | 3 ++- zppy/simboard.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/zppy/__main__.py b/zppy/__main__.py index 4827e6ba..870a3743 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -280,7 +280,8 @@ def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: raise ValueError( "www is empty. Provide [default] www or set `enabled = True` in " "the [simboard] section to infer a SimBoard-compatible " - "diagnostics_archive path." + "diagnostics_archive path. Note: inference requires " + "web_portal.base_path in Mache configuration." ) config["default"]["www"] = infer_simboard_www(machine_info, config) diff --git a/zppy/simboard.py b/zppy/simboard.py index 55c9a97a..803ac235 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -20,7 +20,8 @@ def simboard( This section is an explicit top-level SimBoard configuration entry point, analogous to `[bundle]`: it influences how other tasks are configured, but - it does not launch an HPC job of its own. + it does not launch an HPC job of its own. The unused task-like parameters + are retained so this hook matches the call signature of other zppy tasks. """ if config["simboard"].sections: raise ValueError("The [simboard] section does not support subsections.") @@ -33,14 +34,14 @@ def simboard_enabled(config: ConfigObj) -> bool: Assumes `config` has already been validated against `default.ini`, which provides the `[simboard]` section and its default values. """ - return bool(config["simboard"]["enabled"]) + return config["simboard"]["enabled"] def validate_simboard_config(config: ConfigObj) -> None: if not simboard_enabled(config): return - simulation_type = str(config["simboard"]["simulation_type"]) + simulation_type = config["simboard"]["simulation_type"] if simulation_type == "none": raise ValueError( "simboard.simulation_type must be 'production' or 'development' " @@ -49,7 +50,7 @@ def validate_simboard_config(config: ConfigObj) -> None: def infer_simboard_www(machine_info: MachineInfo, config: ConfigObj) -> str: - simulation_type = str(config["simboard"]["simulation_type"]) + simulation_type = config["simboard"]["simulation_type"] try: web_portal_base_path = machine_info.config.get("web_portal", "base_path") except (NoSectionError, NoOptionError) as exc: From e6e0f1e082dc946041ed65901cccbd7a986af0b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:48:17 +0000 Subject: [PATCH 07/12] Harden simboard config checks --- tests/test_zppy_main.py | 40 ++++++++++++++++++++++++++++++++++++++++ zppy/simboard.py | 13 +++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index 16ddb6c0..aeaf7ddd 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -8,6 +8,7 @@ from validate import Validator from zppy.__main__ import _determine_parameters +from zppy.simboard import simboard def _fake_machine_info() -> MagicMock: @@ -123,6 +124,45 @@ def test_determine_parameters_requires_inferable_web_root() -> None: _determine_parameters(machine_info, config) +def test_determine_parameters_rejects_empty_web_root() -> None: + config = _base_config() + config["simboard"]["enabled"] = True + machine_info = _fake_machine_info() + machine_info.config["web_portal"]["base_path"] = " " + + with pytest.raises( + ValueError, + match=( + "www is empty and simboard.enabled is True, but machine 'pm-cpu' " + "has an empty web_portal.base_path in mache; cannot infer a " + "diagnostics_archive path." + ), + ): + _determine_parameters(machine_info, config) + + +def test_simboard_rejects_subsections(tmp_path: Path) -> None: + config_path = tmp_path / "bad_simboard_subsection.cfg" + config_path.write_text( + "\n".join( + [ + "[simboard]", + "enabled = False", + "simulation_type = production", + "", + " [[nested]]", + " placeholder = value", + ] + ) + ) + config = ConfigObj(str(config_path)) + + with pytest.raises( + ValueError, match="The \\[simboard\\] section does not support subsections." + ): + simboard(config, "", [], "") + + def test_default_ini_rejects_invalid_simulation_type(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard.cfg" default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" diff --git a/zppy/simboard.py b/zppy/simboard.py index 803ac235..1352aacc 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -34,7 +34,16 @@ def simboard_enabled(config: ConfigObj) -> bool: Assumes `config` has already been validated against `default.ini`, which provides the `[simboard]` section and its default values. """ - return config["simboard"]["enabled"] + enabled = config["simboard"]["enabled"] + if isinstance(enabled, bool): + return enabled + if isinstance(enabled, str): + enabled_lower = enabled.lower() + if enabled_lower == "true": + return True + if enabled_lower == "false": + return False + raise ValueError(f"Invalid value {enabled} for simboard.enabled") def validate_simboard_config(config: ConfigObj) -> None: @@ -60,7 +69,7 @@ def infer_simboard_www(machine_info: MachineInfo, config: ConfigObj) -> str: "cannot infer a diagnostics_archive path." ) from exc - web_portal_base_path = web_portal_base_path.rstrip("/") + web_portal_base_path = web_portal_base_path.strip().rstrip("/") if web_portal_base_path == "": raise ValueError( f"www is empty and simboard.enabled is True, but machine " From 3d6647409cc4390663343d6d0ef39a913930b4d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:49:17 +0000 Subject: [PATCH 08/12] Clarify simboard validation assumptions --- tests/test_zppy_main.py | 9 ++++++++- zppy/__main__.py | 3 ++- zppy/simboard.py | 17 +++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index aeaf7ddd..0e341dec 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -143,9 +143,16 @@ def test_determine_parameters_rejects_empty_web_root() -> None: def test_simboard_rejects_subsections(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard_subsection.cfg" + default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" config_path.write_text( "\n".join( [ + "[default]", + "case = case_name", + "input = /input", + "output = /output", + "www = /www", + "", "[simboard]", "enabled = False", "simulation_type = production", @@ -155,7 +162,7 @@ def test_simboard_rejects_subsections(tmp_path: Path) -> None: ] ) ) - config = ConfigObj(str(config_path)) + config = ConfigObj(str(config_path), configspec=str(default_ini)) with pytest.raises( ValueError, match="The \\[simboard\\] section does not support subsections." diff --git a/zppy/__main__.py b/zppy/__main__.py index 870a3743..c5136baa 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -271,7 +271,8 @@ def _determine_parameters(machine_info: MachineInfo, config: ConfigObj) -> Confi def _set_default_www(machine_info: MachineInfo, config: ConfigObj) -> None: # Keep SimBoard-specific validation active even when `www` is already set, - # because `[simboard] enabled = True` still enables SimBoard validation. + # because `[simboard] enabled = True` still requires validating + # `simulation_type`. validate_simboard_config(config) if config["default"]["www"] != "": return diff --git a/zppy/simboard.py b/zppy/simboard.py index 1352aacc..ab636409 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -10,6 +10,10 @@ logger = _setup_custom_logger(__name__) +def normalize_web_portal_base_path(web_portal_base_path: str) -> str: + return web_portal_base_path.strip().rstrip("/") + + def simboard( config: ConfigObj, _script_dir: str, @@ -22,7 +26,13 @@ def simboard( analogous to `[bundle]`: it influences how other tasks are configured, but it does not launch an HPC job of its own. The unused task-like parameters are retained so this hook matches the call signature of other zppy tasks. + This hook assumes the config has already been read and validated. """ + if "simboard" not in config: + raise ValueError( + "Missing [simboard] section. Validate the config against " + "default.ini before calling simboard()." + ) if config["simboard"].sections: raise ValueError("The [simboard] section does not support subsections.") return existing_bundles @@ -43,7 +53,10 @@ def simboard_enabled(config: ConfigObj) -> bool: return True if enabled_lower == "false": return False - raise ValueError(f"Invalid value {enabled} for simboard.enabled") + raise ValueError( + f"Invalid value '{enabled}' for simboard.enabled. Expected boolean " + "or string 'true'/'false'." + ) def validate_simboard_config(config: ConfigObj) -> None: @@ -69,7 +82,7 @@ def infer_simboard_www(machine_info: MachineInfo, config: ConfigObj) -> str: "cannot infer a diagnostics_archive path." ) from exc - web_portal_base_path = web_portal_base_path.strip().rstrip("/") + web_portal_base_path = normalize_web_portal_base_path(web_portal_base_path) if web_portal_base_path == "": raise ValueError( f"www is empty and simboard.enabled is True, but machine " From ee45ff1ad4576216c7a7038b2806959444028858 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:50:32 +0000 Subject: [PATCH 09/12] Add simboard helper coverage --- tests/test_zppy_main.py | 81 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index 0e341dec..29bd8d16 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -8,7 +8,12 @@ from validate import Validator from zppy.__main__ import _determine_parameters -from zppy.simboard import simboard +from zppy.simboard import ( + infer_simboard_www, + normalize_web_portal_base_path, + simboard, + simboard_enabled, +) def _fake_machine_info() -> MagicMock: @@ -69,6 +74,80 @@ def test_determine_parameters_infers_simboard_www( assert updated["default"]["www"] == expected_www +@pytest.mark.parametrize( + ("base_path", "expected_www"), + [ + ( + "/global/cfs/cdirs/e3sm/www", + "/global/cfs/cdirs/e3sm/www/diagnostics_archive/production/", + ), + ( + "/global/cfs/cdirs/e3sm/www/", + "/global/cfs/cdirs/e3sm/www/diagnostics_archive/production/", + ), + ( + " /global/cfs/cdirs/e3sm/www/ ", + "/global/cfs/cdirs/e3sm/www/diagnostics_archive/production/", + ), + ], +) +def test_infer_simboard_www_normalizes_web_root( + base_path: str, expected_www: str +) -> None: + machine_info = _fake_machine_info() + machine_info.config["web_portal"]["base_path"] = base_path + config = _base_config() + + assert infer_simboard_www(machine_info, config) == expected_www + + +@pytest.mark.parametrize( + ("raw_path", "expected_path"), + [ + ("/global/cfs/cdirs/e3sm/www", "/global/cfs/cdirs/e3sm/www"), + ("/global/cfs/cdirs/e3sm/www/", "/global/cfs/cdirs/e3sm/www"), + (" /global/cfs/cdirs/e3sm/www/ ", "/global/cfs/cdirs/e3sm/www"), + (" ", ""), + ], +) +def test_normalize_web_portal_base_path(raw_path: str, expected_path: str) -> None: + assert normalize_web_portal_base_path(raw_path) == expected_path + + +@pytest.mark.parametrize( + ("enabled_value", "expected_enabled"), + [ + (True, True), + (False, False), + ("true", True), + ("TRUE", True), + ("false", False), + ("FALSE", False), + ], +) +def test_simboard_enabled_parses_bool_values( + enabled_value: Any, expected_enabled: bool +) -> None: + config = _base_config() + config["simboard"]["enabled"] = enabled_value + + assert simboard_enabled(config) is expected_enabled + + +def test_simboard_enabled_rejects_invalid_value() -> None: + config = _base_config() + config["simboard"]["enabled"] = "maybe" + + with pytest.raises( + ValueError, + match=( + "Invalid value 'maybe' for simboard.enabled. Expected boolean " + "or string 'true'/'false'." + ), + ): + simboard_enabled(config) + + def test_determine_parameters_preserves_explicit_www_when_simboard_enabled() -> None: config = _base_config() config["default"]["www"] = "/custom/www" From 5198b186da66a96730d85302ef12ba8b640cd49d Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 24 Jul 2026 12:44:07 -0500 Subject: [PATCH 10/12] Fix pre-commit errors --- tests/test_zppy_main.py | 8 ++++++-- zppy/simboard.py | 4 +--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_zppy_main.py b/tests/test_zppy_main.py index 29bd8d16..1cb133c8 100644 --- a/tests/test_zppy_main.py +++ b/tests/test_zppy_main.py @@ -222,7 +222,9 @@ def test_determine_parameters_rejects_empty_web_root() -> None: def test_simboard_rejects_subsections(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard_subsection.cfg" - default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" + default_ini = ( + Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" + ) config_path.write_text( "\n".join( [ @@ -251,7 +253,9 @@ def test_simboard_rejects_subsections(tmp_path: Path) -> None: def test_default_ini_rejects_invalid_simulation_type(tmp_path: Path) -> None: config_path = tmp_path / "bad_simboard.cfg" - default_ini = Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" + default_ini = ( + Path(__file__).resolve().parents[1] / "zppy" / "defaults" / "default.ini" + ) config_path.write_text( "\n".join( [ diff --git a/zppy/simboard.py b/zppy/simboard.py index ab636409..f425fcef 100644 --- a/zppy/simboard.py +++ b/zppy/simboard.py @@ -90,9 +90,7 @@ def infer_simboard_www(machine_info: MachineInfo, config: ConfigObj) -> str: "mache; cannot infer a diagnostics_archive path." ) - inferred_www = ( - f"{web_portal_base_path}/diagnostics_archive/{simulation_type}/" - ) + inferred_www = f"{web_portal_base_path}/diagnostics_archive/{simulation_type}/" logger.info( "Inferred www=%s from mache web_portal.base_path because " "simboard.enabled is True.", From de3b2a05419dd424ba0b085383b7420265d30320 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:59:43 +0000 Subject: [PATCH 11/12] Fix NoOptionError crash before SimBoard www inference Wrap `web_portal.base_path` and `web_portal.base_url` reads in `_determine_parameters()` with try/except to catch `NoSectionError`/`NoOptionError`, defaulting to `""`. This lets the `NoOptionError` be handled properly downstream in `infer_simboard_www()`, which raises the intended `ValueError` when `www` is empty and `simboard.enabled` is True. Fixes `test_determine_parameters_requires_inferable_web_root` which was failing with `configparser.NoOptionError` before reaching the SimBoard error-handling code. --- zppy/__main__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/zppy/__main__.py b/zppy/__main__.py index c5136baa..1c96939c 100644 --- a/zppy/__main__.py +++ b/zppy/__main__.py @@ -225,12 +225,18 @@ def _determine_parameters(machine_info: MachineInfo, config: ConfigObj) -> Confi config["default"]["diagnostics_base_path"] = machine_info.config.get( "diagnostics", "base_path" ) - config["default"]["web_portal_base_path"] = machine_info.config.get( - "web_portal", "base_path" - ) - config["default"]["web_portal_base_url"] = machine_info.config.get( - "web_portal", "base_url" - ) + try: + config["default"]["web_portal_base_path"] = machine_info.config.get( + "web_portal", "base_path" + ) + except (configparser.NoSectionError, configparser.NoOptionError): + config["default"]["web_portal_base_path"] = "" + try: + config["default"]["web_portal_base_url"] = machine_info.config.get( + "web_portal", "base_url" + ) + except (configparser.NoSectionError, configparser.NoOptionError): + config["default"]["web_portal_base_url"] = "" # Determine machine to decide which header files to use if ("machine" not in config["default"]) or (config["default"]["machine"] == ""): From bfa34d452c81b4c5e975ddf85bf45763c8b8fc53 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:27:04 +0000 Subject: [PATCH 12/12] Add docs for the new [simboard] configuration section - Add docs/source/user_guide/tasks/simboard.rst describing the [simboard] section, its expected behavior table, a configuration example, and its parameters. - Add simboard to the tasks index (list and toctree). - Update parameters.rst: www is now optional (inferred when simboard.enabled=True); add a SimBoard section parameters table at the end. --- docs/source/user_guide/parameters.rst | 34 ++++++++- docs/source/user_guide/tasks/index.rst | 3 + docs/source/user_guide/tasks/simboard.rst | 90 +++++++++++++++++++++++ 3 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 docs/source/user_guide/tasks/simboard.rst diff --git a/docs/source/user_guide/parameters.rst b/docs/source/user_guide/parameters.rst index ee44b65a..053d3572 100644 --- a/docs/source/user_guide/parameters.rst +++ b/docs/source/user_guide/parameters.rst @@ -68,9 +68,11 @@ There are 6 output-specific parameters: - *(none)* - Where the post-processing results (``post/`` directory) should go. * - ``www`` - - **Yes** - - *(none)* + - No + - ``""`` - Where the post-processing visuals should go (to be viewed online). + Leave empty and set ``[simboard] enabled = True`` to have ``zppy`` + infer this path from Mache. See :doc:`tasks/simboard` for details. * - ``campaign`` - No - ``"none"`` @@ -404,6 +406,34 @@ These are no longer defined in ``zppy/defaults/default.ini``: These are still defined in ``zppy/defaults/default.ini``, but have no effect: .. code-block:: text + ncclimo_cmd nrows ncols + +SimBoard section parameters +============================ + +The ``[simboard]`` section controls SimBoard-compatible publishing. It is +a configuration-only hook; see :doc:`tasks/simboard` for full details. + +.. list-table:: + :header-rows: 1 + :widths: 22 10 18 50 + + * - Parameter + - Required + - Default + - Description + * - ``enabled`` + - No + - ``False`` + - Set to ``True`` to enable SimBoard-compatible publishing. + When ``True`` and ``[default] www`` is empty, ``zppy`` infers + ``www`` from Mache's ``web_portal.base_path``. + * - ``simulation_type`` + - No + - ``"production"`` + - Archive sub-directory for the run. One of ``"production"``, + ``"development"``, or ``"none"``. + Must not be ``"none"`` when ``enabled = True``. diff --git a/docs/source/user_guide/tasks/index.rst b/docs/source/user_guide/tasks/index.rst index 9dc8dd63..985e5e57 100644 --- a/docs/source/user_guide/tasks/index.rst +++ b/docs/source/user_guide/tasks/index.rst @@ -21,6 +21,8 @@ Listed for reference (bundle jobs are submitted after task jobs are generated in - Description * - :doc:`bundle` - Bundle multiple tasks into a single SLURM job + * - :doc:`simboard` + - Configure SimBoard-compatible diagnostics publishing * - :doc:`climo` - Generate climatology files using NCO's ``ncclimo`` * - :doc:`ts` @@ -50,6 +52,7 @@ Listed for reference (bundle jobs are submitted after task jobs are generated in :hidden: bundle + simboard climo ts e3sm_to_cmip diff --git a/docs/source/user_guide/tasks/simboard.rst b/docs/source/user_guide/tasks/simboard.rst new file mode 100644 index 00000000..b0fa991f --- /dev/null +++ b/docs/source/user_guide/tasks/simboard.rst @@ -0,0 +1,90 @@ +.. _task-simboard: + +simboard — SimBoard Publishing Configuration +============================================ + +The ``simboard`` section is a configuration-only hook that controls +SimBoard-compatible publishing behavior. Like :doc:`bundle`, it does not +launch an HPC job of its own; instead it influences how other tasks are +configured — specifically, it can infer the ``www`` output path from the +machine's Mache configuration. + +When ``enabled = True`` and ``www`` is left empty in ``[default]``, +``zppy`` derives ``www`` from the ``web_portal.base_path`` recorded in +Mache for the current machine: + +.. code-block:: text + + /diagnostics_archive// + +This gives SimBoard a single, predictable archive root to scan for +diagnostics. + +Expected behavior +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 15 65 + + * - ``simboard.enabled`` + - ``www`` + - Behavior + * - ``False`` + - any + - ``zppy`` does nothing SimBoard-specific. + * - ``True`` + - empty + - Infer the SimBoard archive path from Mache's + ``web_portal.base_path``. + * - ``True`` + - set + - Use the explicit ``www`` path and do not override it. + ``simboard.enabled`` still controls SimBoard-specific validation + (e.g., ``simulation_type`` must not be ``"none"``). + * - ``True`` + - empty, but path cannot be inferred + - Raise a clear configuration error. + +Configuration example +--------------------- + +.. code-block:: cfg + + [default] + case = v3.LR.historical_0051 + input = /path/to/input + output = /path/to/output + # Leave www empty to let zppy infer it from Mache when simboard is enabled. + www = + + [simboard] + enabled = True + simulation_type = production + +Parameters +---------- + +.. list-table:: + :header-rows: 1 + :widths: 22 10 18 50 + + * - Parameter + - Required + - Default + - Description + * - ``enabled`` + - No + - ``False`` + - Set to ``True`` to enable SimBoard-compatible publishing behavior. + When enabled and ``[default] www`` is empty, ``zppy`` infers + ``www`` from Mache's ``web_portal.base_path``. + * - ``simulation_type`` + - No + - ``"production"`` + - Diagnostic classification for the archive path. One of + ``"production"``, ``"development"``, or ``"none"``. + Must not be ``"none"`` when ``enabled = True``. + +.. note:: + The ``[simboard]`` section does not support subsections.