diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index fba6fca..4640b94 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -109,75 +109,79 @@ jobs: pytest tests/unit/global_time_series/test_*.py # If the branch updates documentation, then the docs will need to be updated. - # publish-docs: - # if: ${{ github.event_name == 'push' }} - # runs-on: ubuntu-latest - # defaults: - # run: - # shell: bash -l {0} - # timeout-minutes: 10 # Increased timeout for docs - # steps: - # - uses: actions/checkout@v2 - # with: - # persist-credentials: false - # fetch-depth: 0 - - # - name: Cache Conda - # uses: actions/cache@v3 - # env: - # CACHE_NUMBER: 1 # Match the build job cache number - # with: - # path: ~/conda_pkgs_dir - # key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ - # hashFiles('conda/dev.yml') }}-docs - - # - name: Build Conda Environment - # uses: conda-incubator/setup-miniconda@v3 - # with: - # activate-environment: zppy_interfaces_dev - # miniforge-variant: Miniforge3 - # miniforge-version: latest - # environment-file: conda/dev.yml - # channel-priority: strict - # auto-update-conda: true - # python-version: "3.13" # Use stable Python version for docs - - # - if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }} - # name: Show Conda Environment Info - # run: | - # conda config --set anaconda_upload no - # conda info - # conda list - - # - name: Install `zppy-interfaces` Package - # run: pip install . - - # # sphinx-multiversion allows for version docs. - # - name: Build Sphinx Docs - # run: | - # cd docs - # sphinx-multiversion source _build/html - - # - name: Copy Docs and Commit - # run: | - # # gh-pages branch must already exist - # git clone https://github.com/E3SM-Project/zppy-interfaces.git --branch gh-pages --single-branch gh-pages - - # # Only replace main docs with latest changes. Docs for tags should be untouched. - # cd gh-pages - # rm -r _build/html/main - # cp -r ../docs/_build/html/main _build/html/main - # git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - # git config --local user.name "github-actions[bot]" - - # # The below command will fail if no changes were present, so we ignore it - # git add . - # git commit -m "Update documentation" -a || true - - # - name: Push Changes - # uses: ad-m/github-push-action@master - # with: - # branch: gh-pages - # directory: gh-pages - # github_token: ${{ secrets.GITHUB_TOKEN }} - # force: true + publish-docs: + needs: check-jobs-to-skip + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + shell: bash -l {0} + timeout-minutes: 10 # Increased timeout for docs + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Cache Conda + uses: actions/cache@v3 + env: + CACHE_NUMBER: 1 # Match the build job cache number + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ + hashFiles('conda/dev.yml') }}-docs + + - name: Build Conda Environment + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: zppy_interfaces_dev + miniforge-variant: Miniforge3 + miniforge-version: latest + environment-file: conda/dev.yml + channel-priority: strict + auto-update-conda: true + python-version: "3.13" # Use stable Python version for docs + + - if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }} + name: Show Conda Environment Info + run: | + conda config --set anaconda_upload no + conda info + conda list + + - name: Install `zppy-interfaces` Package + run: python -m pip install . + + # sphinx-multiversion allows for version docs. + - name: Build Sphinx Docs + run: | + cd docs + sphinx-multiversion source _build/html + + - name: Copy Docs and Commit + run: | + # gh-pages branch must already exist + git clone https://github.com/E3SM-Project/zppy-interfaces.git --branch gh-pages --single-branch gh-pages + + # Only replace main docs with latest changes. Docs for tags should be untouched. + cd gh-pages + mkdir -p _build/html + rm -rf _build/html/main + cp -r ../docs/_build/html/main _build/html/main + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + + # The below command will fail if no changes were present, so we ignore it + git add . + git commit -m "Update documentation" -a || true + + - name: Push Changes + uses: ad-m/github-push-action@v0.8.0 + with: + branch: gh-pages + directory: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + force: true diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..063ce72 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = zppy-interfaces +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/_static/style.css b/docs/source/_static/style.css new file mode 100644 index 0000000..8aa6c28 --- /dev/null +++ b/docs/source/_static/style.css @@ -0,0 +1,3 @@ +.wy-nav-content { + max-width: 1200px !important; +} diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html new file mode 100644 index 0000000..3e44f4a --- /dev/null +++ b/docs/source/_templates/layout.html @@ -0,0 +1,4 @@ +{% extends "!layout.html" %} +{% block extrahead %} + +{% endblock %} diff --git a/docs/source/_templates/versions.html b/docs/source/_templates/versions.html new file mode 100644 index 0000000..31a1257 --- /dev/null +++ b/docs/source/_templates/versions.html @@ -0,0 +1,27 @@ +{%- if current_version %} +
+ + Other Versions + v: {{ current_version.name }} + + +
+ {%- if versions.tags %} +
+
Tags
+ {%- for item in versions.tags %} +
{{ item.name }}
+ {%- endfor %} +
+ {%- endif %} + {%- if versions.branches %} +
+
Branches
+ {%- for item in versions.branches %} +
{{ item.name }}
+ {%- endfor %} +
+ {%- endif %} +
+
+{%- endif %} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..cbe6e34 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +import re + +project = "zppy-interfaces" +author = "E3SM Project" +copyright = f"{datetime.now():%Y}, {author}" + +_repo_root = Path(__file__).resolve().parents[2] +_version_file = _repo_root / "zppy_interfaces" / "version.py" +_version_match = re.search( + r'__version__\s*=\s*"([^"]+)"', _version_file.read_text(encoding="utf-8") +) +release = _version_match.group(1) if _version_match else "unknown" +version = release + +extensions = [] +templates_path = ["_templates"] +exclude_patterns = ["_build"] + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] +html_css_files = ["style.css"] +html_title = "zppy-interfaces documentation" +html_sidebars = { + "**": [ + "globaltoc.html", + "localtoc.html", + "relations.html", + "searchbox.html", + "versions.html", + ] +} + +smv_tag_whitelist = r"^v.*$" +smv_branch_whitelist = r"^(main|master)$" +smv_remote_whitelist = r"^origin$" diff --git a/docs/source/developer_global_time_series.rst b/docs/source/developer_global_time_series.rst new file mode 100644 index 0000000..c2b71b5 --- /dev/null +++ b/docs/source/developer_global_time_series.rst @@ -0,0 +1,51 @@ +*********************************** +Developer Guide: global_time_series +*********************************** + +This page describes how ``zi-global-time-series`` is implemented. For the +runtime parameters and output-facing behavior, see :doc:`global_time_series`. + +Entry point and main flow +========================= + +The CLI entry point is ``zppy_interfaces.global_time_series.__main__:main``. +The implementation follows this sequence: + +#. ``__main__._get_args()`` parses the command-line arguments and builds a + ``zppy_interfaces.global_time_series.utils.Parameters`` instance. +#. ``Parameters`` normalizes booleans, region aliases, and comma-separated plot + selections, then validates that the request is internally consistent. +#. ``main()`` optionally calls ``create_ocean_ts()`` when ocean-dependent + classic plots are requested with ``use_ocn=True``. +#. ``main()`` then calls ``run_coupled_global()``. +#. ``run_coupled_global()`` builds a + ``zppy_interfaces.global_time_series.coupled_global.utils.RequestedVariables`` + object, generates the plots, and optionally creates viewer HTML output. + +Key modules +=========== + +* ``zppy_interfaces/global_time_series/__main__.py`` contains the CLI parser + and the top-level control flow. +* ``zppy_interfaces/global_time_series/utils.py`` defines ``Parameters`` and + the request normalization logic. +* ``zppy_interfaces/global_time_series/create_ocean_ts.py`` creates ocean + support time series before plotting when needed. +* ``zppy_interfaces/global_time_series/coupled_global/driver.py`` coordinates + original plots, component plots, and viewer generation. +* ``zppy_interfaces/global_time_series/coupled_global/mode_pdf.py`` assembles + cumulative PDFs when ``make_viewer=False``. +* ``zppy_interfaces/global_time_series/coupled_global/mode_viewer.py`` and the + ``mix_viewer_*`` helpers build the viewer pages when ``make_viewer=True``. + +Developer notes +=============== + +* The classic plot names in ``plots_original`` are not the same thing as raw + variable names, so the driver keeps them separate from the component-variable + lists. +* Land plots depend on ``zppy_land_fields.csv`` for the accepted variable set, + grouping metadata, units, and long names. +* The driver always writes output relative to ``results_dir``, and viewer mode + adds HTML pages on top of the figure generation rather than replacing the + underlying plot production. diff --git a/docs/source/developer_guide.rst b/docs/source/developer_guide.rst new file mode 100644 index 0000000..c0a1237 --- /dev/null +++ b/docs/source/developer_guide.rst @@ -0,0 +1,62 @@ +****************** +Developer Guide +****************** + +Use these pages when you want to understand how the repository is organized, +where each CLI entry point starts, and how the internal processing flow is +implemented. + +Repository layout +================= + +The implementation is organized under ``zppy_interfaces``: + +* ``zppy_interfaces/global_time_series`` contains the global time-series CLI, + ocean time-series generation helpers, HTML templates, and the + ``zppy_land_fields.csv`` variable catalogue. +* ``zppy_interfaces/pcmdi_diags`` contains CLI entry points for observation + linking, mean-climate diagnostics, variability modes, ENSO, synthetic plots, + and the shared setup logic used by those commands. +* ``zppy_interfaces/multi_utils`` contains shared logging and viewer helpers + used by both interfaces. + +Interface implementation guides +=============================== + +.. toctree:: + :maxdepth: 1 + + developer_global_time_series + developer_pcmdi_diags + +Testing +======= + +.. code-block:: bash + + # Set up branch + cd zppy-interfaces + git status # Check for uncommitted changes + git fetch upstream main + git checkout main + git reset --hard upstream/main + git log --oneline | head -n 1 + # Check that we're up to date with either: + # 1. The latest commit on main: https://github.com/E3SM-Project/zppy-interfaces/commits/main/ + # 2. The commits of the pull request being tested. + + # Set up conda environment + bash # Run bash so we're in an isolated subshell + # Make sure conda is activated + rm -rf build + conda clean --all --y + conda env create -f conda/dev.yml -n env-name + conda activate env-name + pre-commit run --all-files + python -m pip install . + + # Run unit tests + pytest tests/unit/global_time_series/test_*.py + pytest tests/unit/pcmdi_diags/test_*.py + + # Note that integration testing is done as part of zppy's testing. diff --git a/docs/source/developer_pcmdi_diags.rst b/docs/source/developer_pcmdi_diags.rst new file mode 100644 index 0000000..6348fd3 --- /dev/null +++ b/docs/source/developer_pcmdi_diags.rst @@ -0,0 +1,71 @@ +***************************** +Developer Guide: pcmdi_diags +***************************** + +This page describes how the PCMDI-related CLI entry points are implemented. For +the runtime parameters and command behavior, see :doc:`pcmdi_diags`. + +Implementation map +================== + +The PCMDI interface is split across several CLI entry points: + +* ``link_observation.py`` handles observation discovery, linking, and limited + derived-variable generation. +* ``pcmdi_mean_cimate.py`` runs mean-climate diagnostics and reorganizes the + resulting outputs. +* ``pcmdi_variability_modes.py`` runs variability-mode diagnostics and collects + the resulting files. +* ``pcmdi_enso.py`` defines the intended ENSO interface, but ``main()`` still + exits early because the command is not yet supported. +* ``pcmdi_synthetic_plots.py`` builds summary plots and the combined viewer + pages from prior diagnostics output. + +Shared setup path +================= + +``zi-pcmdi-mean-climate``, ``zi-pcmdi-variability-modes``, and +``zi-pcmdi-enso`` all start by calling ``zppy_interfaces.pcmdi_diags.pcmdi_setup``: + +#. Each CLI parses its command-specific arguments plus the shared core + arguments. +#. ``CoreParameters`` stores the shared values needed by downstream commands. +#. ``set_up()`` resolves the reference data source, builds catalogue JSON files, + optionally generates land-sea masks, and returns a ``CoreOutput`` object with + reusable path templates and observation metadata. +#. The calling CLI then generates command lists or viewer inputs using that + shared setup result. + +Command-specific flows +====================== + +Mean climate +------------ + +#. Parse arguments and run the shared setup. +#. Save ``regions.json`` for variable-to-region mapping. +#. Build one ``mean_climate_driver.py`` command per supported variable. +#. Execute those commands in serial or parallel mode. +#. Use ``MeanClimateMetricsCollector`` to move figures, metrics JSON files, and + NetCDF diagnostics into the ``results_dir`` structure expected by later + stages. + +Variability modes +----------------- + +#. Parse arguments and run the shared setup. +#. Build one ``variability_modes_driver.py`` command per requested mode. +#. Execute those commands in serial or parallel mode. +#. Reorganize graphics, metrics, and diagnostics into the structured output + tree used by the viewer layer. + +Synthetic plots and viewer +-------------------------- + +#. Parse ``SyntheticPlotsParameters`` in ``pcmdi_synthetic_plots.py``. +#. Read ``synthetic_metrics_list.json`` and generate summary figures through + ``SyntheticMetricsPlotter``. +#. Build the viewer configuration with helper functions from + ``zppy_interfaces.pcmdi_diags.viewer``. +#. Write the methodology, data, and viewer HTML pages under + ``web_dir/results_dir/viewer``. diff --git a/docs/source/global_time_series.rst b/docs/source/global_time_series.rst new file mode 100644 index 0000000..0e663fe --- /dev/null +++ b/docs/source/global_time_series.rst @@ -0,0 +1,227 @@ +****************** +global_time_series +****************** + +Overview +======== + +The ``global_time_series`` interface is exposed through the +``zi-global-time-series`` command. It generates coupled global time-series +products from post-processed E3SM component outputs and can also prepare the +additional ocean time-series files needed by the classic eight-panel plots. + +This page focuses on how to run the command and interpret its outputs. For the +implementation path through the Python modules, see +:doc:`developer_global_time_series`. + +Step-by-step run flow +===================== + +#. Parse command-line arguments. +#. Convert boolean-like strings such as ``True`` and ``False`` to Python + booleans. +#. Convert ``regions`` to normalized region codes: + ``glb``/``global`` -> ``glb``, ``n``/``north``/``northern`` -> ``n``, and + ``s``/``south``/``southern`` -> ``s``. +#. Convert comma-separated plot selections into lists. Passing ``None`` means + "no plots for this group". +#. Validate the request: + + * at least one plot must be requested; + * if ocean plots are requested through the classic plot set and + ``use_ocn=True``, ``moc_file`` must be provided. +#. If ``use_ocn=True`` and the classic plot selection includes ocean-dependent + plots, create ocean time-series files in ``results_dir/ocn/glb/ts/monthly``. +#. Run the coupled global plotting driver, which writes figures and HTML output + into ``results_dir``. + +Output modes +============ + +The command supports two presentation modes controlled by ``make_viewer``. + +* ``make_viewer=True`` generates viewer-oriented HTML output. Classic plots are + still produced when ``plots_original`` is non-empty, and component plots are + arranged into viewer pages by variable group and region. +* ``make_viewer=False`` generates a simpler file-list style output. Classic + plots remain available, and component plots are emitted as cumulative PDFs + instead of viewer pages. + +Classic plot names +================== + +``plots_original`` accepts the following named plot requests: + +* ``net_toa_flux_restom`` -> ``RESTOM`` +* ``net_atm_energy_imbalance`` -> ``RESTOM`` and ``RESSURF`` +* ``global_surface_air_temperature`` -> ``TREFHT`` +* ``toa_radiation`` -> ``FSNTOA`` and ``FLUT`` +* ``net_atm_water_imbalance`` -> ``PRECC``, ``PRECL``, and ``QFLX`` +* ``change_ohc`` -> ocean heat content plot +* ``max_moc`` -> maximum meridional overturning circulation plot +* ``change_sea_level`` -> sea-level change plot + +The three ocean-related plot names (``change_ohc``, ``max_moc``, +``change_sea_level``) above require ocean support data. + +Component plot requests +======================= + +``plots_atm``, ``plots_ice``, and ``plots_ocn`` accept comma-separated variable +names. These are treated as generic variable names and plotted directly. + +``plots_lnd`` is special: + +* If set to ``None``, no land plots are generated. +* If set to ``all``, every variable listed in + `zppy_interfaces/global_time_series/zppy_land_fields.csv`_ is included. (NOTE: this shows the catalogue as of the latest ``main`` branch, which may differ from the version in the release you are using.) +* Otherwise, each requested land variable must match a row in + ``zppy_interfaces/global_time_series/zppy_land_fields.csv``. + +The CSV file also defines whether each land variable is treated as a land-area +average or a land total, along with the scale factor, units, plotting group, +and descriptive long name used by the generated pages. + +Boolean parsing note +==================== + +``make_viewer`` and ``use_ocn`` are parsed with a strict string check: only +``true`` (case-insensitive) is treated as true. Any other value is treated as +false. + +Parameter reference +=================== + +The following parameters are accepted by ``zi-global-time-series``. + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--make_viewer`` + - No + - ``False`` + - Selects viewer-oriented output when true; otherwise the command writes a + simpler results listing and component PDFs. + * - ``--case_dir`` + - Yes + - N/A + - Case directory used to locate existing post-processing output, including + cached MOC time-series files for ocean-enabled classic plots. + * - ``--experiment_name`` + - Yes + - N/A + - Label used for the experiment in generated plot metadata. + * - ``--figstr`` + - No + - Empty string + - Prefix used in generated figure file names. + * - ``--color`` + - No + - ``Blue`` + - Plot color assigned to the experiment in generated figures. + * - ``--ts_num_years`` + - No + - ``5`` + - Width, in years, of each time-series chunk expected in the input + directory layout. + * - ``--results_dir`` + - Yes + - N/A + - Base output directory for figures, HTML, and any generated ocean + time-series files. + * - ``--regions`` + - No + - ``glb,n,s`` + - Comma-separated set of region codes or aliases. Valid aliases normalize + to ``glb``, ``n``, and ``s``. + * - ``--start_yr`` + - Yes + - N/A + - First simulation year to include. + * - ``--end_yr`` + - Yes + - N/A + - Last simulation year to include. + * - ``--use_ocn`` + - No + - ``False`` + - Enables ocean support data generation for the classic ocean-related + plots. + * - ``--input`` + - Yes when ``use_ocn=True`` + - N/A + - Base directory containing raw MPAS-O input used to construct ocean + time-series files. + * - ``--input_subdir`` + - No + - ``archive/ocn/hist`` + - Subdirectory below ``input`` that contains raw MPAS-O monthly history + files. + * - ``--moc_file`` + - Yes when ocean-enabled classic plots are requested + - ``None`` + - Name of the cached MOC file copied from ``case_dir`` into the generated + ocean results directory. + * - ``--plots_original`` + - No + - ``net_toa_flux_restom,global_surface_air_temperature,toa_radiation,`` + ``net_atm_energy_imbalance,change_ohc,max_moc,change_sea_level,`` + ``net_atm_water_imbalance`` + - Comma-separated list of named classic plot requests. Use ``None`` to + disable the classic plots entirely. + * - ``--plots_atm`` + - No + - ``None`` + - Comma-separated list of atmosphere variables to plot. + * - ``--plots_ice`` + - No + - ``None`` + - Comma-separated list of sea-ice variables to plot. + * - ``--plots_lnd`` + - No + - ``None`` + - Comma-separated list of land variables from ``zppy_land_fields.csv``, or + ``all`` to include the full land catalogue. + * - ``--plots_ocn`` + - No + - ``None`` + - Comma-separated list of ocean variables to plot as component plots. + * - ``--ncols`` + - No + - ``2`` + - Number of columns used when building PDF output. + * - ``--nrows`` + - No + - ``4`` + - Number of rows used when building PDF output. + +Important path conventions +========================== + +The implementation expects the usual zppy post-processing layout inside +``case_dir``. For example, component time-series inputs are read from paths such +as ``post/atm/glb/ts/monthly/yr/`` and +``post/lnd/glb/ts/monthly/yr/``. Ocean support files are written +under ``results_dir/ocn/glb/ts/monthly/yr/``. + +Land variable catalogue +======================= + +The full land-variable list is packaged with the repository: + +* `zppy_interfaces/global_time_series/zppy_land_fields.csv`_ (NOTE: this shows the catalogue as of the latest ``main`` branch, which may differ from the version in the release you are using.) + +That file is the authoritative source for: + +* the accepted ``plots_lnd`` variable names, +* whether each variable is treated as an average or total, +* unit conversions, +* variable grouping on viewer pages, and +* the long descriptions shown to users. + +.. _zppy_interfaces/global_time_series/zppy_land_fields.csv: https://github.com/E3SM-Project/zppy-interfaces/blob/main/zppy_interfaces/global_time_series/zppy_land_fields.csv diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..f83e222 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,55 @@ +.. _index-label: + +****************************** +zppy-interfaces documentation +****************************** + +What is zppy-interfaces? +======================== + +``zppy-interfaces`` adds workflow-specific post-processing utilities as stand-alone functionality or as an intermediate step on top of +external diagnostics packages. The repository +currently provides two main interfaces: + +* ``global_time_series`` for generating coupled global time-series plots, + including land-variable handling driven by a packaged CSV catalogue. +* ``pcmdi_diags`` for preparing inputs, launching PCMDI Metrics diagnostics, + reorganizing outputs, and building viewer-ready summary products. + +The package exposes command-line entry points that are meant to be called from +automation such as ``zppy``. Each entry point accepts explicit command-line +arguments, performs limited validation, and writes outputs into predictable +directory layouts. + +How to use this documentation +============================= + +This documentation is organized by audience: + +* :doc:`user_guide` groups the interface guides for people running the command + line tools. +* :doc:`developer_guide` groups the implementation guides for people working on + the repository internals. + +Processing model +================ + +At a high level, the package follows the same pattern across its command-line +tools: + +#. Parse explicit command-line arguments. +#. Normalize selected values such as booleans, comma-separated lists, and + region names. +#. Assemble input and output paths from the provided workflow context. +#. Optionally derive or reorganize supporting files needed by downstream tools. +#. Invoke the plotting or diagnostics logic. +#. Write figures, metrics, or viewer assets into workflow-specific results + directories. + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + self + user_guide + developer_guide diff --git a/docs/source/pcmdi_diags.rst b/docs/source/pcmdi_diags.rst new file mode 100644 index 0000000..bbc26be --- /dev/null +++ b/docs/source/pcmdi_diags.rst @@ -0,0 +1,516 @@ +*********** +pcmdi_diags +*********** + +.. warning:: + + ``pcmdi_diags`` is currently in a beta release and is undergoing active development. + +Overview +======== + +The ``pcmdi_diags`` package provides multiple command-line entry points that +prepare inputs for PCMDI Metrics, launch diagnostics, reorganize outputs, and +assemble summary viewers. + +This page focuses on how to run those commands and what each one produces. For +the implementation details behind the shared setup and the command modules, see +:doc:`developer_pcmdi_diags`. + +The entry points defined by this repository are: + +* ``zi-pcmdi-link-observation`` +* ``zi-pcmdi-mean-climate`` +* ``zi-pcmdi-variability-modes`` +* ``zi-pcmdi-enso`` +* ``zi-pcmdi-synthetic-plots`` + +Shared workflow +=============== + +``zi-pcmdi-mean-climate``, ``zi-pcmdi-variability-modes``, and the unfinished +``zi-pcmdi-enso`` command all use ``zppy_interfaces.pcmdi_diags.pcmdi_setup`` to +perform the same initial preparation work. + +Step-by-step shared setup +------------------------- + +#. Parse the shared core parameters. +#. Decide whether multiprocessing is active. If ``num_workers < 2``, the code + disables multiprocessing even if ``multiprocessing=True`` was provided. +#. Determine the reference data set: + + * ``run_type=model_vs_obs`` uses ``obs_sets``; + * ``run_type=model_vs_model`` uses ``model_name_ref`` and ``tableID_ref``. +#. Check whether any requested derived variables need to be created. The setup + logic can derive ``rstcre`` from ``rsutcs`` and ``rsut``, and ``rltcre`` + from ``rlutcs`` and ``rlut``. +#. Build test and reference catalogue JSON files inside a working + ``pcmdi_diags`` directory. +#. Optionally generate land-sea mask files under ``fixed/`` when + ``generate_sftlf`` is enabled. +#. Build path templates for the downstream diagnostics and return them to the + calling command. + +Shared core parameters +---------------------- + +These parameters are accepted by the mean-climate, variability-modes, and ENSO +commands. + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--num_workers`` + - Yes + - N/A + - Maximum number of workers used when multiprocessing is enabled. + * - ``--multiprocessing`` + - Yes + - N/A + - Boolean-like string controlling parallel execution. The setup code still + disables parallel execution when ``num_workers`` is less than 2. + * - ``--subsection`` + - Yes + - N/A + - Label used in generated catalogue file names such as + ``__catalogue.json``. + * - ``--climo_ts_dir_primary`` + - Yes + - N/A + - Directory containing the primary model climatology or time-series files. + * - ``--climo_ts_dir_ref`` + - Yes + - N/A + - Directory containing the reference model or observational files. + * - ``--model_name`` + - Yes + - N/A + - Primary model identifier. The code expects four dot-separated parts: + ``mip.exp.model.realm``. + * - ``--model_tableID`` + - Yes + - N/A + - Table identifier paired with ``model_name`` for output naming. + * - ``--figure_format`` + - Yes + - N/A + - Graphics file format expected from the underlying diagnostics. + * - ``--run_type`` + - Yes + - N/A + - Must be ``model_vs_obs`` or ``model_vs_model``. + * - ``--obs_sets`` + - Required for ``model_vs_obs`` + - N/A + - Comma-separated observation set identifiers used when the run compares a + model against observations. + * - ``--model_name_ref`` + - Required for ``model_vs_model`` + - N/A + - Reference model name used when the run compares one model against + another. + * - ``--vars`` + - Yes + - N/A + - Comma-separated variable list. Base variable names are extracted by + splitting on ``_`` and ``-`` where needed. + * - ``--tableID_ref`` + - Required for ``model_vs_model`` + - N/A + - Table identifier paired with ``model_name_ref``. + * - ``--generate_sftlf`` + - Yes + - N/A + - Boolean-like string controlling land-sea mask generation. The code + treats ``true``, ``y``, and ``yes`` as enabled. + * - ``--case_id`` + - Yes + - N/A + - Case identifier appended to generated commands and output file names. + * - ``--results_dir`` + - Yes + - N/A + - Base directory where organized outputs are written. + * - ``--debug`` + - No + - N/A + - Enables debug logging when set to ``true``. + +Boolean parsing notes +--------------------- + +The PCMDI-related commands do not all parse booleans the same way: + +* ``multiprocessing`` and most ``debug`` flags use only ``true`` + (case-insensitive) as the true value. +* ``generate_sftlf`` treats ``true``, ``y``, and ``yes`` as true. +* ``zi-pcmdi-synthetic-plots`` accepts a broader set of truthy and falsy values + such as ``true``, ``1``, ``yes``, ``on``, ``false``, ``0``, and ``off`` for + its viewer toggles and ``save_all_data``. + +zi-pcmdi-link-observation +========================= + +Purpose +------- + +``zi-pcmdi-link-observation`` prepares observational time-series files for +PCMDI workflows. It looks up observation aliases, creates symlinks into a +temporary observation directory, and can derive ``rstcre`` or ``rltcre`` when +their source variables exist. + +Step-by-step +------------ + +#. Parse the command-line arguments. +#. Resolve each requested variable to an observation alias using + ``reference_alias.json``. +#. Search the observation source tree for the matching NetCDF file. +#. Symlink the matching file into ``obstmp_dir`` or write a renamed copy when + the observational variable must be mapped from a CMIP-style alias. +#. Derive ``rstcre`` and ``rltcre`` if those variables were requested and are + not already present. + +Parameter reference +------------------- + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--model_name_ref`` + - Yes + - N/A + - Reference model name used to build linked observation file names. + * - ``--tableID_ref`` + - Yes + - N/A + - Reference table identifier paired with ``model_name_ref``. + * - ``--vars`` + - Yes + - N/A + - Comma-separated variables to link or derive. + * - ``--obs_sets`` + - Yes + - N/A + - Observation set identifiers. When multiple sets are supplied and the + count matches ``vars``, each variable uses its corresponding set; + otherwise the first set is reused for all variables. + * - ``--obs_ts`` + - Yes + - N/A + - Root directory containing the source observation time series. + * - ``--obstmp_dir`` + - Yes + - N/A + - Output directory where symlinks or renamed files are written. + * - ``--debug`` + - No + - N/A + - Enables debug logging when set to ``true``. + +zi-pcmdi-mean-climate +===================== + +Purpose +------- + +``zi-pcmdi-mean-climate`` prepares PCMDI mean-climate runs, executes the +underlying ``mean_climate_driver.py`` commands, and reorganizes the resulting +figures, metrics JSON files, and NetCDF diagnostics into the ``results_dir`` +layout expected by downstream viewers. + +Step-by-step +------------ + +#. Run the shared setup described above. +#. Save a ``regions.json`` file that maps each requested variable key to the + requested region list. +#. Generate one ``mean_climate_driver.py`` command per requested variable that + is present in the reference catalogue. +#. Run those commands in parallel or serial mode. +#. Reorganize figures into grouped seasonal directories. +#. Reorganize JSON metrics and NetCDF diagnostics into ``metrics_data``. + +Additional parameters +--------------------- + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--regions`` + - Yes + - N/A + - Comma-separated list of regions used both for command preparation and + for organizing the collected outputs. + +zi-pcmdi-variability-modes +========================== + +Purpose +------- + +``zi-pcmdi-variability-modes`` prepares variability-mode diagnostics, launches +``variability_modes_driver.py`` commands, and then collects graphics, metrics, +and diagnostic files into a structured output tree. + +Step-by-step +------------ + +#. Run the shared setup described above. +#. Read the observation metadata for the requested ``vars`` value. +#. Derive reference years and file paths from the observation catalogue. +#. Generate one ``variability_modes_driver.py`` command per requested mode. +#. Run those commands in parallel or serial mode. +#. Collect graphics into grouped seasonal directories. +#. Collect JSON metrics and NetCDF diagnostics into ``metrics_data``. + +Additional parameters +--------------------- + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--var_modes`` + - Yes + - N/A + - Comma-separated list of variability modes to process. Specific modes + such as ``NPO``, ``NPGO``, ``PSA1``, and ``PSA2`` automatically select a + higher EOF number than the default of 1. + +zi-pcmdi-enso +============= + +.. warning:: + + ``pcmdi_diags`` is currently in a beta release and is undergoing active development. ``zi-pcmdi-enso`` is not yet supported and should be treated as unavailable. + +Purpose +------- + +``zi-pcmdi-enso`` contains argument parsing, observation preparation helpers, +driver command generation, and output validation logic for ENSO diagnostics. +However, the current ``main()`` implementation exits immediately with an error +message stating that the command is not yet supported. + +Current status +-------------- + +The CLI is present and documented here so callers can understand the intended +interface, but the command should currently be treated as unavailable. + +Additional parameters +--------------------- + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--enso_groups`` + - Yes + - N/A + - Comma-separated ENSO metric group names intended for + ``enso_driver.py --metricsCollection``. + +zi-pcmdi-synthetic-plots +======================== + +Purpose +------- + +``zi-pcmdi-synthetic-plots`` generates summary metric plots and builds viewer +pages that combine mean-climate, variability, and ENSO outputs into a single +web presentation. + +Step-by-step +------------ + +#. Parse the command-line arguments into ``SyntheticPlotsParameters``. +#. Build the expected input path under ``www/put_model_here/pcmdi_diags``. +#. Read ``synthetic_metrics_list.json``. +#. Generate the summary plots through ``SyntheticMetricsPlotter``. +#. Assemble viewer configuration and supporting assets. +#. Write methodology, data, and viewer HTML pages under + ``web_dir/results_dir/viewer``. + +Parameter reference +------------------- + +.. list-table:: + :header-rows: 1 + :widths: 18 10 14 58 + + * - Parameter + - Required + - Default + - Description + * - ``--synthetic_sets`` + - Yes + - N/A + - Synthetic plot set selection accepted by the CLI entry point. + * - ``--figure_format`` + - Yes + - N/A + - Graphics format used for generated summary plots. + * - ``--www`` + - Yes + - N/A + - Base web directory that contains ``put_model_here/pcmdi_diags`` inputs. + * - ``--results_dir`` + - Yes + - N/A + - Results subdirectory below both the input and output web locations. + * - ``--case`` + - Yes + - N/A + - Case name shown in generated plots and viewer pages. + * - ``--model_name`` + - Yes + - N/A + - Model name shown in generated plots and viewer pages. + * - ``--model_tableID`` + - Yes + - N/A + - Table identifier shown in generated plots and viewer pages. + * - ``--web_dir`` + - Yes + - N/A + - Output web root where the viewer and summary products are written. + * - ``--clim_viewer`` + - Yes + - N/A + - Boolean controlling whether climatology summary products are included. + * - ``--clim_vars`` + - Yes + - N/A + - Comma-separated climatology variables used in the viewer. + * - ``--clim_years`` + - Yes + - N/A + - Year range label for climatology products. + * - ``--clim_regions`` + - Yes + - N/A + - Comma-separated climatology region list used in the viewer. + * - ``--cmip_clim_dir`` + - Yes + - N/A + - Directory containing CMIP climatology comparison data. + * - ``--cmip_clim_set`` + - Yes + - N/A + - Name of the CMIP climatology set used by the viewer. + * - ``--mova_viewer`` + - Yes + - N/A + - Boolean controlling whether atmospheric variability summaries are + included. + * - ``--mova_modes`` + - Yes + - N/A + - Comma-separated atmospheric variability modes used in summary products. + * - ``--mova_vars`` + - Yes + - N/A + - Comma-separated atmospheric variability variables accepted by the CLI. + * - ``--mova_years`` + - Yes + - N/A + - Year range label for atmospheric variability products. + * - ``--movc_viewer`` + - Yes + - N/A + - Boolean controlling whether coupled variability summaries are included. + * - ``--movc_modes`` + - Yes + - N/A + - Comma-separated coupled variability modes used in summary products. + * - ``--movc_vars`` + - Yes + - N/A + - Comma-separated coupled variability variables accepted by the CLI. + * - ``--movc_years`` + - Yes + - N/A + - Year range label for coupled variability products. + * - ``--cmip_movs_dir`` + - Yes + - N/A + - Directory containing CMIP variability comparison data. + * - ``--cmip_movs_set`` + - Yes + - N/A + - Name of the CMIP variability set used by the viewer. + * - ``--enso_viewer`` + - Yes + - N/A + - Boolean controlling whether ENSO summaries are included. + * - ``--enso_vars`` + - Yes + - N/A + - Comma-separated ENSO variables accepted by the CLI. + * - ``--enso_years`` + - Yes + - N/A + - Year range label for ENSO products. + * - ``--cmip_enso_dir`` + - Yes + - N/A + - Directory containing CMIP ENSO comparison data. + * - ``--cmip_enso_set`` + - Yes + - N/A + - Name of the CMIP ENSO set used by the viewer. + * - ``--pcmdi_webtitle`` + - Yes + - N/A + - Title shown on generated viewer pages. + * - ``--pcmdi_version`` + - Yes + - N/A + - Version label shown on generated viewer pages. + * - ``--run_type`` + - Yes + - N/A + - Run-type label converted to a subtitle in the viewer. + * - ``--pcmdi_external_prefix`` + - Yes + - N/A + - Base directory used to find shared PCMDI web assets and data. + * - ``--pcmdi_viewer_template`` + - Yes + - N/A + - Template directory below ``pcmdi_external_prefix`` that contains viewer + assets such as ``e3sm_pmp_logo.png``. + * - ``--save_all_data`` + - Yes + - N/A + - Boolean controlling whether the plotter saves all underlying data. + * - ``--debug`` + - No + - N/A + - Enables debug logging when set to ``true``. diff --git a/docs/source/user_guide.rst b/docs/source/user_guide.rst new file mode 100644 index 0000000..e04dc63 --- /dev/null +++ b/docs/source/user_guide.rst @@ -0,0 +1,12 @@ +********** +User Guide +********** + +Use these pages when you want to run the packaged command-line interfaces, +understand the required parameters, and find the expected output layout. + +.. toctree:: + :maxdepth: 1 + + global_time_series + pcmdi_diags