CMIP7 ScenarioMIP global integration

.github/workflows/ci-main-only.yaml

@@ -4,15 +4,63 @@ on:
   # pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
 
 jobs:
   tests:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-latest" ]
+        # Not running on windows, see https://github.com/openscm/gcages/issues/36
+        # os: [ "ubuntu-latest", "macos-latest", "windows-latest" ]
+        os: [ "ubuntu-latest", "macos-latest" ]
         # Just test on a sensible version
-        python-version: [ "3.12" ]
+        # Use 3.11 while cicero-scm uses distutils in openscm-runner,
+        # see https://github.com/openscm/openscm-runner/blob/261bd177b86a708b0080eed245b6511ebb0051bb/src/openscm_runner/adapters/ciceroscm_adapter/ciceroscm_wrapper.py#L10
+        python-version: [ "3.11" ]
+    runs-on: "${{ matrix.os }}"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          uv-dependency-install-flags: "--all-extras --group tests"
+      - name: Set DYLD path
+        if: matrix.os == 'macos-latest'
+        run: |
+          # # Useful if the path appears to have changed
+          # find / -name 'libgfortran.5.dylib' 2>/dev/null
+          find /opt/homebrew -name libgfortran.5.dylib 2>/dev/null | head -1
+          echo "DYLD_LIBRARY_PATH=$(dirname $(find /opt/homebrew -name libgfortran.5.dylib 2>/dev/null | head -1))" >> $GITHUB_ENV
+      - name: Try to run MAGICC
+        if: matrix.os == 'windows-latest'
+        run: |
+          # This is the thing I can't figure out, even just getting the version to appear doesn't work
+          # x-ref https://github.com/openscm/gcages/issues/36
+          D:\a\gcages\gcages\tests\regression\ar6\ar6-workflow-inputs\magicc-v7.5.3\bin\magicc.exe --version
+      - name: Run tests that are otherwise skipped (mostly regression tests for specific set ups)
+        run: |
+          uv run pytest -r a -v -s tests -m "skip_ci_default"
+
+  tests-resolution-strategies:
+    strategy:
+      fail-fast: false
+      matrix:
+        # Only test on ubuntu here for now.
+        # We could consider doing this on different platforms too,
+        # although that probably belongs better with the PyPI tests.
+        os: [ "ubuntu-latest" ]
+        # Tests with lowest direct resolution.
+        # We don't do lowest because that is essentially testing
+        # whether downstream dependencies
+        # have set their minimum support dependencies correctly,
+        # which isn't our problem to solve.
+        resolution-strategy: [ "lowest-direct" ]
+        # Only test against the oldest supported python version
+        # because python is itself a direct dependency
+        # (so we're testing against the lowest direct python too).
+        python-version: [ "3.9" ]
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
@@ -24,10 +72,20 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - uses: ./.github/actions/setup
+      - name: Setup uv
+        id: setup-uv
+        uses: astral-sh/setup-uv@v4
         with:
+          version: "0.5.21"
           python-version: ${{ matrix.python-version }}
-          uv-dependency-install-flags: "--all-extras --group tests"
+      - name: Create venv
+        run: |
+          uv venv --seed
+      - name: Install dependencies
+        run: |
+          uv pip install --requirements requirements-only-tests-locked.txt
+          uv pip compile --python ${{ matrix.python-version }} --resolution ${{ matrix.resolution-strategy }} --all-extras pyproject.toml -o requirements-tmp.txt
+          uv pip install --requirements requirements-tmp.txt .
       - name: Run tests
         run: |
-          uv run pytest -r a -v -s tests -m "skip_ci_default"
+          MAGICC_EXECUTABLE_7="tests/regression/ar6/ar6-workflow-inputs/magicc-v7.5.3/bin/magicc" uv run --no-sync pytest -r a -v -s tests -m "skip_ci_default"

.github/workflows/ci.yaml

@@ -74,6 +74,14 @@ jobs:
         os: [ "ubuntu-latest" ]
         # Test against all security and bugfix versions: https://devguide.python.org/versions/
         python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
+        include:
+          # Include other OS for latest Python
+          # because these seem to be the flakiest from experience
+          # so are worth the extra testing
+          - os: "windows-latest"
+            python-version: "3.13"
+          - os: "macos-latest"
+            python-version: "3.13"
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
@@ -149,7 +157,7 @@ jobs:
           uv pip install --requirements requirements-tmp.txt .
       - name: Run tests
         run: |
-          MAGICC_EXECUTABLE_7="tests/regression/ar6/ar6-workflow-inputs/magicc-v7.5.3/bin/magicc" uv run --no-sync pytest tests -r a -v
+          uv run --no-sync pytest tests -r a -v -m "not skip_ci_default"
 
   tests-without-extras:
     # Run the tests without installing extras.
@@ -233,7 +241,10 @@ jobs:
         # Required for building the locked version of the package
         # with pdm-build-locked (see pyproject.toml)
         run: |
-          pdm lock --group :all --strategy inherit_metadata
+          pdm lock --group :all --strategy inherit_metadata --python=">=3.10,<3.12"
+          pdm lock --group :all  --python=">=3.12,<3.13" --append
+          pdm lock --group :all  --python=">=3.13" --append
+          pdm lock --group :all  --python=">=3.9,<3.10" --append
       - name: Build package
         run: |
           uv build

.github/workflows/deploy.yaml

@@ -42,7 +42,10 @@ jobs:
         # Required for building the locked version of the package
         # with pdm-build-locked (see pyproject.toml)
         run: |
-          pdm lock --group :all --strategy inherit_metadata
+          pdm lock --group :all --strategy inherit_metadata --python=">=3.10,<3.12"
+          pdm lock --group :all  --python=">=3.12,<3.13" --append
+          pdm lock --group :all  --python=">=3.13" --append
+          pdm lock --group :all  --python=">=3.9,<3.10" --append
       - name: Publish to PyPI
         run: |
           uv build

.github/workflows/install-pypi.yaml

@@ -85,3 +85,39 @@ jobs:
       run: |
         # Can't run doctests here because the paths are different.
         pytest tests -r a -vv tests
+    - name: Install all AR6 dependencies application
+      if: contains(matrix.install-target, 'locked')
+      run: |
+        pip install 'gcages[ar6-locked]'
+    - name: Install all AR6 dependencies library
+      if: "!contains(matrix.install-target, 'locked')"
+      run: |
+        pip install 'gcages[ar6]'
+    - name: Set MAGICC executable environment variable
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        echo "MAGICC_EXECUTABLE_7=./tests/regression/ar6/ar6-workflow-inputs/magicc-v7.5.3/bin/magicc" >> $GITHUB_ENV
+    - name: Set MAGICC executable environment variable
+      if: matrix.os == 'macos-latest'
+      run: |
+        echo "MAGICC_EXECUTABLE_7=./tests/regression/ar6/ar6-workflow-inputs/magicc-v7.5.3/bin/magicc-darwin-arm64" >> $GITHUB_ENV
+        # # Useful if the path appears to have changed
+        # find / -name 'libgfortran.5.dylib' 2>/dev/null
+        echo "DYLD_LIBRARY_PATH=/opt/homebrew/lib/gcc/12/" >> $GITHUB_ENV
+    - name: Set MAGICC executable environment variable
+      if: matrix.os == 'windows-latest'
+      run: |
+        chcp 65001  # set code page to utf-8
+        echo "MAGICC_EXECUTABLE_7=./tests/regression/ar6/ar6-workflow-inputs/magicc-v7.5.3/bin/magicc.exe" >> $env:GITHUB_ENV
+    - name: Install distutils
+      # Workaround needed while cicero-scm uses distutils in openscm-runner,
+      # see https://github.com/openscm/openscm-runner/blob/261bd177b86a708b0080eed245b6511ebb0051bb/src/openscm_runner/adapters/ciceroscm_adapter/ciceroscm_wrapper.py#L10
+      if: (matrix.python-version == '3.12') || (matrix.python-version == '3.13')
+      run: |
+        pip install setuptools
+    - name: Run AR6 regression tests
+      # I can't figure out how to get MAGICC to run in CI,
+      # see https://github.com/openscm/gcages/issues/36
+      if: matrix.os != 'windows-latest'
+      run: |
+        pytest tests -r a -vv tests/regression/ar6

.github/workflows/release.yaml

@@ -35,7 +35,10 @@ jobs:
         # Required for building the locked version of the package
         # with pdm-build-locked (see pyproject.toml)
         run: |
-          pdm lock --group :all --strategy inherit_metadata
+          pdm lock --group :all --strategy inherit_metadata --python=">=3.10,<3.12"
+          pdm lock --group :all  --python=">=3.12,<3.13" --append
+          pdm lock --group :all  --python=">=3.13" --append
+          pdm lock --group :all  --python=">=3.9,<3.10" --append
       - name: Add version to environment
         run: |
           PROJECT_VERSION=`sed -ne 's/^version = "\([0-9\.a]*\)"/\1/p' pyproject.toml`

.gitignore

@@ -1,3 +1,6 @@
+# Binary files
+*.feather
+
 # MAGICC
 magicc-v7.5.3
 

changelog/45.breaking.md

@@ -0,0 +1,4 @@
+- renamed `harmonise_scenario` to `harmonise_single_scenario` in [gcages.ar6.harmonisation][]
+- renamed `year` to `harmonisation_year` in [gcages.ar6.harmonisation.harmonise_single_scenario][]
+- Renamed `KEY_TESTING_MODEL_SCENARIOS` to `KEY_AR6_TESTING_MODEL_SCENARIOS` in [gcages.testing][]
+- added a required `model_scenarios` argument to [gcages.testing.get_key_testing_model_scenario_parameters][gcages.testing.get_key_testing_model_scenario_parameters]

changelog/45.feature.md

@@ -0,0 +1,2 @@
+Added global-level harmonisation in line with what was used for CMIP7 ScenarioMIP, see [gcages.cmip7_scenariomip.harmonisation][].
+Note that this is still being tweaked so will change in future.

changelog/46.feature.md

@@ -0,0 +1,2 @@
+Added species infilling in line with what was used for CMIP7 ScenarioMIP, see [gcages.cmip7_scenariomip.infilling][].
+Note that this is still being tweaked so will change in future.

docs/changelog.md

@@ -21,6 +21,122 @@ from the examples given in that link.
 
 <!-- towncrier release notes start -->
 
+## gcages v0.14.0 (2026-02-05)
+
+### 🐛 Bug Fixes
+
+- Fixed Biochar and Soil Carbon Management accounting in the CMIP7 ScenarioMIP pre-processing. Soil carbon removal is now subtracted from `AFOLU|Land|Land Use and Land-Use Change` while biochar from `AFOLU|Land|Other. ([#42](https://github.com/openscm/gcages/pull/42))
+
+### 🔧 Trivial/Internal Changes
+
+- [#41](https://github.com/openscm/gcages/pull/41)
+
+
+## gcages v0.13.0 (2025-12-02)
+
+### ⚠️ Breaking Changes
+
+- # Additional CDR split
+
+  Improved CDR granularity adding the `Biochar` and `Soil Carbon Management` new componentes from the Carbon Removal tree. ([#40](https://github.com/openscm/gcages/pull/40))
+
+
+## gcages v0.12.0 (2025-10-10)
+
+### ⚠️ Breaking Changes
+
+- Split out the Carbon Removal variable tree in a way that better facilitates downstream spatial downscaling.
+  This means that we no longer split out `Carbon Removal|Geological Storage|Other Sources` and `Carbon Removal|Geological Storage|Synthetic Fuels`, and have created four new gridding sector components:
+
+  - `Other CDR` (based on the emissions tree)
+  - `Enhanced Weathering`, `Direct Air Capture`, `Ocean` (based on the carbon removal tree)
+
+  ([#39](https://github.com/openscm/gcages/pull/39))
+
+
+## gcages v0.11.0 (2025-07-23)
+
+### ⚠️ Breaking Changes
+
+- Updated minimum supported version of scmdata to v0.18.0. As a result, we could also drop the pin of xarray (as scmdata v0.18.0 fixes xarray compatibility issues, see https://github.com/openscm/scmdata/blob/main/docs/source/changelog.md). ([#38](https://github.com/openscm/gcages/pull/38))
+
+### 🔧 Trivial/Internal Changes
+
+- [#38](https://github.com/openscm/gcages/pull/38)
+
+
+## gcages v0.10.1 (2025-07-17)
+
+### 🐛 Bug Fixes
+
+- Added explicit requirement on importlib-resources for running the AR6 setup as this seems to be needed to make things work on MacOS with Python 3.10 and 3.11 ([#37](https://github.com/openscm/gcages/pull/37))
+
+
+## gcages v0.10.0 (2025-07-16)
+
+### ⚠️ Breaking Changes
+
+- - Renamed the extra "cmip7_scenariomip" to "cmip7scenariomip"
+  - Stopped packaging the tests as part of releases (this significantly reduces the size of the distributions)
+
+  ([#35](https://github.com/openscm/gcages/pull/35))
+
+### 🐛 Bug Fixes
+
+- - Fixed the dependencies required for running the AR6 setup
+  - Made it possible to run the AR6 setup on windows
+
+  ([#35](https://github.com/openscm/gcages/pull/35))
+
+### 🔧 Trivial/Internal Changes
+
+- [#35](https://github.com/openscm/gcages/pull/35)
+
+
+## gcages v0.9.0 (2025-07-16)
+
+### ⚠️ Breaking Changes
+
+- - Changed CMIP7 ScenarioMIP pre-processing. Carbon removal variables are now pre-processed into the Emissions tree, rather than being kept separate as part of the Carbon Removal tree. This greatly simplifies downstream use as all variables are in one tree.
+  - Updated to requiring pandas-openscm>=0.5.1
+
+  ([#30](https://github.com/openscm/gcages/pull/30))
+- Updated minimum numpy version to 1.26.0, the earliest that is not in end-of-life. Fixed the numpy pin for Python 3.13 to >=2.1.0, the first numpy version which supported Python 3.13. ([#34](https://github.com/openscm/gcages/pull/34))
+
+### 🎉 Improvements
+
+- Added unit conversion support to [gcages.harmonisation.common.assert_harmonised][] ([#33](https://github.com/openscm/gcages/pull/33))
+
+### 🔧 Trivial/Internal Changes
+
+- [#30](https://github.com/openscm/gcages/pull/30), [#31](https://github.com/openscm/gcages/pull/31), [#32](https://github.com/openscm/gcages/pull/32), [#34](https://github.com/openscm/gcages/pull/34)
+
+
+## gcages v0.8.0 (2025-07-12)
+
+### ⚠️ Breaking Changes
+
+- Reverted the addition of a CO2 AFOLU gridding sector in CMIP7 ScenarioMIP pre-processing. ([#29](https://github.com/openscm/gcages/pull/29))
+
+
+## gcages v0.7.0 (2025-07-11)
+
+### ⚠️ Breaking Changes
+
+- Managed the removal of subsectors for `Emissions|CO2|AFOLU` tree ([#23](https://github.com/openscm/gcages/pull/23))
+- Introduced the handling of the `Carbon Removal` tree ([#25](https://github.com/openscm/gcages/pull/25))
+
+### 🐛 Bug Fixes
+
+- Moved in `Agriculture` input sectors previously accounted in `Agriculture Waste Burning` ([#18](https://github.com/openscm/gcages/pull/18))
+
+
+## gcages v0.6.1 (2025-07-10)
+
+This release included the changes for the v0.7.0 release,
+but was released with the wrong version number.
+
+
 ## gcages v0.6.0 (2025-05-04)
 
 ### ⚠️ Breaking Changes

docs/how-to-guides/how-to-run-the-ar6-workflow.py

@@ -265,7 +265,7 @@
 # %% [markdown]
 # With this file, we can initialise a harmoniser exactly like that used in AR6.
 
-# %%
+# %% editable=true slideshow={"slide_type": ""}
 harmoniser = AR6Harmoniser.from_ar6_config(
     ar6_historical_emissions_file=AR6_HISTORICAL_EMISSIONS_FILE,
     n_processes=None,  # run serially for this demo
@@ -274,17 +274,17 @@
 # %% [markdown]
 # And harmonise
 
-# %%
+# %% editable=true slideshow={"slide_type": ""}
 harmonised = harmoniser(pre_processed)
 harmonised
 
-# %% [markdown]
+# %% [markdown] editable=true slideshow={"slide_type": ""}
 # You can see the modification to the pathways
 # as a result of the harmonisation in the plot below.
 # In scenarios that have more emissions,
 # the same idea is applied to all variables.
 
-# %%
+# %% editable=true slideshow={"slide_type": ""}
 pdf = (
     pix.concat(
         [
@@ -353,7 +353,7 @@
 # %% [markdown]
 # With the infilling databases, we can initialise our infiller.
 
-# %%
+# %% editable=true slideshow={"slide_type": ""}
 infiller = AR6Infiller.from_ar6_config(
     ar6_infilling_db_file=AR6_INFILLING_DB_FILE,
     ar6_infilling_db_cfcs_file=AR6_INFILLING_DB_CFCS_FILE,
@@ -470,7 +470,7 @@
 if platform.system() == "Darwin":
     if platform.processor() == "arm":
         MAGICC_EXE = MAGICC_EXE_PATH / "magicc-darwin-arm64"
-        os.environ["DYLD_LIBRARY_PATH"] = "/opt/homebrew/opt/gfortran/lib/gcc/current/"
+        os.environ["DYLD_LIBRARY_PATH"] = "/opt/homebrew/lib/gcc/current/"
 
 elif platform.system() == "Linux":
     MAGICC_EXE = MAGICC_EXE_PATH / "magicc"