Draft code to enable mpas ocean/seaice time series processing with re…

zppy/defaults/default.ini

@@ -151,6 +151,15 @@ job_nbr = integer(default=0)
 tpd = integer(default=1)
 # Model component having generated input files (eam, eamxx, elm, mosart, ...)
 input_component = string(default="")
+# MPAS calendar used to construct a CF-style numeric time coordinate
+# from timeMonthly_avg_daysSinceStartOfSim when native MPAS time-series
+# files do not contain a standard time variable.
+mpas_calendar = string(default="noleap")
+# MPAS simulation start time used as the reference time for constructing
+# a CF-style numeric time coordinate:
+# time:units = "days since <mpas_start_time>"
+# This should match the MPAS/E3SM config_start_time when possible.
+mpas_start_time = string(default="0001-01-01 00:00:00")
 
   [[__many__]]
   area_nm = string(default=None)
@@ -159,6 +168,8 @@ input_component = string(default="")
   job_nbr = integer(default=None)
   tpd = integer(default=None)
   input_component = string(default=None)
+  mpas_calendar = string(default=None)
+  mpas_start_time = string(default=None)
 
 [e3sm_to_cmip]
 # Metadata json file

zppy/templates/ts.bash

@@ -54,7 +54,7 @@ vars={{ vars }}
 # https://unix.stackexchange.com/questions/237297/the-fastest-way-to-remove-a-string-in-a-variable
 # https://stackoverflow.com/questions/26457052/remove-a-substring-from-a-bash-variable
 # Remove U, since it is a 3D variable and thus will not work with rgn_avg
-vars=${vars//,U}
+# vars=${vars//,U}
 {%- else %}
 vars={{ vars }}
 {%- endif %}
@@ -66,7 +66,7 @@ if grep -q "*" input.txt; then
   echo 'ERROR (1)' > {{ prefix }}.status
   exit 1
 fi
-# Generate time series files
+# Generate time series files.
 # If the user-defined parameter "vars" is "", then ${vars}, defined above, will be too.
 cat input.txt | run_nco ncclimo \
 -c {{ case }} \
@@ -86,32 +86,284 @@ cat input.txt | run_nco ncclimo \
 --yr_srt={{ yr_start }} \
 --yr_end={{ yr_end }} \
 --ypf={{ ypf }} \
-{% if mapping_file == '' -%}
+{%- if mapping_file == '' %}
 -o output \
-{%- elif mapping_file == 'glb' -%}
+{%- elif mapping_file == 'glb' %}
 -o output \
 --rgn_avg \
 --area={{ area_nm }} \
-{%- else -%}
+{%- else %}
+{%- if prc_typ in ['mpasocean', 'mpasseaice'] %}
+-o trash \
+{%- else %}
 --map={{ mapping_file }} \
 -o trash \
 -O output \
 {%- endif %}
+{%- endif %}
 {%- if frequency != 'monthly' %}
 --clm_md=hfs \
 --dpf={{ dpf }} \
 --tpd={{ tpd }} \
 {%- endif %}
 --prc_typ={{ prc_typ }}
 
-
-
+# Check ncclimo status before any optional separate regridding.
 if [ $? != 0 ]; then
   cd {{ scriptDir }}
   echo 'ERROR (2)' > {{ prefix }}.status
   exit 2
 fi
 
+{%- if mapping_file != '' and mapping_file != 'glb' and prc_typ in ['mpasocean', 'mpasseaice'] %}
+
+# Regrid MPAS-Ocean and MPAS-Seaice files separately.
+#
+# For MPAS components, generate native-grid time-series files first, then
+# regrid them with ncremap. This lets us restore MPAS-native time metadata,
+# construct a CF-style numeric time coordinate, and repair missing_value
+# attributes after regridding if needed.
+#
+# For MPAS-Seaice, also provide valid spatial-only SGS helper variables for
+# regridding. MPAS-Seaice monthly files may include time-dependent SGS
+# diagnostics such as:
+#
+#   timeMonthly_avg_iceAreaCell(Time,nCells)
+#
+# This variable should remain in the final time-series output. However,
+# NCO's regridder requires sgs_frc_nm and sgs_msk_nm to be spatial-only
+# fields matching the horizontal source grid size, i.e., nCells. Therefore,
+# create temporary spatial-only helper variables with separate names and use
+# them only for regridding.
+mkdir -p output
+
+mpas_calendar="{{ mpas_calendar }}"
+mpas_start_time="{{ mpas_start_time }}"
+mpas_start_time="${mpas_start_time/_/ }"
+
+for f in trash/*.nc ; do
+  bname=$(basename "${f}")
+  tmp_f="tmp_regrid_${bname}"
+  tmp_frc="tmp_sgs_frc_${bname}"
+  tmp_msk="tmp_sgs_msk_${bname}"
+  out_f="output/${bname}"
+
+  cp "${f}" "${tmp_f}"
+
+  # Detect whether the time dimension is named Time or time.
+  # MPAS native ncclimo files often have a Time dimension but no
+  # coordinate variable named Time/time.
+  if run_nco ncks -m "${tmp_f}" | grep -qE "^[[:space:]]*Time[[:space:]]*=|\(Time[,)]|, Time[,)]"; then
+    time_dim="Time"
+  elif run_nco ncks -m "${tmp_f}" | grep -qE "^[[:space:]]*time[[:space:]]*=|\(time[,)]|, time[,)]"; then
+    time_dim="time"
+  else
+    echo "ERROR: Cannot find Time/time dimension in ${tmp_f}"
+    cd {{ scriptDir }}
+    echo 'ERROR (3)' > {{ prefix }}.status
+    exit 3
+  fi
+
+  # Infer the main variable name from the output filename.
+  # Example:
+  #   timeMonthly_avg_seaSurfaceTemperature_000101_000512.nc
+  # becomes:
+  #   timeMonthly_avg_seaSurfaceTemperature
+  main_var=$(echo "${bname}" | sed -E 's/_[0-9]{6}_[0-9]{6}\.nc$//')
+
+  use_sgs_helpers="false"
+
+  {%- if prc_typ == 'mpasseaice' %}
+
+  # Construct explicit static SGS helpers for MPAS-Seaice.
+  #
+  # ncremap -P mpasseaice automatically tries to use
+  # timeMonthly_avg_iceAreaCell as sgs_frc_nm. However, that variable is
+  # time-dependent, with shape Time x nCells, while NCO requires sgs_frc_nm
+  # to be spatial-only, with shape nCells.
+  #
+  # Therefore, whenever timeMonthly_avg_iceAreaCell is available, construct:
+  #
+  #   sgs_frc_static(nCells) = time mean of timeMonthly_avg_iceAreaCell
+  #   sgs_msk_static(nCells) = 1 where sgs_frc_static > 0, else 0
+  #
+  # Do not fall back to implicit ncremap SGS handling when this helper can be
+  # constructed, because the implicit path may reuse the time-dependent field.
+  if run_nco ncks -m "${tmp_f}" | grep -q "timeMonthly_avg_iceAreaCell"; then
+
+    # Construct spatial-only SGS fraction helper.
+    run_nco ncwa -O -a "${time_dim}" -y avg \
+      -v timeMonthly_avg_iceAreaCell \
+      "${tmp_f}" "${tmp_frc}"
+
+    run_nco ncrename -O \
+      -v timeMonthly_avg_iceAreaCell,sgs_frc_static \
+      "${tmp_frc}"
+
+    # Construct a spatial-only binary SGS mask from the static fraction.
+    # This avoids relying on timeMonthly_avg_icePresentIntMask, which may
+    # not be present in every split file.
+    run_nco ncap2 -O \
+      -s 'sgs_msk_static=(sgs_frc_static > 0.0)' \
+      "${tmp_frc}" "${tmp_msk}"
+
+    run_nco ncks -O -x -v sgs_frc_static "${tmp_msk}" "${tmp_msk}.only_msk"
+    mv "${tmp_msk}.only_msk" "${tmp_msk}"
+
+    # Append helper variables to the temporary file.
+    # This does not remove or overwrite the original time-dependent variables.
+    run_nco ncks -A "${tmp_frc}" "${tmp_f}"
+    run_nco ncks -A "${tmp_msk}" "${tmp_f}"
+
+    use_sgs_helpers="true"
+  else
+    echo "ERROR: timeMonthly_avg_iceAreaCell is missing in ${tmp_f}; cannot construct static SGS helpers for MPAS-Seaice."
+    cd {{ scriptDir }}
+    echo 'ERROR (3)' > {{ prefix }}.status
+    exit 3
+  fi
+
+  {%- endif %}
+
+  if [ "${use_sgs_helpers}" == "true" ]; then
+    run_nco ncremap \
+      -P {{ prc_typ }} \
+      --d2f \
+      --no_stagger \
+      -t 1 \
+      --nco_opt='--no_tmp_fl --hdr_pad=10000' \
+      --map={{ mapping_file }} \
+      --sgs_frc=sgs_frc_static \
+      --sgs_msk=sgs_msk_static \
+      --sgs_nrm=1.0 \
+      "${tmp_f}" \
+      "${out_f}"
+  else
+    run_nco ncremap \
+      -P {{ prc_typ }} \
+      --d2f \
+      --no_stagger \
+      -t 1 \
+      --nco_opt='--no_tmp_fl --hdr_pad=10000' \
+      --map={{ mapping_file }} \
+      "${tmp_f}" \
+      "${out_f}"
+  fi
+
+  rc=$?
+
+  # Remove temporary/helper variables from the final regridded output.
+  # Keep timeMonthly_avg_iceAreaCell only when it is the requested main variable.
+  if [ ${rc} == 0 ] && [ "{{ prc_typ }}" == "mpasseaice" ]; then
+    vars_to_remove="sgs_frc_static,sgs_msk_static"
+
+    if [ "${main_var}" != "timeMonthly_avg_iceAreaCell" ]; then
+      vars_to_remove="${vars_to_remove},timeMonthly_avg_iceAreaCell"
+    fi
+
+    # Remove helper variables only if at least one is present in the output.
+    remove_present="false"
+    for rm_var in $(echo "${vars_to_remove}" | tr ',' ' '); do
+      if run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*[^:]+[[:space:]]+${rm_var}(\(|\[)"; then
+        remove_present="true"
+        break
+      fi
+    done
+
+    if [ "${remove_present}" == "true" ]; then
+      run_nco ncks -O -x -v "${vars_to_remove}" "${out_f}" "${out_f}.clean"
+      mv "${out_f}.clean" "${out_f}"
+      rc=$?
+    fi
+  fi
+
+  # Restore MPAS-native time metadata variables if needed.
+  # MPAS files often have a Time dimension and xtime_* variables, but no
+  # standard numeric coordinate variable named Time/time.
+  if [ ${rc} == 0 ]; then
+    for tvar in xtime_startMonthly xtime_endMonthly timeMonthly_avg_daysSinceStartOfSim; do
+      if run_nco ncks -m "${tmp_f}" | grep -q "${tvar}"; then
+        if ! run_nco ncks -m "${out_f}" | grep -q "${tvar}"; then
+          run_nco ncks -A -v "${tvar}" "${tmp_f}" "${out_f}"
+          rc=$?
+          if [ ${rc} != 0 ]; then
+            break
+          fi
+        fi
+      fi
+    done
+  fi
+
+  # Construct a CF-style numeric time coordinate if it is missing.
+  # Use MPAS days-since-start metadata:
+  #
+  #   timeMonthly_avg_daysSinceStartOfSim(Time)
+  #
+  # to create:
+  #
+  #   time(Time)
+  #     units    = "days since ${mpas_start_time}"
+  #     calendar = "${mpas_calendar}"
+  #
+  # This improves compatibility with downstream tools that expect a
+  # standard numeric time coordinate.
+  if [ ${rc} == 0 ]; then
+    if ! run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*(float|double)[[:space:]]+time(\(|\[)"; then
+      if run_nco ncks -m "${tmp_f}" | grep -q "timeMonthly_avg_daysSinceStartOfSim"; then
+        tmp_time="tmp_time_${bname}"
+
+        run_nco ncks -O \
+          -v timeMonthly_avg_daysSinceStartOfSim \
+          "${tmp_f}" "${tmp_time}"
+
+        run_nco ncrename -O \
+          -v timeMonthly_avg_daysSinceStartOfSim,time \
+          "${tmp_time}"
+
+        run_nco ncatted -O \
+          -a units,time,o,c,"days since ${mpas_start_time}" \
+          -a calendar,time,o,c,"${mpas_calendar}" \
+          -a long_name,time,o,c,time \
+          -a standard_name,time,o,c,time \
+          "${tmp_time}"
+
+        run_nco ncks -A -v time "${tmp_time}" "${out_f}"
+        rc=$?
+
+        rm -f "${tmp_time}"
+      fi
+    fi
+  fi
+
+  # Add missing_value metadata only for the main floating-point data variable
+  # if it is missing. Avoid modifying coordinate or integer mask variables.
+  # Do not force-create _FillValue here, because some NetCDF workflows are
+  # sensitive to adding _FillValue after variable creation. Existing _FillValue
+  # attributes are preserved.
+  if [ ${rc} == 0 ]; then
+    if run_nco ncks -m "${out_f}" | grep -qE "^[[:space:]]*(float|double)[[:space:]]+${main_var}(\(|\[)"; then
+      if ! run_nco ncks -m "${out_f}" | grep -A8 -E "^[[:space:]]*(float|double)[[:space:]]+${main_var}(\(|\[)" | grep -q "missing_value"; then
+        run_nco ncatted -O \
+          -a missing_value,"${main_var}",o,f,1.0e36 \
+          "${out_f}"
+        rc=$?
+      fi
+    fi
+  fi
+
+  rm -f "${tmp_f}" "${tmp_frc}" "${tmp_msk}" "${tmp_msk}.only_msk" "${out_f}.clean"
+
+  if [ ${rc} != 0 ]; then
+    echo "ERROR: MPAS component regridding failed for ${bname}"
+    cd {{ scriptDir }}
+    echo 'ERROR (4)' > {{ prefix }}.status
+    exit 4
+  fi
+
+done
+
+{%- endif %}
+
 # Move output ts files to final destination
 {
   dest={{ output }}/post/{{ component }}/{{ grid }}/ts/{{ frequency }}/{{ '%dyr' % (ypf) }}
@@ -120,8 +372,8 @@ fi
 }
 if [ $? != 0 ]; then
   cd {{ scriptDir }}
-  echo 'ERROR (3)' > {{ prefix }}.status
-  exit 3
+  echo 'ERROR (5)' > {{ prefix }}.status
+  exit 5
 fi
 
 # Delete temporary workdir