Unify missing values display

shapash/explainer/smart_plotter.py

@@ -38,6 +38,7 @@
     adjust_title_height,
     compute_digit_number,
     compute_sorted_variables_interactions_list_indices,
+    format_missing_value,
     maximum_difference_sort_value,
     top_contributors,
     truncate_str,
@@ -2345,7 +2346,7 @@ def clustering_by_explainability_plot(
                     for idx, row in df_pred.iterrows():
                         text = f"Id: {idx}<br />"
                         if el not in ["predictions", "targets", "errors"]:
-                            text += f"{el}: {row[el]}<br />"
+                            text += f"{el}: {format_missing_value(row[el])}<br />"
                         text += f"Predicted Value: {row['proba_values']:.{self._round_digit}f}<br />"
                         if "error" in df_pred.columns:
                             text += f"Error: {row['error']:.{self._round_digit}f}<br />"
@@ -2360,19 +2361,30 @@ def clustering_by_explainability_plot(
                         if el not in ["predictions", "targets", "errors"]:
                             is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
                             n_unique = df_pred[el].nunique(dropna=True)
+                            cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
                             if is_num and n_unique > 5:
-                                mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
-                                std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
-                                hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
-                                hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+                                mean_el = cluster_values.mean()
+                                std_el = cluster_values.std()
+                                if pd.isna(mean_el) or pd.isna(std_el):
+                                    hv_text_cluster += f"<br />{el} mean: {format_missing_value(mean_el)}"
+                                    hv_text_cluster += f"<br />{el} std: {format_missing_value(std_el)}"
+                                else:
+                                    hv_text_cluster += (
+                                        f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+                                    )
+                                    hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
                             else:
-                                top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
-                                top_element_percentage = (
-                                    np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
-                                    / df_pred.loc[df_pred["cluster"] == c, el].size
-                                    * 100
+                                # dropna=False so that null values can be reported as the top modality
+                                top_element = cluster_values.mode(dropna=False).iloc[0]
+                                if pd.isna(top_element):
+                                    top_element_count = cluster_values.isna().sum()
+                                else:
+                                    top_element_count = np.sum(cluster_values == top_element)
+                                top_element_percentage = top_element_count / cluster_values.size * 100
+                                hv_text_cluster += (
+                                    f"<br />{el} top: {format_missing_value(top_element)}"
+                                    f" ({top_element_percentage:.1f}%)"
                                 )
-                                hv_text_cluster += f"<br />{el} top: {top_element} ({top_element_percentage:.1f}%)"
                         mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "proba_values"].mean()
                         hv_text_cluster += f"<br />Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
                         if "error" in df_pred.columns:
@@ -2490,7 +2502,7 @@ def clustering_by_explainability_plot(
                 for idx, row in df_pred.iterrows():
                     text = f"Id: {idx}<br />"
                     if el not in ["predictions", "targets", "errors"]:
-                        text += f"{el}: {row[el]}<br />"
+                        text += f"{el}: {format_missing_value(row[el])}<br />"
                     text += f"Predicted Value: {row['predict_value']:.{self._round_digit}f}<br />"
                     if "error" in df_pred.columns:
                         text += f"Error: {row['error']:.{compute_digit_number(row['error'])}f}<br />"
@@ -2504,19 +2516,27 @@ def clustering_by_explainability_plot(
                     if el not in ["predictions", "targets", "errors"]:
                         is_num = is_numeric_dtype(df_pred[el]) and not is_bool_dtype(df_pred[el])
                         n_unique = df_pred[el].nunique(dropna=True)
+                        cluster_values = df_pred.loc[df_pred["cluster"] == c, el]
                         if is_num and n_unique > 5:
-                            mean_el = df_pred.loc[df_pred["cluster"] == c, el].mean()
-                            std_el = df_pred.loc[df_pred["cluster"] == c, el].std()
-                            hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
-                            hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
+                            mean_el = cluster_values.mean()
+                            std_el = cluster_values.std()
+                            if pd.isna(mean_el) or pd.isna(std_el):
+                                hv_text_cluster += f"<br />{el} mean: {format_missing_value(mean_el)}"
+                                hv_text_cluster += f"<br />{el} std: {format_missing_value(std_el)}"
+                            else:
+                                hv_text_cluster += f"<br />{el} mean: {mean_el:.{compute_digit_number(mean_el, 3)}f}"
+                                hv_text_cluster += f"<br />{el} std: {std_el:.{compute_digit_number(std_el, 3)}f}"
                         else:
-                            top_element = df_pred.loc[df_pred["cluster"] == c, el].mode()[0]
-                            top_element_percentage = (
-                                np.sum(df_pred.loc[df_pred["cluster"] == c, el] == top_element)
-                                / df_pred.loc[df_pred["cluster"] == c, el].size
-                                * 100
+                            # dropna=False so that null values can be reported as the top modality
+                            top_element = cluster_values.mode(dropna=False).iloc[0]
+                            if pd.isna(top_element):
+                                top_element_count = cluster_values.isna().sum()
+                            else:
+                                top_element_count = np.sum(cluster_values == top_element)
+                            top_element_percentage = top_element_count / cluster_values.size * 100
+                            hv_text_cluster += (
+                                f"<br />{el} top: {format_missing_value(top_element)} ({top_element_percentage:.1f}%)"
                             )
-                            hv_text_cluster += f"<br />{el} top: {top_element} ({top_element_percentage:.1f}%)"
                     mean_predicted_value = df_pred.loc[df_pred["cluster"] == c, "predict_value"].mean()
                     hv_text_cluster += f"<br />Mean predicted value: {mean_predicted_value:.{compute_digit_number(mean_predicted_value, 3)}f}"
                     if "error" in df_pred.columns:

shapash/plots/plot_bar_chart.py

@@ -1,7 +1,7 @@
 from plotly import graph_objs as go
 from plotly.offline import plot
 
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import add_line_break, adjust_title_height, format_missing_value, truncate_str
 
 
 def plot_bar_chart(
@@ -103,6 +103,7 @@ def plot_bar_chart(
         bars = []
         for num, expl in enumerate(zip(var_dict, x_val, contrib, strict=False)):
             feat_name, x_val_el, contrib_value = expl
+            x_val_el = format_missing_value(x_val_el)
             is_grouped = False
             if x_val_el == "":
                 ylabel = f"<i>{feat_name}</i>"
@@ -116,7 +117,7 @@ def plot_bar_chart(
                     feat_groups_values = x_init[features_groups[group_name]].loc[index_value[0]]
                     hoverlabel = "<br />".join(
                         [
-                            f"<b>{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :</b>{add_line_break(f_value, 40, maxlen=160)}"
+                            f"<b>{add_line_break(features_dict.get(f_name, f_name), 40, maxlen=120)} :</b>{add_line_break(format_missing_value(f_value), 40, maxlen=160)}"
                             for f_name, f_value in feat_groups_values.to_dict().items()
                         ]
                     )

shapash/plots/plot_contribution.py

@@ -7,7 +7,13 @@
 from plotly.subplots import make_subplots
 from sklearn.neighbors import KernelDensity
 
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import (
+    MISSING_VALUE_DISPLAY,
+    add_line_break,
+    adjust_title_height,
+    format_missing_value,
+    truncate_str,
+)
 from shapash.webapp.utils.utils import round_to_k
 
 NAN_PLACEHOLDER_K = 0.2
@@ -166,7 +172,8 @@ def plot_scatter(
         fig.add_trace(density_plot)
 
     nan_mask_arr = pd.isna(feature_values.iloc[:, 0]).to_numpy()
-    has_nan_numeric = bool(nan_mask_arr.any()) and feature_values.iloc[:, 0].dtype.kind in "biufc"
+    has_nan = bool(nan_mask_arr.any())
+    has_nan_numeric = has_nan and feature_values.iloc[:, 0].dtype.kind in "biufc"
     marker = None
     if has_nan_numeric:
         non_nan_arr = feature_values_array[~nan_mask_arr].astype(float)
@@ -178,6 +185,11 @@ def plot_scatter(
             nan_x = 0.0
         feature_values_array = np.where(nan_mask_arr, nan_x, feature_values_array)
         marker = {"symbol": np.where(nan_mask_arr, "x", "circle").tolist()}
+    elif has_nan:
+        # non-numeric columns: display null values as an explicit "missing" modality
+        feature_values_array = feature_values_array.astype(object).copy()
+        feature_values_array[nan_mask_arr] = MISSING_VALUE_DISPLAY
+        marker = {"symbol": np.where(nan_mask_arr, "x", "circle").tolist()}
 
     fig.add_scatter(
         x=feature_values_array,
@@ -199,9 +211,9 @@ def plot_scatter(
     # The values are used in the hovertext and the indexes are used for
     # the interactions between the graphics.
     customdata_values = feature_values_array
-    if has_nan_numeric:
+    if has_nan:
         customdata_values = feature_values_array.astype(object).copy()
-        customdata_values[nan_mask_arr] = "missing"
+        customdata_values[nan_mask_arr] = MISSING_VALUE_DISPLAY
     customdata = np.stack((customdata_values, feature_values.index.values), axis=-1)
 
     fig.update_traces(customdata=customdata, hovertemplate=hovertemplate)
@@ -325,7 +337,7 @@ def plot_violin(
     for i, c in enumerate(xs):
         if pd.isna(c):
             is_c = feature_values.iloc[:, 0].isna()
-            c_label = "missing"
+            c_label = MISSING_VALUE_DISPLAY
         else:
             is_c = feature_values.iloc[:, 0] == c
             c_label = c
@@ -440,7 +452,7 @@ def plot_violin(
     )
 
     # To change ticktext
-    xs_labels = ["missing" if pd.isna(x) else x for x in xs]
+    xs_labels = [format_missing_value(x) for x in xs]
     _update_xaxis_labels(fig, xs_labels, zoom)
 
     _update_contributions_fig(
@@ -754,8 +766,12 @@ def _add_violin_and_scatter(
         x = _create_jittered_points(x, percentage_series, side=side)
         if colorpoints is not None:
             colorpoints_selected = colorpoints.loc[feature_cond].values.flatten()
+        # display null values as "missing" in the hover text
+        point_values = np.array(
+            [format_missing_value(v) for v in feature_values.loc[feature_cond].values.flatten()], dtype=object
+        )
         customdata = np.stack(
-            (feature_values.loc[feature_cond].values.flatten(), contributions.loc[feature_cond].index.values),
+            (point_values, contributions.loc[feature_cond].index.values),
             axis=-1,
         )
         marker = None

shapash/plots/plot_line_comparison.py

@@ -3,7 +3,7 @@
 from plotly import graph_objs as go
 from plotly.offline import plot
 
-from shapash.utils.utils import add_line_break, adjust_title_height, truncate_str
+from shapash.utils.utils import add_line_break, adjust_title_height, format_missing_value, truncate_str
 
 
 def plot_line_comparison(
@@ -108,7 +108,7 @@ def plot_line_comparison(
                 f"Id: <b>{add_line_break(id_i, 40, 160)}</b>"
                 + f"<br /><b>{add_line_break(feat, 40, 160)}</b> <br />"
                 + f"Contribution: {contrib[i]:.4f} <br />Value: "
-                + str(add_line_break(pred_x_val, 40, 160))
+                + str(add_line_break(format_missing_value(pred_x_val), 40, 160))
             )
 
         lines.append(

shapash/utils/utils.py

@@ -199,6 +199,34 @@ def truncate_str(text, maxlen=40):
     return text
 
 
+MISSING_VALUE_DISPLAY = "missing"
+
+
+def format_missing_value(value, missing_display=MISSING_VALUE_DISPLAY):
+    """
+    return a unified display value for null entries
+
+    Parameters
+    ----------
+    value : any
+        value to display, can be null (NaN, None, NaT, pd.NA)
+    missing_display : str
+        text displayed in place of null values
+
+    Returns
+    -------
+    any
+        missing_display if the value is null, the original value otherwise
+    """
+    try:
+        if pd.isna(value):
+            return missing_display
+    except (TypeError, ValueError):
+        # non-scalar values (list, array, ...) are kept unchanged
+        pass
+    return value
+
+
 def compute_digit_number(value, significant_digits: int = 4):
     """
     return int, number of digits to display

shapash/webapp/smart_app.py

@@ -47,7 +47,7 @@
 )
 from shapash.webapp.utils.explanations import Explanations
 from shapash.webapp.utils.MyGraph import MyGraph
-from shapash.webapp.utils.utils import check_row, get_index_type, round_to_k
+from shapash.webapp.utils.utils import check_row, get_datatable_data_and_tooltips, get_index_type, round_to_k
 
 
 def _create_input_modal(component_id, label, tooltip):
@@ -431,13 +431,11 @@ def init_components(self):
 
         self.adjust_menu()
 
+        table_data, table_tooltip_data = get_datatable_data_and_tooltips(self.round_dataframe, self.dataframe)
         self.components["table"]["dataset"] = dash_table.DataTable(
             id="dataset",
-            data=self.round_dataframe.to_dict("records"),
-            tooltip_data=[
-                {column: {"value": str(value), "type": "text"} for column, value in row.items()}
-                for row in self.dataframe.to_dict("index").values()
-            ],
+            data=table_data,
+            tooltip_data=table_tooltip_data,
             tooltip_duration=2000,
             columns=[{"name": i, "id": i} for i in self.dataframe.columns],
             tooltip_header={
@@ -2108,11 +2106,7 @@ def update_datatable(
                 df = self.round_dataframe
             else:
                 raise dash.exceptions.PreventUpdate
-            data = df.to_dict("records")
-            tooltip_data = [
-                {column: {"value": str(value), "type": "text"} for column, value in row.items()}
-                for row in df.to_dict("index").values()
-            ]
+            data, tooltip_data = get_datatable_data_and_tooltips(df)
             return (
                 data,
                 tooltip_data,

shapash/webapp/utils/callbacks.py

@@ -11,6 +11,7 @@
 from dash.exceptions import PreventUpdate
 from plotly.graph_objs import Figure
 
+from shapash.utils.utils import format_missing_value
 from shapash.webapp.utils.MyGraph import MyGraph
 
 
@@ -506,7 +507,7 @@ def create_id_card_layout(selected_data: pd.DataFrame, additional_features_dict:
             dbc.Row(
                 [
                     dbc.Col(dbc.Label(row["feature_name"]), width=3, style=label_style),
-                    dbc.Col(dbc.Label(row["feature_value"]), width=5, className="id_card_solid"),
+                    dbc.Col(dbc.Label(format_missing_value(row["feature_value"])), width=5, className="id_card_solid"),
                     dbc.Col(width=1),
                     (
                         dbc.Col(

shapash/webapp/utils/utils.py

@@ -1,6 +1,35 @@
 import pandas as pd
 from pandas.api.types import is_any_real_numeric_dtype
 
+from shapash.utils.utils import format_missing_value
+
+
+def get_datatable_data_and_tooltips(data_df, tooltip_df=None):
+    """
+    Build the data records and tooltips of the dataset DataTable,
+    with a unified display of missing values.
+
+    Parameters
+    ----------
+    data_df : pd.DataFrame
+        Dataframe used for the cells of the datatable
+    tooltip_df : pd.DataFrame (optional)
+        Dataframe used for the tooltips of the datatable, data_df if not provided
+
+    Returns
+    -------
+    tuple
+        data records and tooltip_data of the datatable
+    """
+    if tooltip_df is None:
+        tooltip_df = data_df
+    data = [{col: format_missing_value(val) for col, val in row.items()} for row in data_df.to_dict("records")]
+    tooltip_data = [
+        {col: {"value": str(format_missing_value(val)), "type": "text"} for col, val in row.items()}
+        for row in tooltip_df.to_dict("records")
+    ]
+    return data, tooltip_data
+
 
 def round_to_k(x, k):
     """