diff --git a/MANIFEST.in b/MANIFEST.in index c6b2b0ad..6fc3e078 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,4 @@ recursive-include shapash/webapp/assets * -recursive-include shapash/report/html * -recursive-include shapash/report/template * include LICENSE include README.md -include shapash/report/base_report.ipynb diff --git a/README.md b/README.md index 20d40368..27c1eccb 100644 --- a/README.md +++ b/README.md @@ -198,8 +198,8 @@ app = xpl.run_app() [Live Demo Shapash-Monitor](https://shapash-demo.ossbymaif.fr/) - Step 4: Generate the Shapash Report - > This step allows to generate a standalone html report of your project using the different splits - of your dataset and also the metrics you used: + > This step generates a standalone HTML report from a block-based layout. + You can optionally provide a YAML file to customize report sections and blocks. ```python xpl.generate_report( @@ -208,6 +208,7 @@ xpl.generate_report( x_train=xtrain, y_train=ytrain, y_test=ytest, + yaml_path="path/to/report_config.yml", # Optional: custom block configuration title_story="House prices report", title_description="""This document is a data science report of the kaggle house prices tutorial project. It was generated using the Shapash library.""", diff --git a/docs/assets/images/logos/shapash-fond-clair.png b/docs/assets/images/logos/shapash-fond-clair.png new file mode 100644 index 00000000..6300ec34 Binary files /dev/null and b/docs/assets/images/logos/shapash-fond-clair.png differ diff --git a/docs/overview.rst b/docs/overview.rst index 64fc9755..f1727927 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -90,8 +90,8 @@ The 4 steps to display results: app = xpl.run_app() - Step 4: Generate the Shapash Report - > This step allows to generate a standalone html report of your project using the different splits - of your dataset and also the metrics you used: + > This step generates a standalone HTML report from a configurable block-based layout. + > You can provide a YAML configuration file to customize sections and blocks. .. code:: ipython @@ -101,10 +101,7 @@ The 4 steps to display results: x_train=Xtrain, y_train=ytrain, y_test=ytest, - title_story="House prices report", - title_description="""This document is a data science report of the kaggle house prices tutorial project. - It was generated using the Shapash library.""", - metrics=[{'name': 'MSE', 'path': 'sklearn.metrics.mean_squared_error'}] + yaml_path='path/to/report_config.yml', # Optional: custom block configuration ) - Step 5: From training to deployment : SmartPredictor Object diff --git a/pyproject.toml b/pyproject.toml index fa7c7a6f..8e69a987 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "numpy>=2.0.0", # 3.0.4 yanked upstream: segfaults in datetime ops (pandas #66083) "pandas>=2.2.2,!=3.0.4,<4.0.0", + "phik>=0.12.4", "plotly>=5.0.0,<6.0.0", "scikit-learn>=1.8.0,<1.9.0", "scipy>=1.13.0", @@ -51,13 +52,8 @@ dependencies = [ [project.optional-dependencies] # Optional report = [ - "Jinja2>=3.1.0", - "jupyter-client>=8.3.0", - "nbconvert>=7.2.0", - "notebook>=7.0.0", - "papermill>=2.5.0", - "phik>=0.12.4", - "pyarrow>=17.0.0", + "panel>=1.8.10", + ] xgboost = ["xgboost>=2.1.0"] catboost = ["catboost>=1.2.8"] diff --git a/shapash/explainer/smart_explainer.py b/shapash/explainer/smart_explainer.py index 808b31b4..14429fc3 100644 --- a/shapash/explainer/smart_explainer.py +++ b/shapash/explainer/smart_explainer.py @@ -4,8 +4,7 @@ import copy import logging -import shutil -import tempfile +from pathlib import Path from typing import Any import numpy as np @@ -17,7 +16,6 @@ from shapash.backend.shap_backend import get_shap_interaction_values from shapash.manipulation.select_lines import keep_right_contributions from shapash.manipulation.summarize import create_grouped_features_values -from shapash.report import check_report_requirements from shapash.style.style_utils import colors_loading, select_palette from shapash.utils.check import ( check_additional_data, @@ -36,6 +34,14 @@ from shapash.utils.utils import get_host_name from shapash.webapp.smart_app import SmartApp +try: + from shapash.report import ReportTemplate + from shapash.report.blocks import ReportBlockMixin + from shapash.report.core import generate_report as generate_smart_report +except ImportError: + # [report] optional dependencies may not be installed + ... + from .smart_plotter import SmartPlotter logging.basicConfig(level=logging.INFO) @@ -1654,39 +1660,28 @@ def check_x_y_attributes(self, x_str, y_str): def generate_report( self, - output_file, - project_info_file, - x_train=None, - y_train=None, - y_test=None, - title_story=None, - title_description=None, - metrics=None, - working_dir=None, - notebook_path=None, - kernel_name=None, - max_points=200, - display_interaction_plot=False, - nb_top_interactions=5, - ): + output_file: str, + x_train: pd.DataFrame | None = None, + y_train: pd.Series | pd.DataFrame | list | None = None, + y_test: pd.Series | pd.DataFrame | list | None = None, + yaml_path: str | Path | None = None, + max_points: int = 200, + block_instance: ReportBlockMixin | None = None, + ) -> None: """ Generate an interactive HTML report summarizing the model and its explainability. This method produces a comprehensive HTML report containing visual and textual - insights about the project, dataset, and model performance. - It leverages a predefined or custom Jupyter notebook template to analyze - the model, generate plots, compute metrics, and export the final report. + insights about the project, dataset, and model performance using the + smart_report block-based HTML renderer. - A project information YAML file is required to describe key project details - (e.g., model name, author, date, context). + A report configuration is provided through a YAML file. If no YAML file is + specified, a default configuration is generated automatically. Parameters ---------- output_file : str Path to the output HTML file where the report will be saved. - project_info_file : str - Path to a YAML file containing project metadata to be displayed in the report - (e.g., project name, author, date, description). x_train : pandas.DataFrame, optional Training dataset used to fit the model. Used for generating feature summaries and training-related analyses. @@ -1694,34 +1689,15 @@ def generate_report( Target values corresponding to `x_train`. y_test : pandas.Series or pandas.DataFrame, optional Target values for the test dataset. - title_story : str, optional - Title displayed at the top of the report. - title_description : str, optional - Short descriptive text displayed below the main title. - metrics : list of dict, optional - List of metrics to compute and display in the performance section. - Each dictionary should include: - - `'path'`: str — import path to the metric function (e.g., `"sklearn.metrics.f1_score"`) - - `'name'`: str, optional — display name for the metric - - `'use_proba_values'`: bool, optional — if True, use predicted probabilities instead of labels - Example: - `metrics=[{'name': 'F1 score', 'path': 'sklearn.metrics.f1_score'}]` - working_dir : str, optional - Directory used to temporarily store generated files (e.g., notebook, outputs). - If `None`, a temporary directory is automatically created and deleted after report generation. - notebook_path : str, optional - Path to a custom notebook used as a template for generating the report. - If `None`, the default Shapash report notebook is used. - kernel_name : str, optional - Name of the Jupyter kernel to use for report execution. - Useful when multiple kernels are available and the default one is incorrect. + yaml_path : str, optional + Path to a custom YAML configuration file used to generate the report. + If `None`, a default YAML configuration is generated. max_points : int, optional, default=200 Maximum number of points displayed in contribution plots. - display_interaction_plot : bool, optional, default=False - If True, includes interaction plots in the report. - (Note: this can increase computation time.) - nb_top_interactions : int, optional, default=5 - Number of top feature interactions to include in the report. + block_instance : object, optional + Optional custom block runtime used to resolve block methods during report generation. + The instance must already be fully initialized by the user and should implement + methods named `block_` for YAML block entries. Returns ------- @@ -1737,7 +1713,7 @@ def generate_report( Notes ----- - - The method internally executes a notebook that generates the report content. + - The method renders the report from block definitions in a YAML configuration. - Temporary files are automatically cleaned up unless a custom `working_dir` is provided. - Interaction plots can be disabled to optimize runtime performance. @@ -1745,69 +1721,60 @@ def generate_report( ------- >>> xpl.generate_report( ... output_file="report.html", - ... project_info_file="utils/project_info.yml", ... x_train=x_train, ... y_train=y_train, ... y_test=y_test, - ... title_story="House Prices Project Report", - ... title_description="Comprehensive interpretability analysis for the Kaggle house prices dataset.", - ... metrics=[ - ... {"path": "sklearn.metrics.mean_squared_error", "name": "Mean Squared Error"}, - ... {"path": "sklearn.metrics.mean_absolute_error", "name": "Mean Absolute Error"}, - ... ], ... display_interaction_plot=True, ... nb_top_interactions=5, ... ) """ - check_report_requirements() - if x_train is not None: - x_train = handle_categorical_missing(x_train) - # Avoid Import Errors with requirements specific to the Shapash Report - from shapash.report.generation import execute_report, export_and_save_report # noqa: PLC0415 - - rm_working_dir = False - if not working_dir: - working_dir = tempfile.mkdtemp() - rm_working_dir = True + # input checks if not hasattr(self, "model"): raise AssertionError( "Explainer object was not compiled. Please compile the explainer " "object using .compile(...) method before generating the report." ) - try: - execute_report( - working_dir=working_dir, + if block_instance is not None: + if (x_train is not None) and (block_instance.x_train_init is not x_train): + logging.warning("block_instance's x_train is different from provided x_train. Latter is ignored.") + if (y_train is not None) and (block_instance.y_train is not y_train): + logging.warning("block_instance's y_train is different from provided y_train. Latter is ignored.") + if (y_test is not None) and (block_instance.y_test is not y_test): + logging.warning("block_instance's y_test is different from provided y_test. Latter is ignored.") + if max_points != block_instance.max_points: + logging.warning("block_instance's max_points is different from provided max_points. Latter is ignored.") + + report_runtime = block_instance + + else: + if x_train is not None: + x_train = handle_categorical_missing(x_train) + + report_runtime = ReportBlockMixin( explainer=self, - project_info_file=project_info_file, x_train=x_train, y_train=y_train, y_test=y_test, - config={ - k: v - for k, v in dict( - title_story=title_story, - title_description=title_description, - metrics=metrics, - max_points=max_points, - display_interaction_plot=display_interaction_plot, - nb_top_interactions=nb_top_interactions, - ).items() - if v is not None - }, - notebook_path=notebook_path, - kernel_name=kernel_name, + max_points=max_points, ) - export_and_save_report(working_dir=working_dir, output_file=output_file) + if self._case == "classification": + default_report = ReportTemplate.DEFAULT_CLASSIFICATION + else: + default_report = ReportTemplate.DEFAULT_REGRESSION - if rm_working_dir: - shutil.rmtree(working_dir) + config_file = ( + Path(yaml_path) + if yaml_path is not None + else Path(__file__).resolve().parent.parent / "report" / "assets" / str(default_report) + ) - except Exception as e: - if rm_working_dir: - shutil.rmtree(working_dir) - raise e + generate_smart_report( + runtime=report_runtime, + config_file=config_file, + output_file=output_file, + ) def _local_pred(self, index, label=None): """ diff --git a/shapash/report/__init__.py b/shapash/report/__init__.py index 3cf2aa08..a8dea13b 100644 --- a/shapash/report/__init__.py +++ b/shapash/report/__init__.py @@ -1,20 +1,3 @@ -import importlib +from .common import ReportTemplate, export_report_yml -# This list should be identical to the list in setup.py -report_requirements = ["nbconvert==6.0.7", "papermill", "matplotlib", "notebook", "Jinja2"] - - -def check_report_requirements(): - """ - Checks that all required packages for the report are installed. - This function should be called before executing the report. - """ - for req in report_requirements: - pkg = req.split("=")[0] - try: - importlib.import_module(pkg.lower()) - except ImportError as err: - raise ModuleNotFoundError( - f"The following package is necessary to generate the Shapash Report : {pkg}. " - f"Try 'pip install shapash[report]' to install all required packages." - ) from err +__all__ = ["ReportTemplate", "export_report_yml"] diff --git a/shapash/report/assets/default_classification_report.yml b/shapash/report/assets/default_classification_report.yml new file mode 100644 index 00000000..0dc7eb5a --- /dev/null +++ b/shapash/report/assets/default_classification_report.yml @@ -0,0 +1,122 @@ +# default_report_classification_titanic.yml +# Smart report configuration for Titanic binary classification (survival). +# This file is a template: edit titles, texts, and block parameters for your project. +# You can reorder sections/blocks, remove blocks, or add new ones supported by ReportBlockMixin. + +sections: + # One item in this list = one top-level section in the generated HTML report. + # - type: header + # # `header` renders the report title area. + # params: + # title: Your project title + # subtitle: > + # Some subtitle for your project + + # # `group` lets you organize several blocks under a common section title. + # - type: group + # params: + # title: "Project information" + # blocks: + # # `text` displays a free markdown text block. + # - type: text + # params: + # title: "General information" + # # Replace placeholders with your own project metadata. + # content: + # version: x.y.z + # name: Name of your project + # purpose: ... + # date: 1970-01-01 + # contributors: Your Name + # description: ... + # source code: https://... + # - type: text + # params: + # title: "Dataset information" + # # Describe data provenance, scope, and target variable. + # content: + # path: ... + # origin: ... + # description: ... + # depth: ... + # perimeter: ... + # target variable: name of target variable + # target description: description of target variable + # - type: text + # params: + # title: "Some other subsection" + # # You can duplicate this block to add any extra context. + # content: + # whatever: you want to add + + # `model_analysis` summarizes estimator type, library, and model parameters. + - type: model_analysis + params: + title: "Model analysis" + + - type: group + params: + title: "Dataset analysis" + blocks: + # Global statistics on prediction and (optionally) training datasets. + - type: global_analysis + params: + title: "Global analysis" + + # Interactive per-feature distributions and statistics. + - type: univariate_analysis + params: + title: "Univariate analysis" + + # Target-specific analysis; set `show_train: false` to hide training target. + - type: target_analysis + params: + title: "Target analysis" + show_train: true + + # Correlation matrix; reduce `max_features` for faster/lighter reports. + - type: correlations_plot + params: + title: "Multivariate analysis" + max_features: 20 + + - type: group + params: + title: "Model performance" + blocks: + # Classification confusion matrix using y_true vs y_pred. + - type: confusion_matrix + params: + title: "Confusion matrix" + + # Lift curve for one class. Set `label` to the class id/value to inspect. + - type: lift_curve + params: + title: "Lift curve" + label: 1 + + # Compare prediction and true target distributions. + - type: target_distribution + params: + title: "Target distribution" + + # Add/remove metrics using fully qualified function paths. + - type: performance_metrics + params: + title: "Metrics" + metrics: + - path: "sklearn.metrics.accuracy_score" + name: "Accuracy" + - path: "sklearn.metrics.precision_score" + name: "Precision" + - path: "sklearn.metrics.recall_score" + name: "Recall" + - path: "sklearn.metrics.f1_score" + name: "F1" + + # Optional final note displayed at the end of the report. + - type: callout + params: + body: > + This report uses built-in blocks only. You can later plug a custom + block runtime to extend the layout. diff --git a/shapash/report/assets/default_regression_report.yml b/shapash/report/assets/default_regression_report.yml new file mode 100644 index 00000000..c8c8293f --- /dev/null +++ b/shapash/report/assets/default_regression_report.yml @@ -0,0 +1,129 @@ +# This file is a template: edit titles, texts, and block parameters for your project. +# You can reorder sections/blocks, remove blocks, or add new ones supported by ReportBlockMixin. + +sections: + # One item in this list = one top-level section in the generated HTML report. + + # - type: header + # # `header` renders the report title area. + # params: + # title: Your project title + # subtitle: > + # Some subtitle for your project + + # # `group` lets you organize several blocks under a common section title. + # - type: group + # params: + # title: "Project information" + # blocks: + # # `text` displays a free markdown text block. + # - type: text + # params: + # title: "General information" + # # Replace placeholders with your own project metadata. + # content: + # version: x.y.z + # name: Name of your project + # purpose: ... + # date: 1970-01-01 + # contributors: Your Name + # description: ... + # source code: https://... + # - type: text + # params: + # title: "Dataset information" + # # Describe data provenance, scope, and target variable. + # content: + # path: ... + # origin: ... + # description: ... + # depth: ... + # perimeter: ... + # target variable: name of target variable + # target description: description of target variable + # - type: text + # params: + # title: "Some other subsection" + # # You can duplicate this block to add any extra context. + # content: + # whatever: you want to add + + # `model_analysis` summarizes estimator type, library, and model parameters. + - type: model_analysis + params: + title: "Model analysis" + + - type: group + params: + title: "Dataset analysis" + blocks: + # Global statistics on prediction and (optionally) training datasets. + - type: global_analysis + params: + title: "Global analysis" + + # Interactive per-feature distributions and statistics. + - type: univariate_analysis + params: + title: "Univariate analysis" + + # Target-specific analysis; set `show_train: false` to hide training target. + - type: target_analysis + params: + title: "Target analysis" + show_train: true + + # Correlation matrix; reduce `max_features` for faster/lighter reports. + - type: correlations_plot + params: + title: "Multivariate analysis" + max_features: 20 + + - type: group + params: + title: "Model explainability" + blocks: + # Global ranking of feature importance. + - type: feature_importance + params: + title: "Global feature importance plot" + + # Set `include_all_features: false` if you prefer one feature at a time. + - type: contribution_plot + params: + title: "Features contribution plots" + include_all_features: true + + # # Combined features importance. + # # Take a while to compute for large feature sets. + # - type: top_interactions_plot + # params: + # title: "Top interactions plot" + # nb_top_interaction: 5 + + - type: group + params: + title: "Model performance" + blocks: + # Compare prediction and true target distributions. + - type: target_distribution + params: + title: "Univariate analysis of target variable" + + # Add/remove metrics using fully qualified function paths. + - type: performance_metrics + params: + title: "Metrics" + metrics: + - path: "sklearn.metrics.mean_absolute_error" + name: "Mean absolute error" + - path: "sklearn.metrics.mean_squared_error" + name: "Mean squared error" + - path: "sklearn.metrics.r2_score" + name: "R2 score" + + # Optional final note displayed at the end of the report. + - type: callout + params: + body: > + This report uses built-in blocks only. diff --git a/shapash/report/assets/default_report.yml b/shapash/report/assets/default_report.yml new file mode 100644 index 00000000..fb3854e2 --- /dev/null +++ b/shapash/report/assets/default_report.yml @@ -0,0 +1,77 @@ +# default_report.yml +# Default smart report configuration based on block sections. + +sections: + - type: header + params: + title: "House prices report" + subtitle: > + This document is a data science report of the kaggle house prices tutorial project. + It was generated using the Shapash library. + + - type: project_information + params: + title: "Project information" + project_info_file: "tutorial/generate_report/config/project_information.yml" + + - type: model_analysis + params: + title: "Model analysis" + + - type: group + params: + title: "Dataset analysis" + blocks: + - type: global_analysis + params: + title: "Global analysis" + + - type: univariate_analysis + params: + title: "Univariate analysis" + + - type: target_analysis + params: + title: "Target analysis" + show_train: true + + - type: correlations_plot + params: + title: "Multivariate analysis" + max_features: 20 + + - type: group + params: + title: "Model explainability" + blocks: + - type: feature_importance + params: + title: "Global feature importance plot" + + - type: contribution_plot + params: + title: "Features contribution plots" + include_all_features: true + + - type: group + params: + title: "Model performance" + blocks: + - type: target_distribution + params: + title: "Univariate analysis of target variable" + + - type: performance_metrics + params: + title: "Metrics" + metrics: + - path: "sklearn.metrics.mean_absolute_error" + name: "Mean absolute error" + - path: "sklearn.metrics.mean_squared_error" + name: "Mean squared error" + + - type: callout + params: + body: > + You can add as many blocks, charts, and text sections as you want. + The generated HTML renders report content only (no source code). diff --git a/shapash/report/assets/report_script.js b/shapash/report/assets/report_script.js new file mode 100644 index 00000000..b96e9c78 --- /dev/null +++ b/shapash/report/assets/report_script.js @@ -0,0 +1,188 @@ +function initReportInteractions() { + let scrollFrame = null; + const boundScrollRoots = new WeakSet(); + + function collectRoots() { + const roots = [document]; + const pending = [document]; + const seen = new WeakSet(); + seen.add(document); + + while (pending.length > 0) { + const currentRoot = pending.pop(); + currentRoot.querySelectorAll('*').forEach(element => { + if (element.shadowRoot && !seen.has(element.shadowRoot)) { + seen.add(element.shadowRoot); + roots.push(element.shadowRoot); + pending.push(element.shadowRoot); + } + }); + } + + return roots; + } + + function queryAllRoots(selector) { + return collectRoots().flatMap(root => Array.from(root.querySelectorAll(selector))); + } + + function queryByIdAcrossRoots(id) { + for (const root of collectRoots()) { + if (typeof root.getElementById === 'function') { + const match = root.getElementById(id); + if (match) { + return match; + } + } + } + return null; + } + + function clearActive(navItems, navChildren, navGroupTitles) { + navItems.forEach(element => element.classList.remove('active')); + navChildren.forEach(element => element.classList.remove('active')); + navGroupTitles.forEach(element => element.classList.remove('active')); + } + + function bindScrollListeners() { + collectRoots().forEach(root => { + if (!boundScrollRoots.has(root)) { + root.addEventListener('scroll', queueScrollUpdate, true); + boundScrollRoots.add(root); + } + }); + } + + function bindPanelSelectors() { + queryAllRoots('.js-panel-select[data-panel-group]').forEach(select => { + if (select.dataset.reportBound === 'true') { + return; + } + + function updatePanels() { + const panelGroup = select.getAttribute('data-panel-group'); + const panels = queryAllRoots(`.section-block[data-panel-group="${panelGroup}"]`); + + panels.forEach(panel => { + panel.style.display = 'none'; + }); + + const selectedPanel = queryByIdAcrossRoots(select.value); + if (selectedPanel) { + selectedPanel.style.display = 'block'; + } + + queueScrollUpdate(); + } + + select.addEventListener('change', updatePanels); + select.dataset.reportBound = 'true'; + updatePanels(); + }); + } + + function onScroll() { + const sections = queryAllRoots('.scroll-anchor[id]'); + const navItems = queryAllRoots('.nav-item:not(.nav-group-title):not(.nav-child)'); + const navChildren = queryAllRoots('.nav-child'); + const navGroupTitles = queryAllRoots('.nav-group-title'); + const navCurrentValue = queryAllRoots('.nav-current-value')[0] || null; + const sectionPositions = sections + .map(section => ({ + section, + top: section.getBoundingClientRect().top, + })) + .sort((left, right) => left.top - right.top); + let currentId = ''; + + sectionPositions.forEach(({ section, top }) => { + if (top <= 120) { + currentId = section.getAttribute('id'); + } + }); + + if (!currentId && sectionPositions.length > 0) { + const firstVisibleSection = sectionPositions.find(({ top }) => top > 0); + currentId = firstVisibleSection ? firstVisibleSection.section.getAttribute('id') : ''; + } + + clearActive(navItems, navChildren, navGroupTitles); + let matchedLabel = 'Top of report'; + + navItems.forEach(item => { + if (item.getAttribute('href') === '#' + currentId) { + item.classList.add('active'); + matchedLabel = item.textContent.trim(); + } + }); + + let childMatched = false; + navChildren.forEach(child => { + if (child.getAttribute('href') === '#' + currentId) { + child.classList.add('active'); + childMatched = true; + matchedLabel = child.textContent.trim(); + const group = child.closest('.nav-group'); + if (group) { + const parentTitle = group.querySelector('.nav-group-title'); + if (parentTitle) { + parentTitle.classList.add('active'); + } + } + } + }); + + if (!childMatched) { + navGroupTitles.forEach(title => { + if (title.getAttribute('href') === '#' + currentId) { + title.classList.add('active'); + matchedLabel = title.textContent.trim(); + } + }); + } + + if (navCurrentValue) { + navCurrentValue.textContent = matchedLabel || 'Top of report'; + } + } + + function queueScrollUpdate() { + if (scrollFrame !== null) { + return; + } + + scrollFrame = window.requestAnimationFrame(() => { + scrollFrame = null; + bindScrollListeners(); + bindPanelSelectors(); + onScroll(); + }); + } + + window.addEventListener('resize', queueScrollUpdate); + window.addEventListener('hashchange', queueScrollUpdate); + queueScrollUpdate(); + + let attempts = 0; + function refreshUntilReady() { + queueScrollUpdate(); + attempts += 1; + if (attempts >= 120) { + return; + } + + const hasNavigation = queryAllRoots('.nav-item').length > 0; + const hasSections = queryAllRoots('.scroll-anchor[id]').length > 0; + if (!hasNavigation || !hasSections) { + window.requestAnimationFrame(refreshUntilReady); + } + } + + refreshUntilReady(); +} + +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', initReportInteractions); +} else { + initReportInteractions(); +} diff --git a/shapash/report/assets/report_styles.css b/shapash/report/assets/report_styles.css new file mode 100644 index 00000000..dd4a1cc9 --- /dev/null +++ b/shapash/report/assets/report_styles.css @@ -0,0 +1,275 @@ +:root { + --shapash-yellow: #f4c000; + --shapash-black: #343736; +} + +.main-report { + padding: 24px 32px; + align-items: flex-start; + gap: 20px; + overflow: visible !important; +} + +.report-sidebar { + align-self: flex-start; + position: sticky; + top: 16px; + z-index: 30; + max-height: calc(100vh - 32px); + overflow: hidden; +} + +.report-content { + flex: 1 1 auto; + min-width: 0; +} + +/* Generic key/value and dataframe tables */ +.kv-table, +table.dataframe { + width: 100%; + border-collapse: separate; + border-spacing: 0; + margin: 12px 0 24px; + background: #fff; + border: 1px solid #ececec; + border-radius: 12px; + overflow: hidden; + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.04); +} + +.kv-table th, +.kv-table td, +table.dataframe th, +table.dataframe td { + text-align: center; + vertical-align: middle; +} + +.shapash-callout { + padding: 14px 20px; + border-left: 4px solid var(--shapash-yellow); +} + +.badge-pill { + border: 1px solid #eeeeee; + border-radius: 999px; + padding: 6px 12px; + display: inline-block; +} + +.badge-pill-gold { + border-color: var(--shapash-yellow); +} + +.badge-pill-blue { + border-color: #2255aa; +} + +.badge-pill-gray { + border-color: #eeeeee; +} + +.badge-pill-orange { + border-color: var(--shapash-yellow); +} + +.project-info-grid { + display: grid !important; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + align-items: stretch; +} + +.project-info-card { + display: flex; + flex-direction: column; + height: 100%; + min-width: 0; +} + +.project-info-card .kv-table { + flex: 1 1 auto; + margin-bottom: 0; +} + +.project-info-card .kv-table table.dataframe { + height: 100%; +} + +.fit-content-table { + width: fit-content; + max-width: 50%; + overflow-x: auto; +} + +.fit-content-table table.dataframe { + width: max-content; + table-layout: auto; +} + +.report-nav { + position: static; + z-index: 20; + display: flex; + flex-direction: column; + align-items: stretch; + flex-wrap: nowrap; + gap: calc(8px * var(--nav-scale, 1)); + margin: 0; + padding: calc(10px * var(--nav-scale, 1)) calc(12px * var(--nav-scale, 1)); + border: 1px solid #ececec; + border-radius: 12px; + background: rgba(255, 255, 255, 0.96); + backdrop-filter: blur(4px); + box-shadow: 0 6px 18px rgba(0, 0, 0, 0.06); + height: calc(100vh - 32px); + overflow-y: auto; + overflow-x: hidden; + -ms-overflow-style: none; + scrollbar-width: none; +} + +.report-nav::-webkit-scrollbar { + width: 0; + height: 0; +} + +.nav-logo { + display: flex; + align-items: flex-start; + justify-content: flex-start; + padding: calc(4px * var(--nav-scale, 1)); + margin-bottom: calc(6px * var(--nav-scale, 1)); +} + +.nav-logo img { + display: block; + width: min(70px, 100%); + height: auto; +} + +.nav-current { + display: flex; + flex-direction: column; + gap: 2px; + padding: calc(8px * var(--nav-scale, 1)); + border-radius: 10px; + border: 1px solid #f2d878; + background: #fff8dc; + margin-bottom: calc(4px * var(--nav-scale, 1)); +} + +.nav-current-label { + font-size: calc(0.72rem * var(--nav-scale, 1)); + text-transform: uppercase; + letter-spacing: 0.04em; + color: #7a6a2f; +} + +.nav-current-value { + font-size: calc(0.9rem * var(--nav-scale, 1)); + font-weight: 700; +} + +.nav-group { + display: flex; + flex-direction: column; + align-items: stretch; + gap: calc(8px * var(--nav-scale, 1)); + flex-wrap: nowrap; + padding: calc(4px * var(--nav-scale, 1)) 0; +} + +.nav-group-children { + display: flex; + flex-direction: column; + align-items: stretch; + gap: calc(6px * var(--nav-scale, 1)); + flex-wrap: nowrap; + padding-left: calc(10px * var(--nav-scale, 1)); +} + +.nav-item { + display: block; + padding: calc(6px * var(--nav-scale, 1)) calc(10px * var(--nav-scale, 1)); + border-radius: 8px; + border: 1px solid #dddddd; + color: var(--shapash-black); + text-decoration: none; + font-size: calc(0.9rem * var(--nav-scale, 1)); + line-height: 1.2; + background: #fff; +} + +.nav-group-title { + border-color: #d4d4d4; + font-weight: 700; +} + +.nav-child { + border-style: dashed; + font-size: calc(0.84rem * var(--nav-scale, 1)); +} + +.nav-item:hover, +.nav-item.active { + border-color: var(--shapash-yellow); + background: #fff9e6; +} + +.nav-item.active { + box-shadow: inset 3px 0 0 var(--shapash-yellow); + font-weight: 700; +} + +.scroll-anchor { + display: block; + position: relative; + top: -10px; + visibility: hidden; +} + +/* Responsive adjustments */ +@media (max-width: 1200px) { + + .main-report { + padding: 16px; + gap: 12px; + } + + .kv-table, + table.dataframe { + display: block; + overflow-x: auto; + -webkit-overflow-scrolling: touch; + } + + .kv-val, + .content-block { + overflow-wrap: anywhere; + word-break: break-word; + } + + .project-info-grid { + grid-template-columns: 1fr; + } + + .project-info-card { + width: 100%; + height: auto; + } + + .report-nav { + position: static; + padding: 8px; + height: auto; + } + + .report-sidebar { + position: static; + top: auto; + max-height: none; + overflow: visible; + } +} diff --git a/shapash/report/base_report.ipynb b/shapash/report/base_report.ipynb deleted file mode 100644 index fa72fd8c..00000000 --- a/shapash/report/base_report.ipynb +++ /dev/null @@ -1,184 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "filled-favorite", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "coordinate-shower", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "dir_path = \"\"\n", - "project_info_file = \"\"\n", - "config = dict()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "atlantic-fever", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import warnings\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "from shapash import SmartExplainer\n", - "from shapash.report.project_report import ProjectReport\n", - "from shapash.report.common import load_saved_df\n", - "\n", - "xpl = SmartExplainer.load(os.path.join(dir_path, \"smart_explainer.pickle\"))\n", - "\n", - "x_train = load_saved_df(os.path.join(dir_path, \"x_train.csv\"))\n", - "y_train = load_saved_df(os.path.join(dir_path, \"y_train.csv\"))\n", - "y_test = load_saved_df(os.path.join(dir_path, \"y_test.csv\"))\n", - "\n", - "report = ProjectReport(\n", - " explainer=xpl, project_info_file=project_info_file, x_train=x_train, y_train=y_train, y_test=y_test, config=config\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "altered-medicare", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_title_description()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "specified-vietnamese", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_project_information()" - ] - }, - { - "cell_type": "markdown", - "id": "steady-transfer", - "metadata": {}, - "source": [ - "## Model analysis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "serial-bulgaria", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_model_analysis()" - ] - }, - { - "cell_type": "markdown", - "id": "beginning-silicon", - "metadata": {}, - "source": [ - "## Dataset analysis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "planned-mayor", - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "report.display_dataset_analysis()" - ] - }, - { - "cell_type": "markdown", - "id": "attempted-bikini", - "metadata": {}, - "source": [ - "## Model explainability" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "secondary-dividend", - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - "report.display_model_explainability()" - ] - }, - { - "cell_type": "markdown", - "id": "australian-photograph", - "metadata": {}, - "source": [ - "## Model performance" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "breeding-techno", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_model_performance()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "arbitrary-baker", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "celltoolbar": "Tags", - "hide_input": false, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/shapash/report/blocks.py b/shapash/report/blocks.py new file mode 100644 index 00000000..89b9719c --- /dev/null +++ b/shapash/report/blocks.py @@ -0,0 +1,1178 @@ +"""Block implementations and report data helpers for smart reports.""" + +from __future__ import annotations + +import importlib +import importlib.metadata +import inspect +import logging +from functools import wraps +from typing import Any, TypeAlias, cast + +import numpy as np +import pandas as pd +import panel as pn + +from shapash.plots.plot_evaluation_metrics import plot_confusion_matrix, plot_lift_curve +from shapash.plots.plot_univariate import plot_distribution +from shapash.report.common import compute_col_types, series_dtype +from shapash.report.core import _wrap_section_anchor +from shapash.report.data_analysis import perform_global_dataframe_analysis, perform_univariate_dataframe_analysis +from shapash.report.panel_support import _add_css_classes, _auto_style_viewable, _coerce_viewable +from shapash.report.validation import render_block_error, stats_to_table +from shapash.utils.transform import apply_postprocessing, handle_categorical_missing, inverse_transform +from shapash.utils.utils import compute_sorted_variables_interactions_list_indices + +logger = logging.getLogger(__name__) + +PALETTE = { + "gold": {"bg": "#ffffff", "border": "#f4c000", "title": "#f4c000", "text": "#343736"}, + "blue": {"bg": "#ffffff", "border": "#2255aa", "title": "#2255aa", "text": "#343736"}, + "gray": {"bg": "#ffffff", "border": "#eeeeee", "title": "#666666", "text": "#666666"}, + "orange": {"bg": "#fff9e6", "border": "#f4c000", "title": "#cc8833", "text": "#444444"}, +} + +TARGET_DISTRIBUTION_COLORS = {"pred": "#2255aa", "true": "#f4c000"} +BlockContent: TypeAlias = tuple[str, list[Any]] +TargetValues: TypeAlias = np.ndarray[Any, Any] | list[Any] + + +def block(method): + """Wrap block output in a standard report section container. + + Decorated methods can return either ``(title, body)`` or a bare body value. + The body may be a single supported item or a list of supported items. Each + item can be a string, a pandas ``DataFrame``, a Plotly figure, or a Panel + viewable. Tuples inside the body are rendered as horizontal rows. + """ + + @wraps(method) + def wrapped(self, *args, **kwargs): + result = method(self, *args, **kwargs) + + # get block method results + if isinstance(result, tuple) and len(result) == 2: + title, body = result + else: # handle missing title + body = result + try: + bound_args = inspect.signature(method).bind(self, *args, **kwargs) + bound_args.apply_defaults() + title_value = bound_args.arguments.get("title", "") + except (TypeError, ValueError): + title_value = kwargs.get("title", "") + if isinstance(title_value, str) and title_value.strip(): + title = title_value.strip() + + items = body if isinstance(body, list) else [body] + blocks: list[pn.viewable.Viewable] = [] + + heading_prefix = "###" if getattr(self, "_inside_group", False) else "#" + blocks.append(_add_css_classes(pn.pane.Markdown(f"{heading_prefix} {title}"), "section-title")) + + # Gestion de la grille row/columns du contenu à afficher + for item in items: + if isinstance(item, tuple): + row = pn.Row( + *[ + _auto_style_viewable(_coerce_viewable(i), method_name=method.__name__) + for i in item + if i is not None + ], + sizing_mode="stretch_width", + ) + elif item is not None: + row = _auto_style_viewable(_coerce_viewable(item), method_name=method.__name__) + else: + continue + + blocks.append(row) + + return pn.Column(*blocks, css_classes=["section-block"], sizing_mode="stretch_width") + + return wrapped + + +class ReportBlockMixin: + """Base mixin providing built-in and user-extensible smart report blocks.""" + + def __init__( + self, + explainer=None, + x_train: pd.DataFrame | None = None, + y_train: pd.Series | pd.DataFrame | list | None = None, + y_test: pd.Series | pd.DataFrame | list | None = None, + max_points: int = 200, + ) -> None: + self.explainer = explainer + self.x_train_init = x_train + self.x_train_pre = self._preprocess_train_data(x_train) + self.x_init = getattr(explainer, "x_init", None) + self.df_train_test = self._create_train_test_df(test=self.x_init, train=self.x_train_pre) + self.y_train, self.target_name_train = self._get_values_and_name(y_train, "target") + self.y_test, self.target_name_test = self._get_values_and_name(y_test, "target") + self.target_name = self.target_name_train if self.target_name_train is not None else self.target_name_test + self.max_points = max_points + self._inside_group = False + + if explainer is not None: + if explainer.y_pred is not None: + self.y_pred, _ = self._get_values_and_name(explainer.y_pred, "prediction") + else: + self.y_pred = explainer.model.predict(explainer.x_encoded) + else: + self.y_pred = None + + def render_block(self, block_cfg: dict): + """Dispatch one YAML block entry to the matching block_* method.""" + + block_type = block_cfg.get("type", "") + params = block_cfg.get("params", {}) + + if block_type == "group": + previous_inside_group = getattr(self, "_inside_group", False) + self._inside_group = True + try: + children = [self.render_block(child_cfg) for child_cfg in block_cfg.get("blocks", [])] + finally: + self._inside_group = previous_inside_group + children = [child for child in children if child is not None] + group_title = params.get("title", "") + section_id = block_cfg.get("_section_id") + if group_title: + group_content = pn.Column( + pn.pane.Markdown(f"## {group_title}", css_classes=["group-title"]), + *children, + sizing_mode="stretch_width", + ) + return _wrap_section_anchor(group_content, section_id) + return _wrap_section_anchor(pn.Column(*children, sizing_mode="stretch_width"), section_id) + + method = getattr(self, f"block_{block_type}", None) + if method is None: + if block_type == "custom": + return self._render_custom(block_cfg) + logger.warning("Unknown block type '%s' - skipped.", block_type) + return None + + try: + result = method(**params) + if isinstance(result, pn.viewable.Viewable): + return _wrap_section_anchor(result, block_cfg.get("_section_id")) + raise TypeError( + f"The return type of {method.__name__} is not a panel Viewable. Did you forget the @block decorator?" + ) + except Exception as exc: + logger.error("Block '%s' raised: %s", block_type, exc) + return render_block_error(block_type, exc) + + def _render_custom(self, block_cfg: dict): + """Call an arbitrary importable function.""" + func_path = block_cfg.get("function", "") + params = block_cfg.get("params", {}) + try: + mod_path, fn_name = func_path.rsplit(".", 1) + fn = getattr(importlib.import_module(mod_path), fn_name) + result = fn(self, **params) + if isinstance(result, pn.viewable.Viewable): + return result + if isinstance(result, str): + return pn.pane.Markdown(result) + return pn.panel(result) + except Exception as exc: + logger.error("Custom block '%s' raised: %s", func_path, exc) + return render_block_error(func_path, exc) + + def block_header(self, title: str = "Report", subtitle: str = "") -> pn.Column: + """Render the report header section. + + Parameters + ---------- + title : str, default="Report" + Main report title displayed as a first-level heading. + subtitle : str, default="" + Optional markdown text displayed below the title. + + Returns + ------- + pn.Column + Panel column containing the title and optional subtitle. + + Examples + -------- + >>> runtime.block_header(title="Model report", subtitle="Summary for Q2") + """ + blocks: list[pn.viewable.Viewable] = [pn.pane.Markdown(f"# {title}", css_classes=["main-header"])] + if subtitle: + blocks.append( + pn.pane.Markdown( + subtitle, + css_classes=["shapash-callout"], + ) + ) + return pn.Column(*blocks, sizing_mode="stretch_width") + + @block + def block_text(self, title: str = "", content: dict[str, str] | str | None = None) -> BlockContent: + """Render a key/value text list from YAML items. + + Parameters + ---------- + title : str, default="" + Optional section title. + content : dict[str, str], str or None, default=None + Dict of entries. For example ``{"version": "0.7", "name": "My project"}``. + Or markdown text. + + Returns + ------- + tuple[str, list[str]] + Section title and markdown content rendered by the @block decorator. + """ + if not content: + return title, ["No information available."] + + lines: list[str] = [] + if isinstance(content, dict): + for key, value in content.items(): + lines.append(f"**{key}** : {value}") + else: + lines.append(str(content)) + + # Use markdown hard line breaks so each key/value appears on its own line. + return title, [" \n".join(lines)] + + @block + def block_badge_row(self, title: str = "", badges: list | None = None) -> BlockContent: + """Render a row of summary badges. + + Parameters + ---------- + title : str, default="" + Optional section title. + badges : list or None, default=None + List of dictionaries with keys such as ``label``, ``value``, and ``color``. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and badge row content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_badge_row(badges=[{"label": "AUC", "value": "0.89", "color": "blue"}]) + """ + if badges is None: + badges = [] + pills: list[pn.viewable.Viewable] = [] + for badge in badges: + color_name = badge.get("color", "gray") + if color_name not in PALETTE: + color_name = "gray" + pills.append( + pn.pane.Markdown( + f"**{badge.get('label', '')}**: {badge.get('value', '')}", + css_classes=[f"badge-pill-{color_name}"], + ) + ) + + return title, [tuple(pills)] + + def block_callout(self, body: str = "") -> pn.Column: + """Render a highlighted callout message. + + Parameters + ---------- + body : str, default="" + Markdown message to emphasize in the report. + + Returns + ------- + pn.Column + Panel column containing a styled callout pane. + + Examples + -------- + >>> runtime.block_callout(body="Use this report for decision support only.") + """ + return pn.Column( + pn.pane.Markdown( + body, + css_classes=["shapash-callout"], + ), + sizing_mode="stretch_width", + ) + + @block + def block_global_analysis(self, title: str = "") -> BlockContent: + """Render global summary statistics for prediction and training datasets. + + Parameters + ---------- + title : str, default="" + Optional section title. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and statistics table content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_global_analysis(title="Global dataset comparison") + """ + self._require_train_test_data("global_analysis") + test_stats = perform_global_dataframe_analysis(self.x_init) + train_stats = perform_global_dataframe_analysis(self.x_train_pre) if self.x_train_pre is not None else None + stats_table = stats_to_table( + test_stats=test_stats, + train_stats=train_stats, + names=["Prediction dataset", "Training dataset"], + ) + return title, [stats_table] + + @block + def block_model_analysis(self, title: str = "Model information") -> BlockContent: + """Render model metadata and parameter tables. + + Parameters + ---------- + title : str, default="Model information" + Section title displayed above model details. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and model details content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_model_analysis() + """ + explainer = self._require_explainer("model_analysis") + model = explainer.model + + model_module = model.__class__.__module__ + model_package = model_module.split(".")[0] + package_name = "scikit-learn" if model_package == "sklearn" else model_package + try: + library_version = importlib.metadata.version(package_name) + except importlib.metadata.PackageNotFoundError: + library_version = f"not found for {model_package}" + + model_params = getattr(model, "__dict__", {}) + params_items = list(model_params.items()) + + def _truncate(value: Any, max_len: int) -> str: + text = str(value) + return text if len(text) <= max_len else text[: max_len - 3] + "..." + + if len(params_items) > 15: + split_idx = len(params_items) // 2 + left_df = pd.DataFrame( + { + "Parameter": [_truncate(key, 50) for key, _ in params_items[:split_idx]], + "Value": [_truncate(val, 300) for _, val in params_items[:split_idx]], + } + ) + right_df = pd.DataFrame( + { + "Parameter": [_truncate(key, 50) for key, _ in params_items[split_idx:]], + "Value": [_truncate(val, 300) for _, val in params_items[split_idx:]], + } + ) + params_table = (left_df, pn.Spacer(width=24), right_df) + else: + params_df = pd.DataFrame( + { + "Parameter": [_truncate(key, 50) for key, _ in params_items], + "Value": [_truncate(val, 300) for _, val in params_items], + } + ) + params_table = params_df + + content: list[Any] = [ + pn.pane.Markdown( + "\n".join( + [ + f"**Model used**: {model.__class__.__name__}", + f"**Library**: {model_module}", + f"**Library version**: {library_version}", + "**Model parameters**", + ] + ) + ), + params_table, + ] + + return title, content + + def block_performance_metrics( + self, + title: str = "Model performance", + color: str = "orange", + metrics: list | None = None, + ) -> pn.Column: + """Compute and render selected evaluation metrics as badges. + + Parameters + ---------- + title : str, default="Model performance" + Section title displayed above metric badges. + color : str, default="orange" + Badge color name used for rendered metric pills. + metrics : list or None, default=None + Metric specifications with import path and optional display name. + + Returns + ------- + pn.Column + Panel column containing computed metric badges. + + Examples + -------- + >>> runtime.block_performance_metrics(metrics=[{"path": "sklearn.metrics.accuracy_score"}]) + """ + if self.y_test is None or self.y_pred is None: + raise ValueError("performance_metrics block requires y_test and y_pred.") + + metric_items = [] + if metrics is None: + metrics = [] + for metric_cfg in metrics: + metric_path = metric_cfg.get("path") + metric_name = metric_cfg.get("name", metric_path) + if not metric_path: + continue + module_path, fn_name = metric_path.rsplit(".", 1) + metric_fn = getattr(importlib.import_module(module_path), fn_name) + value = metric_fn(self.y_test, self.y_pred) + metric_items.append({"label": metric_name, "value": f"{value:,.2f}", "color": color}) + + return self.block_badge_row(title=title, badges=metric_items) + + @block + def block_feature_distribution( + self, + feature: str, + title: str = "", + dataset_split: str = "data_train_test", + width: int = 700, + height: int = 500, + ) -> BlockContent: + """Render feature distribution by dataset split. + + Parameters + ---------- + feature : str + Feature name to visualize. + title : str, default="" + Optional custom section title. + dataset_split : str, default="data_train_test" + Column used as hue to separate train/test distributions. + width : int, default=700 + Plot width in pixels. + height : int, default=500 + Plot height in pixels. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and feature distribution viewable rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_feature_distribution(feature="age") + """ + df_train_test = self._require_train_test_data("feature_distribution") + if feature not in df_train_test.columns: + raise ValueError(f"Unknown feature '{feature}' for feature_distribution block.") + + fig = plot_distribution( + df_all=df_train_test, + col=feature, + hue=dataset_split, + colors_dict=self._feature_distribution_colors(), + width=width, + height=height, + ) + if title is None: + return self._feature_label(feature), [fig] + return title, [fig] + + @block + def block_correlations_plot( + self, + title: str = "", + max_features: int = 20, + width: int | None = None, + height: int = 500, + ) -> BlockContent: + """Render a feature correlation matrix. + + Parameters + ---------- + title : str, default="" + Optional section title. + max_features : int, default=20 + Maximum number of features included in the matrix. + width : int or None, default=None + Optional explicit plot width. + height : int, default=500 + Plot height in pixels. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and correlations plot content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_correlations_plot(max_features=15) + """ + df_train_test = self._require_train_test_data("correlations_plot") + explainer = self._require_explainer("correlations_plot") + if width is None: + if len(df_train_test["data_train_test"].unique()) > 1: + resolved_width = 900 + else: + resolved_width = 500 + else: + resolved_width = width + fig = explainer.plot.correlations_plot( + df_train_test, + optimized=True, + facet_col="data_train_test", + max_features=max_features, + width=resolved_width, + height=height, + ) + return title, [fig] + + @block + def block_feature_importance(self, title: str = "", label=None) -> BlockContent: + """Render global feature importance. + + Parameters + ---------- + title : str, default="" + Optional section title. + label : Any, default=None + Optional class/target label for label-specific importance. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and feature-importance content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_feature_importance() + """ + explainer = self._require_explainer("feature_importance") + fig = explainer.plot.features_importance(label=label) + return title, [fig] + + @block + def block_contribution_plot( + self, + feature: str | None = None, + title: str = "", + label=None, + max_points: int | None = None, + include_all_features: bool = False, + ) -> BlockContent: + """Render feature contribution plots. + + Parameters + ---------- + feature : str or None, default=None + Feature name for single-feature mode. + title : str, default="" + Optional section title. + label : Any, default=None + Optional class/target label. + max_points : int or None, default=None + Maximum number of points used by the plotting backend. + include_all_features : bool, default=False + If True, create an interactive selector over contribution plots for all features. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and contribution content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_contribution_plot(feature="age") + >>> runtime.block_contribution_plot(include_all_features=True) + """ + explainer = self._require_explainer("contribution_plot") + + if not include_all_features: + if feature is None: + raise ValueError("contribution_plot block requires 'feature' when include_all_features=False.") + if max_points is None: + effective_max_points = self.max_points + else: + effective_max_points = max_points + fig = explainer.plot.contribution_plot(feature, label=label, max_points=effective_max_points) + for trace in fig.data: + if trace.type == "bar": + trace.marker.color = "lightgrey" + if title is None: + return self._feature_label(feature), [fig] + return title, [fig] + + if getattr(explainer, "x_init", None) is None: + raise ValueError("contribution_plot block with include_all_features=True requires explainer.x_init.") + + feature_names = list(explainer.x_init.columns) + if not feature_names: + return title, [pn.pane.Markdown("No feature available.")] + + sorted_features = sorted( + feature_names, + key=lambda current_feature: (str(self._feature_label(current_feature)).lower(), str(current_feature)), + ) + + feature_panels: dict[str, pn.viewable.Viewable] = {} + for feature_name in sorted_features: + if max_points is None: + effective_max_points = self.max_points + else: + effective_max_points = max_points + fig = explainer.plot.contribution_plot(feature_name, label=label, max_points=effective_max_points) + for trace in fig.data: + if trace.type == "bar": + trace.marker.color = "lightgrey" + + base_label = str(self._feature_label(feature_name)) + label_text = base_label + suffix = 2 + while label_text in feature_panels: + label_text = f"{base_label} ({suffix})" + suffix += 1 + feature_panels[label_text] = fig + + feature_select = pn.widgets.Select( + name="Feature", + options=list(feature_panels.keys()), + value=next(iter(feature_panels)), + sizing_mode="stretch_width", + ) + selected_panel = pn.panel( + pn.bind(cast(Any, lambda selected: feature_panels[selected]), feature_select), sizing_mode="stretch_width" + ) + + if title is None: + resolved_title = "Features contribution plots" + else: + resolved_title = title + return resolved_title, [feature_select, selected_panel] + + @block + def block_interactions_plot( + self, + title: str = "", + col1: str | None = None, + col2: str | None = None, + max_points: int | None = None, + ) -> BlockContent: + """Render an interactions plot between two features. + + Parameters + ---------- + title : str, default="" + Optional section title. + col1 : str or None, default=None + First feature. If None, the method picks a default interaction pair. + col2 : str or None, default=None + Second feature. If None, the method picks a default interaction pair. + max_points : int or None, default=None + Maximum number of points used by the plotting backend. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and interactions plot content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_interactions_plot(col1="age", col2="income") + """ + explainer = self._require_explainer("interactions_plot") + feature_one, feature_two = self._resolve_interaction_pair(col1, col2) + if max_points is None: + effective_max_points = self.max_points + else: + effective_max_points = max_points + fig = explainer.plot.interactions_plot(col1=feature_one, col2=feature_two, max_points=effective_max_points) + if title is None: + resolved_title = f"{self._feature_label(feature_one)} / {self._feature_label(feature_two)}" + else: + resolved_title = title + return resolved_title, [fig] + + @block + def block_top_interactions_plot( + self, + title: str = "Top interactions plot", + nb_top_interaction: int = 5, + max_points: int | None = None, + ) -> BlockContent: + """Render a plot for the top feature interaction pairs. + + Parameters + ---------- + title : str, default="Top interactions plot" + Section title displayed above the interaction figure. + nb_top_interaction : int, default=5 + Number of top interactions to display. + max_points : int or None, default=None + Maximum number of points used by the plotting backend. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and top-interactions plot content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_top_interactions_plot(nb_top_interaction=3) + """ + explainer = self._require_explainer("top_interactions_plot") + if max_points is None: + effective_max_points = self.max_points + else: + effective_max_points = max_points + fig = explainer.plot.top_interactions_plot( + nb_top_interactions=nb_top_interaction, + max_points=effective_max_points, + ) + return title, [fig] + + @block + def block_target_distribution( + self, + title: str = "", + width: int = 700, + height: int = 500, + ) -> BlockContent: + """Render prediction-versus-true target distribution. + + Parameters + ---------- + title : str, default="" + Optional section title. + width : int, default=700 + Plot width in pixels. + height : int, default=500 + Plot height in pixels. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and target distribution content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_target_distribution() + """ + self._require_explainer("target_distribution") + if self.y_test is None or self.y_pred is None: + raise ValueError("target_distribution block requires y_test and predicted values from the explainer.") + + if self.target_name is None: + target_name = "target" + else: + target_name = self.target_name + df_target = pd.concat( + [ + pd.DataFrame({target_name: self.y_pred}).assign(_dataset="pred"), + pd.DataFrame({target_name: self.y_test}).assign(_dataset="true"), + ] + ).reset_index(drop=True) + fig = plot_distribution( + df_all=df_target, + col=target_name, + hue="_dataset", + colors_dict=TARGET_DISTRIBUTION_COLORS, + width=width, + height=height, + ) + if title is None: + return "Target distribution", [fig] + return title, [fig] + + @block + def block_target_analysis( + self, + title: str = "Target analysis", + show_train: bool = True, + width: int = 700, + height: int = 500, + ) -> BlockContent: + """Render target statistics and target distribution analysis. + + Parameters + ---------- + title : str, default="Target analysis" + Section title displayed above target analysis elements. + show_train : bool, default=True + Whether training target information is included. + width : int, default=700 + Plot width in pixels. + height : int, default=500 + Plot height in pixels. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and target-analysis content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_target_analysis(show_train=False) + """ + if self.y_test is None: + raise ValueError("target_analysis block requires y_test.") + + if self.target_name is None: + target_name = "target" + else: + target_name = self.target_name + y_test_series = pd.Series(self.y_test, name=target_name) + y_train_series = pd.Series(self.y_train, name=target_name) if self.y_train is not None and show_train else None + + analysis_source = pd.DataFrame({target_name: y_test_series}) + if y_train_series is not None: + analysis_source = pd.concat( + [analysis_source, pd.DataFrame({target_name: y_train_series})], ignore_index=True + ) + + col_types = compute_col_types(analysis_source) + test_stats = perform_univariate_dataframe_analysis( + pd.DataFrame({target_name: y_test_series}), col_types=col_types + ) + train_stats = ( + perform_univariate_dataframe_analysis(pd.DataFrame({target_name: y_train_series}), col_types=col_types) + if y_train_series is not None + else None + ) + + names = ["Prediction dataset", "Training dataset"] + target_stats = stats_to_table( + test_stats=test_stats[target_name], + train_stats=train_stats[target_name] if train_stats is not None else None, + names=names, + ) + + distribution_frames = [pd.DataFrame({target_name: y_test_series}).assign(data_train_test="test")] + if y_train_series is not None: + distribution_frames.append(pd.DataFrame({target_name: y_train_series}).assign(data_train_test="train")) + distribution_df = pd.concat(distribution_frames, ignore_index=True) + + fig = plot_distribution( + df_all=distribution_df, + col=target_name, + hue="data_train_test", + colors_dict=self._feature_distribution_colors(), + width=width, + height=height, + ) + fig.update_layout( + title={ + **fig.layout.title.to_plotly_json(), + "x": 0.5, + "xanchor": "center", + "y": 0.0, + "yanchor": "bottom", + }, + margin={**fig.layout.margin.to_plotly_json(), "t": 10, "b": 100}, + ) + + dtype_label = str(series_dtype(y_test_series)) + content = [ + pn.pane.Markdown(f"**{target_name}** ({dtype_label})"), + (target_stats, fig), + ] + return title, content + + @block + def block_confusion_matrix(self, title: str = "") -> BlockContent: + """Render confusion matrix for classification predictions. + + Parameters + ---------- + title : str, default="" + Optional section title. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and confusion matrix content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_confusion_matrix() + """ + explainer = self._require_explainer("confusion_matrix") + if self.y_test is None or self.y_pred is None: + raise ValueError("confusion_matrix block requires y_test and predicted values from the explainer.") + y_test = cast(TargetValues, self.y_test) + y_pred = cast(TargetValues, self.y_pred) + fig = plot_confusion_matrix(y_true=y_test, y_pred=y_pred, colors_dict=explainer.colors_dict) + if title is None: + return "Confusion matrix", [fig] + return title, [fig] + + @block + def block_lift_curve( + self, + title: str = "", + label: int | str = -1, + selection: list[Any] | None = None, + nb: int = 100, + target_fraction: float = 0.1, + max_points: int = 2000, + width: int = 900, + height: int = 600, + ) -> BlockContent: + """Render lift curve for classification probabilities. + + Parameters + ---------- + title : str, default="" + Optional section title. + label : int or str, default=-1 + Class identifier used to select the target probability column. + selection : list[Any] or None, default=None + Optional subset of sample indices to include. + nb : int, default=100 + Number of intervals used to build the curve. + target_fraction : float, default=0.1 + Share of ranked population used to compute Lift@k. + max_points : int, default=2000 + Maximum number of observations used by the plot. + width : int, default=900 + Plot width in pixels. + height : int, default=600 + Plot height in pixels. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and lift curve content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_lift_curve() + """ + explainer = self._require_explainer("lift_curve") + + if getattr(explainer, "_case", None) != "classification": + raise ValueError("lift_curve block is only available for classification case.") + + if explainer.y_target is None: + raise ValueError("lift_curve block requires target values on the explainer.") + + label_num, label_code, label_value = explainer.check_label_name(label) + + if explainer.proba_values is None: + explainer.predict_proba() + + fig = plot_lift_curve( + x_data=explainer.x_init, + y_target=explainer.y_target, + y_proba_values=explainer.proba_values, + style_dict=explainer.plot._style_dict, + selection=selection, + label_num=label_num, + label_code=label_code, + label_value=label_value, + nb=nb, + target_fraction=target_fraction, + max_points=max_points, + width=width, + height=height, + ) + + if title is None: + return "Lift curve", [fig] + return title, [fig] + + @block + def block_univariate_analysis( + self, + title: str = "Univariate analysis", + show_train: bool = True, + ) -> BlockContent: + """Render per-feature univariate analysis with interactive selection. + + Parameters + ---------- + title : str, default="Univariate analysis" + Section title displayed above selector and analysis panel. + show_train : bool, default=True + Whether train statistics are shown alongside prediction statistics. + + Returns + ------- + tuple[str, list[pn.viewable.Viewable]] + Section title and univariate analysis content rendered by the @block decorator. + + Examples + -------- + >>> runtime.block_univariate_analysis() + """ + df_train_test = self._require_train_test_data("univariate_analysis") + explainer = self._require_explainer("univariate_analysis") + + df = df_train_test + col_splitter = "data_train_test" + names = ["Prediction dataset", "Training dataset"] + + col_types = compute_col_types(df) + n_splits = df[col_splitter].nunique() + + test_stats = perform_univariate_dataframe_analysis(df.loc[df[col_splitter] == "test"], col_types=col_types) + train_stats = ( + perform_univariate_dataframe_analysis(df.loc[df[col_splitter] == "train"], col_types=col_types) + if n_splits > 1 and show_train + else None + ) + + list_cols_labels = sorted( + explainer.features_dict.get(col, col) for col in df.drop(col_splitter, axis=1).columns + ) + feature_panels: dict[str, pn.viewable.Viewable] = {} + + for col_label in list_cols_labels: + col = explainer.inv_features_dict.get(col_label, col_label) + if col not in test_stats: + continue + + fig = plot_distribution( + df_all=df, + col=col, + hue=col_splitter, + colors_dict=self._feature_distribution_colors(), + ) + fig.update_layout( + title={ + **fig.layout.title.to_plotly_json(), + "x": 0.5, + "xanchor": "center", + "y": 0.0, + "yanchor": "bottom", + }, + margin={**fig.layout.margin.to_plotly_json(), "t": 10, "b": 100}, + ) + col_stats = stats_to_table( + test_stats=test_stats[col], + train_stats=train_stats[col] if train_stats is not None else None, + names=names, + ) + dtype_label = str(series_dtype(df[col])) + tab_body = pn.Column( + pn.pane.Markdown(f"**{col_label}** ({dtype_label})"), + pn.Row( + _coerce_viewable(col_stats), + _coerce_viewable(fig), + sizing_mode="stretch_width", + ), + sizing_mode="stretch_width", + ) + + base_label = str(col_label) + label_text = base_label + suffix = 2 + while label_text in feature_panels: + label_text = f"{base_label} ({suffix})" + suffix += 1 + feature_panels[label_text] = tab_body + + if len(feature_panels) == 0: + return title, [pn.pane.Markdown("No feature available.")] + + feature_select = pn.widgets.Select( + name="Feature", + options=list(feature_panels.keys()), + value=next(iter(feature_panels)), + sizing_mode="stretch_width", + ) + selected_panel = pn.panel(pn.bind(cast(Any, lambda selected: feature_panels[selected]), feature_select)) + + return title, [feature_select, selected_panel] + + def _preprocess_train_data(self, x_train: pd.DataFrame | None) -> pd.DataFrame | None: + if x_train is None or self.explainer is None: + return x_train + x_train_pre = inverse_transform(x_train, self.explainer.preprocessing) + x_train_pre = handle_categorical_missing(x_train_pre) + if self.explainer.postprocessing: + x_train_pre = apply_postprocessing(x_train_pre, self.explainer.postprocessing) + return x_train_pre + + @staticmethod + def _get_values_and_name( + y: pd.DataFrame | pd.Series | list[Any] | None, default_name: str + ) -> tuple[TargetValues | None, str | None]: + if y is None: + return None, None + if isinstance(y, pd.DataFrame): + if len(y.columns) != 1: + raise ValueError("Number of columns found is greater than 1") + return y.values[:, 0], y.columns[0] + if isinstance(y, pd.Series): + return y.values, y.name + if isinstance(y, list): + return y, default_name + raise ValueError(f"Cannot process following type : {type(y)}") + + @staticmethod + def _create_train_test_df(test: pd.DataFrame | None, train: pd.DataFrame | None) -> pd.DataFrame | None: + if (test is not None and "data_train_test" in test.columns) or ( + train is not None and "data_train_test" in train.columns + ): + raise ValueError('"data_train_test" column must be renamed as it is reserved by smart report runtime') + if test is None and train is None: + return None + frames = [] + if test is not None: + frames.append(test.assign(data_train_test="test")) + if train is not None: + frames.append(train.assign(data_train_test="train")) + return pd.concat(frames).reset_index(drop=True) + + def _require_explainer(self, block_type: str): + if self.explainer is None: + raise ValueError(f"{block_type} block requires an explainer on the report instance.") + return self.explainer + + def _require_train_test_data(self, block_type: str) -> pd.DataFrame: + if self.df_train_test is None: + raise ValueError(f"{block_type} block requires x_train and explainer.x_init data on the report instance.") + return self.df_train_test + + def _resolve_interaction_pair(self, col1: str | None, col2: str | None) -> tuple[str, str]: + if col1 and col2: + return col1, col2 + explainer = self._require_explainer("interactions_plot") + list_ind, _ = explainer.plot._select_indices_interactions_plot(selection=None, max_points=self.max_points) + interaction_values = explainer.get_interaction_values(selection=list_ind) + sorted_indices = compute_sorted_variables_interactions_list_indices(interaction_values) + if not sorted_indices: + raise ValueError("No interaction pair available for interactions_plot block.") + first_idx, second_idx = sorted_indices[0] + return explainer.columns_dict[first_idx], explainer.columns_dict[second_idx] + + def _feature_label(self, feature: str) -> str: + if self.explainer is None: + return feature + return self.explainer.features_dict.get(feature, feature) + + def _feature_distribution_colors(self) -> dict: + explainer = self._require_explainer("feature_distribution") + return explainer.colors_dict["report_feature_distribution"] diff --git a/shapash/report/common.py b/shapash/report/common.py index 7ea1181c..78d3f59c 100644 --- a/shapash/report/common.py +++ b/shapash/report/common.py @@ -1,5 +1,6 @@ import builtins import os +import shutil from collections.abc import Callable from enum import Enum from importlib import import_module @@ -11,6 +12,36 @@ from shapash.utils.dtypes import is_text_like +class ReportTemplate(Enum): + """Report templates list""" + + DEFAULT_REGRESSION = "default_regression_report.yml" + DEFAULT_CLASSIFICATION = "default_classification_report.yml" + + # Other templates could be available + # MINI = "mini_report.yml" + # FULL = "full_report.yml" + # ... + + def __str__(self): + return str(self.value) + + +def export_report_yml(template_id: ReportTemplate | str, output_path: str = "."): + """ + Export a report template YAML file to the given output path. + + Parameters + ---------- + template_id : ReportTemplate + Identifier of the report template to export. + output_path : str, default="." + Destination directory or file path where the template is copied. + """ + template_file_path = os.path.join(os.path.dirname(__file__), "assets", str(template_id)) + shutil.copy(template_file_path, output_path) + + class VarType(Enum): """ Helper class to indicate the type of a variable. diff --git a/shapash/report/core.py b/shapash/report/core.py new file mode 100644 index 00000000..e9084cbd --- /dev/null +++ b/shapash/report/core.py @@ -0,0 +1,156 @@ +"""Smart report orchestration for block-based HTML reports.""" + +from __future__ import annotations + +import base64 +import html +import logging +import re +from pathlib import Path + +import panel as pn + +from shapash.report.panel_support import apply_report_css, report_js_text +from shapash.report.validation import load_report_config + +logger = logging.getLogger(__name__) + + +def generate_report(runtime, config_file: Path, output_file: str) -> None: + """Render a Panel report to an HTML file driven by a YAML config.""" + pn.extension("plotly") + cfg_path = config_file.resolve() + cfg = load_report_config(cfg_path) + print(f"Loading config → {cfg_path}") + + _assign_section_ids(cfg["sections"]) + + rendered_blocks = [runtime.render_block(block_cfg) for block_cfg in cfg["sections"]] + nav_bar = build_navigation_bar(cfg["sections"]) + + out_path = Path(output_file).resolve() + out_path.parent.mkdir(parents=True, exist_ok=True) + + apply_report_css(custom_css=cfg.get("custom_css"), base_dir=cfg_path.parent) + report_content = pn.Column( + *[block for block in rendered_blocks if block is not None], + css_classes=["report-content"], + sizing_mode="stretch_width", + ) + report_layout = pn.Row( + pn.Column(nav_bar, css_classes=["report-sidebar"], width=300, sizing_mode="fixed"), + report_content, + css_classes=["main-report"], + sizing_mode="stretch_width", + ) + report_layout.append(pn.pane.HTML(f"", sizing_mode="stretch_width")) + + with open(str(out_path), mode="w", encoding="utf-8") as f: + report_layout.save(f, embed=True) + + logger.info("Report saved → %s", output_file) + + +def _slugify(text: str) -> str: + """Return a stable slug for navigation anchor IDs.""" + slug = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + return slug + + +def _block_label(block_cfg: dict) -> str: + """Resolve a human-readable block label for the navigation bar.""" + params = block_cfg.get("params", {}) + if isinstance(params, dict): + title = params.get("title") + else: + title = "" + if isinstance(title, str) and title.strip(): + return title.strip() + block_type = block_cfg.get("type", "section") + label = str(block_type).replace("_", " ").strip().title() + if label: + return label + return "Section" + + +def _assign_section_ids(blocks: list[dict], used: set[str] | None = None, prefix: str = "section") -> None: + """Assign unique anchor IDs to all blocks (including group children).""" + used_ids = used if used is not None else set() + for idx, block in enumerate(blocks, start=1): + label_slug = _slugify(_block_label(block)) + if label_slug: + base = label_slug + else: + base = f"{prefix}-{idx}" + candidate = base + suffix = 2 + while candidate in used_ids: + candidate = f"{base}-{suffix}" + suffix += 1 + used_ids.add(candidate) + block["_section_id"] = candidate + if block.get("type") == "group": + children = block.get("blocks", []) + if isinstance(children, list): + _assign_section_ids(children, used=used_ids, prefix=f"{candidate}-item") + + +def _wrap_section_anchor(content: pn.viewable.Viewable, section_id: str | None) -> pn.Column: + """Wrap one rendered block with an in-page anchor target.""" + if not section_id: + return pn.Column(content, css_classes=["scroll-section"], sizing_mode="stretch_width") + anchor = pn.pane.HTML(f'
', sizing_mode="stretch_width") + return pn.Column(anchor, content, css_classes=["scroll-section"], sizing_mode="stretch_width") + + +def build_navigation_bar(blocks: list[dict]) -> pn.pane.HTML: + """Build a sticky in-page navigation bar using Panel HTML pane.""" + items_html: list[str] = [] + item_count = 0 + for block in blocks: + block_type = block.get("type") + label = html.escape(_block_label(block)) + section_id = html.escape(str(block.get("_section_id", ""))) + if block_type == "group": + item_count += 1 + children_links: list[str] = [] + for child in block.get("blocks", []): + child_label = html.escape(_block_label(child)) + child_id = html.escape(str(child.get("_section_id", ""))) + item_count += 1 + children_links.append(f'{child_label}') + items_html.append( + "".join( + [ + '
', + f'{label}', + '
', + *children_links, + "
", + "
", + ] + ) + ) + continue + + item_count += 1 + items_html.append(f'{label}') + + logo_path = Path(__file__).resolve().parent.parent / "style" / "shapash-fond-clair.png" + logo_data = base64.b64encode(logo_path.read_bytes()).decode("ascii") + logo_html = f'' + + nav_scale = max(0.62, min(1.0, 24 / max(1, item_count))) + nav_html = "".join( + [ + f'", + ] + ) + return pn.pane.HTML(nav_html, sizing_mode="stretch_width") diff --git a/shapash/report/generation.py b/shapash/report/generation.py deleted file mode 100644 index ddd6c47f..00000000 --- a/shapash/report/generation.py +++ /dev/null @@ -1,98 +0,0 @@ -""" -Report generation helper module. -""" - -import os -from typing import TYPE_CHECKING - -import pandas as pd -import papermill as pm -from nbconvert import HTMLExporter - -from shapash.utils.utils import get_project_root - -if TYPE_CHECKING: - from shapash.explainer.smart_explainer import SmartExplainer - - -def execute_report( - working_dir: str, - explainer: "SmartExplainer", - project_info_file: str, - x_train: pd.DataFrame | None = None, - y_train: pd.DataFrame | None = None, - y_test: pd.Series | pd.DataFrame | None = None, - config: dict | None = None, - notebook_path: str | None = None, - kernel_name: str | None = None, -): - """ - Executes the base_report.ipynb notebook and saves the results in working_dir. - - Parameters - ---------- - working_dir : str - Directory in which will be saved the executed notebook. - explainer : shapash.explainer.smart_explainer.SmartExplainer - Compiled shapash explainer. - project_info_file : str - Path to the file used to display some information about the project in the report. - x_train : pd.DataFrame - DataFrame used for training the model. - y_train : pd.Series or pd.DataFrame - Series of labels in the training set. - y_test : pd.Series or pd.DataFrame - Series of labels in the test set. - config : dict, optional - Report configuration options. - notebook_path : str, optional - Path to the notebook used to generate the report. If None, the Shapash base report - notebook will be used. - kernel_name : str, optional - Name of the kernel used to generate the report. This parameter can be usefull if - you have multiple jupyter kernels and that the method does not use the right kernel - by default. - """ - if config is None: - config = {} - explainer.save(path=os.path.join(working_dir, "smart_explainer.pickle")) - if x_train is not None: - x_train.to_csv(os.path.join(working_dir, "x_train.csv")) - if y_train is not None: - y_train.to_csv(os.path.join(working_dir, "y_train.csv")) - if y_test is not None: - y_test.to_csv(os.path.join(working_dir, "y_test.csv")) - root_path = get_project_root() - if notebook_path is None or notebook_path == "": - notebook_path = os.path.join(root_path, "shapash", "report", "base_report.ipynb") - - pm.execute_notebook( - notebook_path, - os.path.join(working_dir, "base_report.ipynb"), - parameters=dict(dir_path=working_dir, project_info_file=project_info_file, config=config), - kernel_name=kernel_name, - ) - - -def export_and_save_report(working_dir: str, output_file: str): - """ - Exports a previously executed notebook and saves it as a static HTML file. - - Parameters - ---------- - working_dir : str - Path to the directory containing the executed notebook. - output_file : str - Path to the html file that will be created. - """ - - exporter = HTMLExporter( - exclude_input=True, - extra_template_basedirs=[os.path.join(get_project_root(), "shapash", "report", "template")], - template_name="custom", - exclude_anchor_links=True, - ) - (body, resources) = exporter.from_filename(filename=os.path.join(working_dir, "base_report.ipynb")) - - with open(output_file, "w") as file: - file.write(body) diff --git a/shapash/report/html/double_table.html b/shapash/report/html/double_table.html deleted file mode 100644 index a4acd513..00000000 --- a/shapash/report/html/double_table.html +++ /dev/null @@ -1,12 +0,0 @@ -
-
- {% with columns=columns1, rows=rows1 %} - {% include "table_two_columns.html" %} - {% endwith %} -
-
- {% with columns=columns2, rows=rows2 %} - {% include "table_two_columns.html" %} - {% endwith %} -
-
diff --git a/shapash/report/html/dropdown.html b/shapash/report/html/dropdown.html deleted file mode 100644 index 201da706..00000000 --- a/shapash/report/html/dropdown.html +++ /dev/null @@ -1,19 +0,0 @@ -
-
- - -
-
diff --git a/shapash/report/html/explainability.html b/shapash/report/html/explainability.html deleted file mode 100644 index a490a45e..00000000 --- a/shapash/report/html/explainability.html +++ /dev/null @@ -1,65 +0,0 @@ -{% if labels|length > 1 %} -{% with menuId='dropdownMenuLabel', menuText='Response', values=labels, menuDivVisible='explain-all' %} -{% include "dropdown.html" %} -{% endwith %} -{% else %} -{% endif %} -

Global feature importance plot

-{% for label in labels %} -
- {{ label['feature_importance_plot'] }} -
-{% endfor %} -

Features contribution plots

-{% for label in labels %} -
- {% with menuId='dropdownMenu2', menuText='Feature', values=label['features'], - menuDivVisible='explain-contrib-'~label['index'] %} - {% include "dropdown.html" %} - {% endwith %} - {% for col in label['features'] %} -
-

{{ col['name'] }} - {{ col['type'] }}

- {% if col['name'] != col['description'] %} -
{{ col['description'] }}
- {% else %} - {% endif %} - {{ col['plot'] }} -
- {% endfor %} -
-{% endfor %} -{% set has_interaction = false %} -{% for label in labels %} -{% if label['features_interaction']|length > 0 %} -{% set has_interaction = true %} -{% endif %} -{% endfor %} - -{% if has_interaction %} -

Features Top Interaction plots

-{% for label in labels %} -{% if label['features_interaction']|length > 0 %} -
- {% with menuId='dropdownMenu3', menuText='Interactions', values=label['features_interaction'], - menuDivVisible='explain-contrib-interaction-'~label['index'] %} - {% include "dropdown.html" %} - {% endwith %} - {% for col in label['features_interaction'] %} -
-

{{ col['name'] }} - {{ col['type'] }}

- {% if col['name'] != col['description'] %} -
{{ col['description'] }}
- {% endif %} - {{ col['plot'] }} -
- {% endfor %} -
-{% endif %} -{% endfor %} -{% endif %} diff --git a/shapash/report/html/table_two_columns.html b/shapash/report/html/table_two_columns.html deleted file mode 100644 index 962641da..00000000 --- a/shapash/report/html/table_two_columns.html +++ /dev/null @@ -1,19 +0,0 @@ - - - {% if columns %} - - {% for col in columns %} - - {% endfor %} - - {% endif %} - - - {% for row in rows %} - - - - - {% endfor %} - -
{{ col }}
{{ row['name'] }}{{ row['value'] }}
diff --git a/shapash/report/html/univariate.html b/shapash/report/html/univariate.html deleted file mode 100644 index b29da33f..00000000 --- a/shapash/report/html/univariate.html +++ /dev/null @@ -1,18 +0,0 @@ -{% if features|length > 1 %} -{% with menuId='dropdownMenu1', menuText='Feature', values=features, menuDivVisible=groupId %} - {% include "dropdown.html" %} -{% endwith %} -{% endif %} -{% for col in features %} -
-

{{ col['name'] }} - {{ col['type'] }}

- {% if col['name'] != col['description'] and col['description']|length %} -
{{ col['description'] }}
- {% else %} - {% endif %} -
-
{{ col['table'] }}
-
{{ col['image'] }}
-
-
-{% endfor %} diff --git a/shapash/report/panel_support.py b/shapash/report/panel_support.py new file mode 100644 index 00000000..fe4414ce --- /dev/null +++ b/shapash/report/panel_support.py @@ -0,0 +1,172 @@ +"""Panel helpers for smart report rendering.""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +import pandas as pd +import panel as pn +import plotly.graph_objs as go + +# Use Tabulator only for wide tables; smaller tables keep the simpler DataFrame pane rendering. +TABULATOR_MIN_COLUMNS = 10 + + +def report_js_text() -> str: + """Load report JavaScript once for Panel report export.""" + js_path = Path(__file__).resolve().parent / "assets" / "report_script.js" + return js_path.read_text(encoding="utf-8") + + +def _resolve_custom_css_paths( + custom_css: str | Path | Iterable[str | Path] | None, + base_dir: str | Path | None, +) -> list[Path]: + if custom_css is None: + return [] + + values: Iterable[str | Path] + if isinstance(custom_css, str | Path): + values = [custom_css] + else: + values = custom_css + + resolved: list[Path] = [] + base_path = Path(base_dir).resolve() if base_dir is not None else None + for value in values: + css_path = Path(value) + if not css_path.is_absolute() and base_path is not None: + css_path = base_path / css_path + css_path = css_path.resolve() + if not css_path.exists(): + raise FileNotFoundError(f"Custom CSS file not found: {css_path}") + if css_path.suffix.lower() != ".css": + raise ValueError(f"Custom CSS file must use .css extension: {css_path}") + resolved.append(css_path) + return resolved + + +def apply_report_css( + custom_css: str | Path | Iterable[str | Path] | None = None, + base_dir: str | Path | None = None, +) -> None: + """Register smart-report CSS in Panel global configuration.""" + css_paths = [Path(__file__).resolve().parent / "assets" / "report_styles.css"] + css_paths.extend(_resolve_custom_css_paths(custom_css=custom_css, base_dir=base_dir)) + + for css_path in css_paths: + css = css_path.read_text(encoding="utf-8") + if css not in pn.config.raw_css: + pn.config.raw_css.append(css) + + +def _dedupe_css_classes(*class_groups: Any) -> list[str]: + classes: list[str] = [] + for group in class_groups: + if not group: + continue + if isinstance(group, str): + items = [group] + else: + items = list(group) + for item in items: + if item and item not in classes: + classes.append(item) + return classes + + +def _add_css_classes(viewable: pn.viewable.Viewable, *classes: str) -> pn.viewable.Viewable: + current = getattr(viewable, "css_classes", None) + merged = _dedupe_css_classes(current, classes) + if merged: + viewable.css_classes = merged + return viewable + + +def _auto_style_viewable(viewable: Any, method_name: str | None = None) -> Any: + if isinstance(viewable, pn.pane.Markdown): + return _add_css_classes(viewable, "content-block") + + if isinstance(viewable, pn.pane.DataFrame): + classes = ["kv-table"] + if getattr(viewable, "width_policy", None) == "min": + classes.append("fit-content-table") + return _add_css_classes(viewable, *classes) + + if isinstance(viewable, pn.pane.Plotly): + return viewable + + if isinstance(viewable, pn.widgets.Select): + return viewable + + tabulator_type = getattr(pn.widgets, "Tabulator", None) + if tabulator_type is not None and isinstance(viewable, tabulator_type): + return _add_css_classes(viewable, "kv-table") + + if isinstance(viewable, pn.Spacer): + return viewable + + param_function_type = getattr(pn.param, "ParamFunction", None) + if param_function_type is not None and isinstance(viewable, param_function_type): + return viewable + + param_method_type = getattr(pn.param, "ParamMethod", None) + if param_method_type is not None and isinstance(viewable, param_method_type): + return viewable + + if isinstance(viewable, pn.Row): + if method_name == "block_badge_row": + for child in getattr(viewable, "objects", []): + if isinstance(child, pn.pane.Markdown): + _add_css_classes(child, "badge-pill") + return viewable + + if isinstance(viewable, pn.Column): + if method_name == "block_project_information": + _add_css_classes(viewable, "project-info-grid") + for child in getattr(viewable, "objects", []): + if isinstance(child, pn.Column): + _add_css_classes(child, "project-info-card") + for grandchild in getattr(child, "objects", []): + _auto_style_viewable(grandchild, method_name=method_name) + else: + _auto_style_viewable(child, method_name=method_name) + return viewable + + for child in getattr(viewable, "objects", []): + _auto_style_viewable(child, method_name=method_name) + return viewable + + method_info = f" in '{method_name}'" if method_name else "" + allowed_types = "Markdown, DataFrame, Plotly, Select, Tabulator, Spacer, ParamFunction, ParamMethod, Row, Column" + raise TypeError( + f"Unsupported Panel object type returned{method_info}: {type(viewable).__name__}. " + f"Allowed Panel return types: {allowed_types}." + ) + + +def _coerce_viewable(item: Any) -> pn.viewable.Viewable: + if isinstance(item, pn.viewable.Viewable): + return item + if isinstance(item, str): + return pn.pane.Markdown(item) + if isinstance(item, pd.DataFrame): + tabulator_type = getattr(pn.widgets, "Tabulator", None) + if tabulator_type is not None and item.shape[1] > TABULATOR_MIN_COLUMNS: + return tabulator_type( + item, + disabled=True, + show_index=False, + layout="fit_columns", + width_policy="max", + sizing_mode="stretch_width", + ) + return pn.pane.DataFrame(item, index=False, width_policy="min", sizing_mode="stretch_width") + if isinstance(item, go.Figure): + return pn.pane.Plotly(item, config={"responsive": True}, sizing_mode="stretch_width") + raise TypeError( + f"Unsupported block return type: {type(item).__name__}. " + "Supported types: strings, pandas DataFrame, Plotly Figures, Panel Viewable." + ) diff --git a/shapash/report/project_report.py b/shapash/report/project_report.py deleted file mode 100644 index a5864501..00000000 --- a/shapash/report/project_report.py +++ /dev/null @@ -1,584 +0,0 @@ -import importlib.metadata -import logging -import os -import sys -from datetime import date -from numbers import Number -from typing import cast - -import jinja2 -import numpy as np -import pandas as pd -import plotly - -from shapash import SmartExplainer -from shapash.plots.plot_evaluation_metrics import plot_confusion_matrix -from shapash.plots.plot_univariate import plot_distribution -from shapash.report.common import compute_col_types, display_value, get_callable, series_dtype -from shapash.report.data_analysis import perform_global_dataframe_analysis, perform_univariate_dataframe_analysis -from shapash.report.visualisation import ( - print_css_style, - print_html, - print_javascript_misc, - print_md, -) -from shapash.utils.io import load_yml -from shapash.utils.transform import apply_postprocessing, handle_categorical_missing, inverse_transform -from shapash.utils.utils import compute_sorted_variables_interactions_list_indices, get_project_root, truncate_str -from shapash.webapp.utils.utils import round_to_k - -logging.basicConfig(level=logging.INFO) - -template_loader = jinja2.FileSystemLoader(searchpath=os.path.join(get_project_root(), "shapash", "report", "html")) -template_env = jinja2.Environment(loader=template_loader, autoescape=True) - - -class ProjectReport: - """ - The ProjectReport class allows to generate general information about a - Data Science project. - It analyzes the data and the model used in order to provide interesting - insights that can be shared with non technical person. - - Parameters - ---------- - explainer : shapash.explainer.smart_explainer.SmartExplainer - A shapash SmartExplainer object that has already be compiled. - project_info_file : str - Path to the yml file containing information about the project (author, description, ...). - config : dict, optional - Contains configuration options for the report. - - Attributes - ---------- - explainer : shapash.explainer.smart_explainer.SmartExplainer - A shapash SmartExplainer object that has already be compiled. - metadata : dict - Information about the project (author, description, ...). - x_train : pd.DataFrame - DataFrame used for training the model. - y_train : pd.Series or pd.DataFrame - Series of labels in the train set. - y_test : pd.Series or pd.DataFrame - Series of labels in the test set. - config : dict, optional - Configuration options for the report. - - """ - - def __init__( - self, - explainer: SmartExplainer, - project_info_file: str, - x_train: pd.DataFrame | None = None, - y_train: pd.DataFrame | None = None, - y_test: pd.DataFrame | None = None, - config: dict | None = None, - ): - self.explainer = explainer - self.metadata = load_yml(path=project_info_file) - self.x_train_init = x_train - if x_train is not None: - x_train_pre = inverse_transform(x_train, self.explainer.preprocessing) - self.x_train_pre = handle_categorical_missing(x_train_pre) - - if self.explainer.postprocessing: - self.x_train_pre = apply_postprocessing(self.x_train_pre, self.explainer.postprocessing) - else: - self.x_train_pre = None - self.x_init = self.explainer.x_init - self.config = config if config is not None else dict() - self.col_names = list(self.explainer.columns_dict.values()) - # x_init is always set on a compiled explainer, so `test` is never None here and - # `_create_train_test_df` cannot return None. - self.df_train_test = cast(pd.DataFrame, self._create_train_test_df(test=self.x_init, train=self.x_train_pre)) - if self.explainer.y_pred is not None: - self.y_pred = np.array(self.explainer.y_pred.T)[0] - else: - self.y_pred = self.explainer.model.predict(self.explainer.x_encoded) - self.y_test, target_name_test = self._get_values_and_name(y_test, "target") - self.y_train, target_name_train = self._get_values_and_name(y_train, "target") - self.target_name = target_name_train or target_name_test - - if "max_points" in self.config.keys(): - self.max_points = self.config["max_points"] - else: - self.max_points = 200 - - if "display_interaction_plot" in self.config.keys(): - self.display_interaction_plot = self.config["display_interaction_plot"] - else: - self.display_interaction_plot = False - - if "nb_top_interactions" in self.config.keys(): - self.nb_top_interactions = self.config["nb_top_interactions"] - else: - self.nb_top_interactions = 5 - - if "title_story" in self.config.keys(): - self.title_story = self.config["title_story"] - elif self.explainer.title_story != "": - self.title_story = self.explainer.title_story - else: - self.title_story = "Shapash report" - self.title_description = self.config["title_description"] if "title_description" in self.config.keys() else "" - - print_css_style() - print_javascript_misc() - - if "metrics" in self.config.keys(): - if not isinstance(self.config["metrics"], list) or not isinstance(self.config["metrics"][0], dict): - raise ValueError("The metrics parameter expects a list of dict.") - for metric in self.config["metrics"]: - for key in metric: - if key not in ["path", "name", "use_proba_values"]: - raise ValueError(f"Unknown key : {key}. Key should be in ['path', 'name', 'use_proba_values']") - if key == "use_proba_values" and not isinstance(metric["use_proba_values"], bool): - raise ValueError('"use_proba_values" metric key expects a boolean value.') - - @staticmethod - def _get_values_and_name( - y: pd.DataFrame | pd.Series | list | None, default_name: str - ) -> tuple[list | None, str | None]: - """ - Extracts vales and column name from a Pandas Series, DataFrame, or assign a default - name if y is a list of values. - - Parameters - ---------- - y : list or pd.Series or pd.DataFrame - Column we want to extract the name and values - default_name : - Name assigned if no name was found for y - - Returns - ------- - values : list - list of values of y - name : str - name of y - """ - if y is None: - return None, None - elif isinstance(y, pd.DataFrame): - if len(y.columns) != 1: - raise ValueError("Number of columns found is greater than 1") - name = y.columns[0] - values = y.values[:, 0] - elif isinstance(y, pd.Series): - name = y.name - values = y.values - elif isinstance(y, list): - name = default_name - values = y - else: - raise ValueError(f"Cannot process following type : {type(y)}") - return values, name - - @staticmethod - def _create_train_test_df(test: pd.DataFrame | None, train: pd.DataFrame | None) -> pd.DataFrame | None: - """ - Creates a DataFrame that contains train and test dataset with the column 'data_train_test' - allowing to distinguish the values. - - Parameters - ---------- - test : pd.DataFrame, optional - test dataframe - train : pd.DataFrame, optional - train dataframe - - Returns - ------- - pd.DataFrame - The concatenation of train and test as a dataframe containing train and test values with - a new 'data_train_test' column allowing to distinguish the values. - """ - if (test is not None and "data_train_test" in test.columns) or ( - train is not None and "data_train_test" in train.columns - ): - raise ValueError('"data_train_test" column must be renamed as it is used in ProjectReport') - if test is None and train is None: - return None - return pd.concat( - [ - test.assign(data_train_test="test") if test is not None else None, - train.assign(data_train_test="train") if train is not None else None, - ] - ).reset_index(drop=True) - - def display_title_description(self): - """ - Displays title of the report and its description if defined. - """ - print_html(f"""

{self.title_story}

""") - if self.title_description != "": - print_html(f'
{self.title_description}
') - - def display_project_information(self): - """ - Displays general information about the project as defined in the metdata file. - """ - for section in self.metadata.keys(): - print_md(f"## {section.title()}") - for k, v in self.metadata[section].items(): - if k.lower() == "date" and v.lower() == "auto": - print_md(f"**{k.title()}** : {date.today()}") - else: - print_md(f"**{k.title()}** : {v}") - print_md("---") - - def display_model_analysis(self): - """ - Displays information about the model used : class name, library name, library version, - model parameters, ... - """ - print_md(f"**Model used :** {self.explainer.model.__class__.__name__}") - - print_md(f"**Library :** {self.explainer.model.__class__.__module__}") - - for _, module in sorted(sys.modules.items()): - if not hasattr(module, "__name__"): - continue - - module_name = module.__name__.split(".")[0] - expected_name = self.explainer.model.__class__.__module__.split(".")[0] - - if expected_name == module_name: - try: - package_name = "scikit-learn" if module_name == "sklearn" else module_name - version = importlib.metadata.version(package_name) - print_md(f"**Library version :** {version}") - except importlib.metadata.PackageNotFoundError: - print_md(f"**Library version :** not found for {module_name}") - break - - print_md("**Model parameters :** ") - model_params = self.explainer.model.__dict__ - table_template = template_env.get_template("double_table.html") - print_html( - table_template.render( - columns1=["Parameter key", "Parameter value"], - rows1=[ - {"name": truncate_str(str(k), 50), "value": truncate_str(str(v), 300)} - for k, v in list(model_params.items())[: len(model_params) // 2 :] - ], # Getting half of the parameters - columns2=["Parameter key", "Parameter value"], - rows2=[ - {"name": truncate_str(str(k), 50), "value": truncate_str(str(v), 300)} - for k, v in list(model_params.items())[len(model_params) // 2 :] - ], # Getting 2nd half of the parameters - ) - ) - print_md("---") - - def display_dataset_analysis( - self, - global_analysis: bool = True, - univariate_analysis: bool = True, - target_analysis: bool = True, - multivariate_analysis: bool = True, - ): - """ - This method performs and displays an exploration of the data given. - It allows to compare train and test values for each part of the analysis. - - The parameters of the method allow to filter which part to display or not. - - Parameters - ---------- - global_analysis : bool - Whether or not to display the global analysis part. - univariate_analysis : bool - Whether or not to display the univariate analysis part. - target_analysis : bool - Whether or not to display the target analysis part that plots - the distribution of the target variable. - multivariate_analysis : bool - Whether or not to display the multivariate analysis part - """ - if global_analysis: - print_md("### Global analysis") - self._display_dataset_analysis_global() - - if univariate_analysis: - print_md("### Univariate analysis") - self._perform_and_display_analysis_univariate( - df=self.df_train_test, - col_splitter="data_train_test", - split_values=["test", "train"], - names=["Prediction dataset", "Training dataset"], - group_id="univariate", - ) - if target_analysis: - df_target = self._create_train_test_df( - test=( - pd.DataFrame({self.target_name: self.y_test}, index=range(len(self.y_test))) - if self.y_test is not None - else None - ), - train=( - pd.DataFrame({self.target_name: self.y_train}, index=range(len(self.y_train))) - if self.y_train is not None - else None - ), - ) - if df_target is not None: - if target_analysis: - print_md("### Target analysis") - self._perform_and_display_analysis_univariate( - df=df_target, - col_splitter="data_train_test", - split_values=["test", "train"], - names=["Prediction dataset", "Training dataset"], - group_id="target", - ) - if multivariate_analysis: - print_md("### Multivariate analysis") - fig_corr = self.explainer.plot.correlations_plot( - self.df_train_test, - optimized=True, - facet_col="data_train_test", - max_features=20, - width=900 if len(self.df_train_test["data_train_test"].unique()) > 1 else 500, - height=500, - ) - print_html(plotly.io.to_html(fig_corr)) - print_md("---") - - def _display_dataset_analysis_global(self): - df_stats_global = self._stats_to_table( - test_stats=perform_global_dataframe_analysis(self.x_init), - train_stats=perform_global_dataframe_analysis(self.x_train_pre), - names=["Prediction dataset", "Training dataset"], - ) - print_html(df_stats_global.to_html(classes="greyGridTable")) - - def _perform_and_display_analysis_univariate( - self, df: pd.DataFrame, col_splitter: str, split_values: list, names: list, group_id: str - ): - col_types = compute_col_types(df) - n_splits = df[col_splitter].nunique() - inv_columns_dict = {v: k for k, v in self.explainer.columns_dict.items()} - test_stats_univariate = perform_univariate_dataframe_analysis( - df.loc[df[col_splitter] == split_values[0]], col_types=col_types - ) - if n_splits > 1: - train_stats_univariate = perform_univariate_dataframe_analysis( - df.loc[df[col_splitter] == split_values[1]], col_types=col_types - ) - - univariate_template = template_env.get_template("univariate.html") - univariate_features_desc = list() - list_cols_labels = [ - self.explainer.features_dict.get(col, col) for col in df.drop(col_splitter, axis=1).columns.to_list() - ] - for col_label in sorted(list_cols_labels): - col = self.explainer.inv_features_dict.get(col_label, col_label) - fig = plot_distribution( - df_all=df, - col=col, - hue=col_splitter, - colors_dict=self.explainer.colors_dict["report_feature_distribution"], - ) - df_col_stats = self._stats_to_table( - test_stats=test_stats_univariate[col], - train_stats=train_stats_univariate[col] if n_splits > 1 else None, - names=names, - ) - - univariate_features_desc.append( - { - "feature_index": int(inv_columns_dict.get(col, 0)), - "name": col, - "type": str(series_dtype(df[col])), - "description": col_label, - "table": df_col_stats.to_html(classes="greyGridTable"), - "image": plotly.io.to_html(fig, include_plotlyjs=False, full_html=False), - } - ) - print_html(univariate_template.render(features=univariate_features_desc, groupId=group_id)) - - @staticmethod - def _stats_to_table( - test_stats: dict, - names: list, - train_stats: dict | None = None, - ) -> pd.DataFrame: - if train_stats is not None: - return pd.DataFrame({names[1]: pd.Series(train_stats), names[0]: pd.Series(test_stats)}) - else: - return pd.DataFrame({names[0]: pd.Series(test_stats)}) - - def display_model_explainability(self): - """ - Displays explainability of the model as computed in SmartPlotter object - """ - print_md("*Note : the explainability graphs were generated using the test set only.*") - explainability_template = template_env.get_template("explainability.html") - inv_columns_dict = {v: k for k, v in self.explainer.columns_dict.items()} - explain_data = list() - multiclass = True if (self.explainer._classes and len(self.explainer._classes) > 2) else False - c_list = self.explainer._classes if multiclass else [1] # list just used for multiclass - for index_label, label in enumerate(c_list): # Iterating over all labels in multiclass case - label_value = self.explainer.check_label_name(label)[2] if multiclass else "" - - # Feature Importance - fig_features_importance = self.explainer.plot.features_importance(label=label) - - # Contribution Plot - explain_contrib_data = list() - list_cols_labels = [self.explainer.features_dict.get(col, col) for col in self.col_names] - for feature_label in sorted(list_cols_labels): - feature = self.explainer.inv_features_dict.get(feature_label, feature_label) - fig = self.explainer.plot.contribution_plot(feature, label=label, max_points=self.max_points) - # Apparently matkers are not supported during conversion into html - for el in fig.data: - if el.type == "bar": - el.marker.color = "lightgrey" - explain_contrib_data.append( - { - "feature_index": int(inv_columns_dict[feature]), - "name": feature, - "description": self.explainer.features_dict[feature], - "plot": plotly.io.to_html(fig, include_plotlyjs=False, full_html=False), - } - ) - - # Interaction Plot - explain_contrib_data_interaction = list() - if self.display_interaction_plot: - list_ind, _ = self.explainer.plot._select_indices_interactions_plot( - selection=None, max_points=self.max_points - ) - interaction_values = self.explainer.get_interaction_values(selection=list_ind) - sorted_top_features_indices = compute_sorted_variables_interactions_list_indices(interaction_values) - indices_to_plot = sorted_top_features_indices[: self.nb_top_interactions] - - for i, ids in enumerate(indices_to_plot): - id0, id1 = ids - - fig_one_interaction = self.explainer.plot.interactions_plot( - col1=self.explainer.columns_dict[id0], - col2=self.explainer.columns_dict[id1], - max_points=self.max_points, - ) - - explain_contrib_data_interaction.append( - { - "feature_index": i, - "name": self.explainer.columns_dict[id0] + " / " + self.explainer.columns_dict[id1], - "description": self.explainer.features_dict[self.explainer.columns_dict[id0]] - + " / " - + self.explainer.features_dict[self.explainer.columns_dict[id1]], - "plot": plotly.io.to_html(fig_one_interaction, include_plotlyjs=False, full_html=False), - } - ) - - # Aggregating the data - explain_data.append( - { - "index": index_label, - "name": label_value, - "feature_importance_plot": plotly.io.to_html( - fig_features_importance, include_plotlyjs=False, full_html=False - ), - "features": explain_contrib_data, - "features_interaction": explain_contrib_data_interaction, - } - ) - print_html(explainability_template.render(labels=explain_data)) - print_md("---") - - def display_model_performance(self): - """ - Displays the performance of the model. The metrics are computed using the config dict. - - Metrics should be given as a list of dict. Each dict contains they following keys : - 'path' (path to the metric function, ex: 'sklearn.metrics.mean_absolute_error'), - 'name' (optional, name of the metric as displayed in the report), - and 'use_proba_values' (optional, possible values are False (default) or True - if the metric uses proba values instead of predicted values). - - For example : - config['metrics'] = [ - { - 'path': 'sklearn.metrics.mean_squared_error', - 'name': 'Mean absolute error', # Optional : name that will be displayed next to the metric - 'y_pred': 'predicted_values' # Optional - }, - { - 'path': 'Scoring_AP.utils.lift10', # Custom function path - 'name': 'Lift10', - 'y_pred': 'proba_values' # Use proba values instead of predicted values - } - ] - """ - if self.y_test is None: - logging.info("No labels given for test set. Skipping model performance part") - return - - print_md("### Univariate analysis of target variable") - df = pd.concat( - [ - pd.DataFrame({self.target_name: self.y_pred}).assign(_dataset="pred"), - ( - pd.DataFrame({self.target_name: self.y_test}).assign(_dataset="true") - if self.y_test is not None - else None - ), - ] - ).reset_index(drop=True) - self._perform_and_display_analysis_univariate( - df=df, - col_splitter="_dataset", - split_values=["pred", "true"], - names=["Prediction values", "True values"], - group_id="target-distribution", - ) - - if "metrics" not in self.config.keys(): - logging.info("No 'metrics' key found in report config dict. Skipping model performance part.") - return - print_md("### Metrics") - - for metric in self.config["metrics"]: - if "name" not in metric.keys(): - metric["name"] = metric["path"] - - if ( - metric["path"] in ["confusion_matrix", "sklearn.metrics.confusion_matrix"] - or metric["name"] == "confusion_matrix" - ): - print_md(f"**{metric['name']} :**") - fig = plot_confusion_matrix( - y_true=self.y_test, y_pred=self.y_pred, colors_dict=self.explainer.colors_dict - ) - print_html(plotly.io.to_html(fig, include_plotlyjs=False, full_html=False)) - else: - try: - metric_fn = get_callable(path=metric["path"]) - # Look if we should use proba values instead of predicted values - if "use_proba_values" in metric.keys() and metric["use_proba_values"] is True: - y_pred = self.explainer.proba_values - else: - y_pred = self.y_pred - res = metric_fn(self.y_test, y_pred) - except Exception as e: - logging.info(f"Could not compute following metric : {metric['path']}. \n{e}") - continue - if isinstance(res, Number): - res = display_value(round_to_k(res, 3)) - print_md(f"**{metric['name']} :** {res}") - elif isinstance(res, list | tuple | np.ndarray): - print_md(f"**{metric['name']} :**") - print_html(pd.DataFrame(res).to_html(classes="greyGridTable")) - elif isinstance(res, str): - print_md(f"**{metric['name']} :**") - print_html(f"
{res}
") - else: - logging.info( - f"Could not compute following metric : {metric['path']}. \n" - f"Result of type {res} cannot be displayed" - ) - print_md("---") diff --git a/shapash/report/template/custom/conf.json b/shapash/report/template/custom/conf.json deleted file mode 100644 index a5974f9f..00000000 --- a/shapash/report/template/custom/conf.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "base_template": "classic", - "mimetypes": { - "text/html": true - }, - "preprocessors": { - "100-pygments": { - "enabled": true, - "type": "nbconvert.preprocessors.CSSHTMLHeaderPreprocessor" - } - } -} diff --git a/shapash/report/template/custom/index.html.j2 b/shapash/report/template/custom/index.html.j2 deleted file mode 100644 index 58a6be6f..00000000 --- a/shapash/report/template/custom/index.html.j2 +++ /dev/null @@ -1,65 +0,0 @@ -{%- extends 'classic/index.html.j2' -%} - -{%- block header -%} - -{{ super() }} - - - - - - -{%- endblock header -%} - -{% block body_header %} - -
-
-
-
- -
- -
-
-

Shapash Report

-
- -
- -
-
-
-
-{% endblock body_header %} - -{% block input_group -%} -{% endblock input_group %} - -{% block body_footer %} -
-
-
-
-
- -{% endblock body_footer %} diff --git a/shapash/report/validation.py b/shapash/report/validation.py new file mode 100644 index 00000000..5ea058a0 --- /dev/null +++ b/shapash/report/validation.py @@ -0,0 +1,85 @@ +"""Validation of the yaml configuration and helper functions for report rendering.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd +import panel as pn +import yaml + + +def load_report_config(cfg_path: Path) -> dict: + """Load and validate a report YAML configuration file.""" + if not cfg_path.exists(): + raise FileNotFoundError(f"Config not found: {cfg_path}") + + try: + with cfg_path.open(encoding="utf-8") as file: + cfg = yaml.safe_load(file) + except yaml.YAMLError as exc: + raise ValueError(f"Invalid YAML syntax in '{cfg_path}': {exc}") from exc + + validate_report_schema(cfg, cfg_path) + return cfg + + +def validate_report_schema(cfg: object, cfg_path: Path) -> None: + """Validate the minimal schema expected by the report renderer.""" + if not isinstance(cfg, dict): + raise ValueError(f"Invalid YAML structure in '{cfg_path}': top-level content must be a mapping.") + + sections = cfg.get("sections") + if not isinstance(sections, list) or not sections: + raise ValueError(f"Invalid YAML structure in '{cfg_path}': 'sections' must be a non-empty list.") + + for idx, block in enumerate(sections, start=1): + _validate_block(block, idx, cfg_path) + + +def _validate_block(block: object, idx: int, cfg_path: Path, parent: str = "sections") -> None: + if not isinstance(block, dict): + raise ValueError(f"Invalid YAML structure in '{cfg_path}': {parent}[{idx}] must be a mapping.") + + block_type = block.get("type") + if not isinstance(block_type, str) or not block_type.strip(): + raise ValueError(f"Invalid YAML structure in '{cfg_path}': {parent}[{idx}].type must be a non-empty string.") + + params = block.get("params", {}) + if not isinstance(params, dict): + raise ValueError(f"Invalid YAML structure in '{cfg_path}': {parent}[{idx}].params must be a mapping.") + + if block_type == "custom": + function_path = block.get("function") + if not isinstance(function_path, str) or not function_path.strip(): + raise ValueError( + f"Invalid YAML structure in '{cfg_path}': {parent}[{idx}].function is required for custom blocks." + ) + + if block_type == "group": + child_blocks = block.get("blocks", []) + if not isinstance(child_blocks, list): + raise ValueError( + f"Invalid YAML structure in '{cfg_path}': {parent}[{idx}].blocks must be a list for group blocks." + ) + for child_idx, child_block in enumerate(child_blocks, start=1): + _validate_block(child_block, child_idx, cfg_path, parent=f"{parent}[{idx}].blocks") + + +def render_block_error(block_id: str, exc: Exception): + """Render a consistent error panel for block failures.""" + return pn.pane.Alert( + f'Block "{block_id}" failed\n\n{exc}', + alert_type="danger", + sizing_mode="stretch_width", + ) + + +def stats_to_table(test_stats: dict, names: list[str], train_stats: dict | None = None) -> pd.DataFrame: + """Build a stats table and drop columns that are entirely missing.""" + if train_stats is not None: + stats_table = pd.DataFrame({names[1]: pd.Series(train_stats), names[0]: pd.Series(test_stats)}) + else: + stats_table = pd.DataFrame({names[0]: pd.Series(test_stats)}) + + return stats_table.dropna(axis=1, how="all") diff --git a/shapash/report/visualisation.py b/shapash/report/visualisation.py deleted file mode 100644 index c26e9838..00000000 --- a/shapash/report/visualisation.py +++ /dev/null @@ -1,146 +0,0 @@ -import base64 -import io - -import matplotlib.pyplot as plt -import pandas as pd -from IPython.display import HTML, Latex, Markdown, display - - -def print_md(text: str): - """ - Renders markdown text. - """ - display(Markdown(text)) - - -def print_latex(text: str): - """ - Renders Latex text. - """ - display(Latex(text)) - - -def print_html(text: str): - """ - Renders HTML text. - """ - display(HTML(text)) - - -def print_css_style(): - """Print the CSS""" - print_html( - """ - - """ - ) - - -def print_javascript_misc(): - """Print the JS""" - print_html( - """ - - """ - ) - - -def convert_fig_to_html(fig): - """Convert Matplotlib figure 'fig' into a tag for HTML use using base64 encoding.""" - s = io.BytesIO() - fig.savefig(s, format="png", bbox_inches="tight") - plt.close() - s = base64.b64encode(s.getvalue()).decode("utf-8").replace("\n", "") - return f'' - - -def html_str_df_and_image(df: pd.DataFrame, fig: plt.Figure) -> str: - """Convert dataframe to HTML display""" - return f""" -
-
{df.to_html(classes="greyGridTable")}
-
{convert_fig_to_html(fig)}
-
- """ - - -def print_figure(fig): - """Print a figure as HTML""" - print_html(convert_fig_to_html(fig)) diff --git a/shapash/style/shapash-fond-clair.png b/shapash/style/shapash-fond-clair.png new file mode 100644 index 00000000..6300ec34 Binary files /dev/null and b/shapash/style/shapash-fond-clair.png differ diff --git a/tests/data/clean_titanic_pandas_3.pkl b/tests/data/clean_titanic_pandas_3.pkl index 3608ee55..0ddac85e 100644 Binary files a/tests/data/clean_titanic_pandas_3.pkl and b/tests/data/clean_titanic_pandas_3.pkl differ diff --git a/tests/data/report_test_config.yml b/tests/data/report_test_config.yml new file mode 100644 index 00000000..55b0dcce --- /dev/null +++ b/tests/data/report_test_config.yml @@ -0,0 +1,10 @@ +sections: + - type: header + params: + title: "Integration report" + subtitle: "default test config" + + - type: project_information + params: + title: "Project information" + project_info_file: "tests/data/metadata.yaml" diff --git a/tests/integration_tests/test_report_generation.py b/tests/integration_tests/test_report_generation.py index caf39504..8f05420b 100644 --- a/tests/integration_tests/test_report_generation.py +++ b/tests/integration_tests/test_report_generation.py @@ -2,17 +2,18 @@ import shutil import tempfile import unittest +from pathlib import Path import catboost as cb import category_encoders as ce import numpy as np import pandas as pd -from category_encoders import OrdinalEncoder +import yaml from shapash import SmartExplainer -from shapash.report.generation import execute_report, export_and_save_report current_path = os.path.dirname(os.path.abspath(__file__)) +report_test_cfg_path = os.path.join(current_path, "../data/report_test_config.yml") class TestGeneration(unittest.TestCase): @@ -32,122 +33,57 @@ def setUp(self): self.xpl.compile(x=df_encoded[["x1", "x2", "x3", "x4"]]) self.df = df_encoded - def test_execute_report_1(self): - tmp_dir_path = tempfile.mkdtemp() - - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), - config=None, - notebook_path=None, - ) - assert os.path.exists(os.path.join(tmp_dir_path, "smart_explainer.pickle")) - assert os.path.exists(os.path.join(tmp_dir_path, "base_report.ipynb")) - - shutil.rmtree(tmp_dir_path) - - def test_execute_report_2(self): - tmp_dir_path = tempfile.mkdtemp() - - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), - x_train=self.df[["x1", "x2", "x3", "x4"]], - config=None, - notebook_path=None, - ) - assert os.path.exists(os.path.join(tmp_dir_path, "x_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "smart_explainer.pickle")) - assert os.path.exists(os.path.join(tmp_dir_path, "base_report.ipynb")) - - shutil.rmtree(tmp_dir_path) - - def test_execute_report_3(self): - tmp_dir_path = tempfile.mkdtemp() - - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), - x_train=self.df[["x1", "x2", "x3", "x4"]], - y_test=self.df["y"], - config=None, - notebook_path=None, - ) - assert os.path.exists(os.path.join(tmp_dir_path, "x_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "y_test.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "smart_explainer.pickle")) - assert os.path.exists(os.path.join(tmp_dir_path, "base_report.ipynb")) - - shutil.rmtree(tmp_dir_path) - - def test_execute_report_4(self): - tmp_dir_path = tempfile.mkdtemp() - - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), - x_train=self.df[["x1", "x2", "x3", "x4"]], - y_train=self.df["y"], - y_test=self.df["y"], - config=None, - notebook_path=None, - ) - assert os.path.exists(os.path.join(tmp_dir_path, "x_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "y_test.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "y_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "smart_explainer.pickle")) - assert os.path.exists(os.path.join(tmp_dir_path, "base_report.ipynb")) - - shutil.rmtree(tmp_dir_path) - - def test_execute_report_5(self): + def test_generate_report_default_config(self): tmp_dir_path = tempfile.mkdtemp() + outfile = os.path.join(tmp_dir_path, "report.html") self.xpl.palette_name = "eurybia" - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), + self.xpl.generate_report( + output_file=outfile, x_train=self.df[["x1", "x2", "x3", "x4"]], y_train=self.df["y"], y_test=self.df["y"], - notebook_path=None, + yaml_path=report_test_cfg_path, ) self.xpl.palette_name = "default" - assert os.path.exists(os.path.join(tmp_dir_path, "x_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "y_test.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "y_train.csv")) - assert os.path.exists(os.path.join(tmp_dir_path, "smart_explainer.pickle")) - assert os.path.exists(os.path.join(tmp_dir_path, "base_report.ipynb")) + assert os.path.exists(outfile) shutil.rmtree(tmp_dir_path) - def test_generate_report_1(self): + def test_generate_report_with_custom_yaml_config(self): tmp_dir_path = tempfile.mkdtemp() - outfile = os.path.join(tmp_dir_path, "report.html") + cfg_path = Path(tmp_dir_path) / "custom_report_config.yml" + custom_css_path = Path(tmp_dir_path) / "custom_report.css" + outfile = str(Path(tmp_dir_path) / "report_custom.html") + + custom_css_marker = ".integration-custom-css-marker{border:0;}" + custom_css_path.write_text(custom_css_marker, encoding="utf-8") + + config = { + "custom_css": str(custom_css_path), + "sections": [ + { + "type": "header", + "params": {"title": "Integration report", "subtitle": "custom yaml"}, + }, + { + "type": "project_information", + "params": { + "title": "Project information", + "project_info_file": os.path.join(current_path, "../data/metadata.yaml"), + }, + }, + ] + } + with cfg_path.open("w", encoding="utf-8") as stream: + yaml.safe_dump(config, stream, sort_keys=False, allow_unicode=True) self.xpl.generate_report( output_file=outfile, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), + yaml_path=str(cfg_path), ) assert os.path.exists(outfile) + report_html = Path(outfile).read_text(encoding="utf-8") + assert custom_css_marker in report_html shutil.rmtree(tmp_dir_path) - - def test_export_and_save_report_1(self): - tmp_dir_path = tempfile.mkdtemp() - - execute_report( - working_dir=tmp_dir_path, - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../data/metadata.yaml"), - ) - - outfile = os.path.join(tmp_dir_path, "report.html") - export_and_save_report(working_dir=tmp_dir_path, output_file=outfile) - assert os.path.exists(outfile) - shutil.rmtree(tmp_dir_path) diff --git a/tests/unit_tests/explainer/test_smart_explainer.py b/tests/unit_tests/explainer/test_smart_explainer.py index 5d1bd1be..c56f7317 100644 --- a/tests/unit_tests/explainer/test_smart_explainer.py +++ b/tests/unit_tests/explainer/test_smart_explainer.py @@ -25,6 +25,7 @@ from shapash.explainer.multi_decorator import MultiDecorator from shapash.explainer.smart_state import SmartState from shapash.utils.check import check_model +from shapash.report.blocks import ReportBlockMixin def init_sme_to_pickle_test(): @@ -1110,9 +1111,8 @@ def test_run_app_2(self, mock_get_host_name, mock_make_server, mock_custom_threa xpl.run_app() assert xpl.y_target is not None - @patch("shapash.report.generation.export_and_save_report") - @patch("shapash.report.generation.execute_report") - def test_generate_report(self, mock_execute_report, mock_export_and_save_report): + @patch("shapash.explainer.smart_explainer.generate_smart_report") + def test_generate_report(self, mock_generate_report): """ Test generate report method """ @@ -1124,9 +1124,41 @@ def test_generate_report(self, mock_execute_report, mock_export_and_save_report) clf = cb.CatBoostClassifier(n_estimators=1).fit(df[["x1", "x2"]], df["y"]) xpl = SmartExplainer(clf) xpl.compile(x=df[["x1", "x2"]]) - xpl.generate_report(output_file="test", project_info_file="test") - mock_execute_report.assert_called_once() - mock_export_and_save_report.assert_called_once() + xpl.generate_report(output_file="test") + runtime_arg = mock_generate_report.call_args.kwargs["runtime"] + assert runtime_arg.explainer is xpl + mock_generate_report.assert_called_once_with( + runtime=runtime_arg, + config_file=unittest.mock.ANY, + output_file="test", + ) + + @patch("shapash.explainer.smart_explainer.generate_smart_report") + def test_generate_report_with_user_block_instance(self, mock_generate_report): + """Custom block runtime provided by user must be passed through unchanged.""" + df = pd.DataFrame(range(0, 21), columns=["id"]) + df["y"] = df["id"].apply(lambda x: 1 if x < 10 else 0) + df["x1"] = np.random.randint(1, 123, df.shape[0]) + df["x2"] = np.random.randint(1, 3, df.shape[0]) + df = df.set_index("id") + clf = cb.CatBoostClassifier(n_estimators=1).fit(df[["x1", "x2"]], df["y"]) + xpl = SmartExplainer(clf) + xpl.compile(x=df[["x1", "x2"]]) + + class _UserRuntime(ReportBlockMixin): + def __init__(self): + super().__init__() + self.user_initialized = True + + def render_block(self, block_cfg): + return None + + block_instance = _UserRuntime() + xpl.generate_report(output_file="test", block_instance=block_instance) + + runtime_arg = mock_generate_report.call_args.kwargs["runtime"] + assert runtime_arg is block_instance + assert runtime_arg.user_initialized is True def test_compute_features_stability_1(self): df = pd.DataFrame(np.random.randint(1, 100, size=(15, 4)), columns=list("ABCD")) diff --git a/tests/unit_tests/report/test_project_report.py b/tests/unit_tests/report/test_project_report.py deleted file mode 100644 index 89f1197a..00000000 --- a/tests/unit_tests/report/test_project_report.py +++ /dev/null @@ -1,250 +0,0 @@ -import os -import unittest -from unittest.mock import patch - -import catboost as cb -import numpy as np -import pandas as pd -from category_encoders import OrdinalEncoder - -from shapash import SmartExplainer -from shapash.report.project_report import ProjectReport - -expected_attrs = [ - "explainer", - "metadata", - "x_train_init", - "y_test", - "x_init", - "config", - "col_names", - "df_train_test", - "title_story", - "title_description", -] - -current_path = os.path.dirname(os.path.abspath(__file__)) - - -class TestProjectReport(unittest.TestCase): - def setUp(self): - self.df = pd.DataFrame(range(0, 21), columns=["id"]) - self.df["y"] = self.df["id"].apply(lambda x: 1 if x < 10 else 0) - self.df["x1"] = np.random.randint(1, 123, self.df.shape[0]) - self.df["x2"] = np.random.randint(1, 3, self.df.shape[0]) - self.df = self.df.set_index("id") - self.clf = cb.CatBoostClassifier(n_estimators=1).fit(self.df[["x1", "x2"]], self.df["y"]) - self.xpl = SmartExplainer(model=self.clf) - self.xpl.compile(x=self.df[["x1", "x2"]]) - self.report1 = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - ) - self.report2 = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - ) - - def test_init_1(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - ) - for attr in expected_attrs: - assert hasattr(report, attr) - - def test_init_2(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - ) - for attr in expected_attrs: - assert hasattr(report, attr) - - def test_init_3(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - y_test=self.df["y"], - ) - for attr in expected_attrs: - assert hasattr(report, attr) - - def test_init_4(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - y_test=self.df["y"], - config={}, - ) - for attr in expected_attrs: - assert hasattr(report, attr) - - def test_init_5(self): - ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - y_test=self.df["y"], - config={"metrics": [{"path": "sklearn.metrics.mean_squared_error"}]}, - ) - - def test_init_6(self): - self.assertRaises( - ValueError, - ProjectReport, - self.xpl, - os.path.join(current_path, "../../data/metadata.yaml"), - self.df[["x1", "x2"]], - self.df["y"], - {"metrics": ["sklearn.metrics.mean_squared_error"]}, - ) - - @patch("shapash.report.project_report.print_html") - def test_display_title_description_1(self, mock_print_html): - self.report1.display_title_description() - mock_print_html.assert_called_once() - - @patch("shapash.report.project_report.print_html") - def test_display_title_description_2(self, mock_print_html): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - y_test=self.df["y"], - config={ - "title_story": "My project report", - "title_description": """This document is a data science project report.""", - }, - ) - report.display_title_description() - self.assertEqual(mock_print_html.call_count, 2) - - @patch("shapash.report.project_report.print_md") - def test_display_general_information_1(self, mock_print_html): - report = ProjectReport( - explainer=self.xpl, project_info_file=os.path.join(current_path, "../../data/metadata.yaml") - ) - report.display_project_information() - self.assertTrue(mock_print_html.called) - - @patch("shapash.report.project_report.print_md") - def test_display_model_information_1(self, mock_print_md): - report = ProjectReport( - explainer=self.xpl, project_info_file=os.path.join(current_path, "../../data/metadata.yaml") - ) - report.display_model_analysis() - self.assertTrue(mock_print_md.called) - - def test_display_dataset_analysis_1(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=self.df[["x1", "x2"]], - ) - report.display_dataset_analysis() - - def test_display_dataset_analysis_2(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - ) - report.display_dataset_analysis() - - def test_display_dataset_analysis_3(self): - """ - Test we don't have a problem when only categorical features - """ - df = self.df.copy() - df["x1"] = "a" - df["x2"] = df["x2"].astype(str) - encoder = OrdinalEncoder(cols=["x1", "x2"], handle_unknown="return_nan", return_df=True).fit(df) - - df = encoder.transform(df) - - clf = cb.CatBoostClassifier(n_estimators=1).fit(df[["x1", "x2"]], df["y"]) - xpl = SmartExplainer(model=clf) - xpl.compile(x=df[["x1", "x2"]]) - report = ProjectReport( - explainer=xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - x_train=df[["x1", "x2"]], - ) - - report.display_dataset_analysis() - - def test_display_model_explainability_1(self): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - ) - report.display_model_explainability() - - def test_display_model_explainability_2(self): - """ - Tests multiclass case - """ - df = pd.DataFrame(range(0, 21), columns=["id"]) - df["y"] = df["id"].apply(lambda x: 0 if x < 5 else 1 if (5 <= x < 10) else 2 if (10 <= x < 15) else 3) - df["x1"] = np.random.randint(1, 123, df.shape[0]) - df["x2"] = np.random.randint(1, 3, df.shape[0]) - df = df.set_index("id") - clf = cb.CatBoostClassifier(n_estimators=1).fit(df[["x1", "x2"]], df["y"]) - xpl = SmartExplainer(model=clf) - xpl.compile(x=df[["x1", "x2"]]) - report = ProjectReport(explainer=xpl, project_info_file=os.path.join(current_path, "../../data/metadata.yaml")) - report.display_model_explainability() - - @patch("shapash.report.project_report.logging") - def test_display_model_performance_1(self, mock_logging): - """ - No y_test given - """ - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - ) - report.display_model_performance() - mock_logging.info.assert_called_once() - - @patch("shapash.report.project_report.logging") - def test_display_model_performance_2(self, mock_logging): - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - y_test=self.df["y"], - config=dict(metrics=[{"path": "sklearn.metrics.mean_squared_error"}]), - ) - report.display_model_performance() - self.assertEqual(mock_logging.call_count, 0) - - @patch("shapash.report.project_report.logging") - def test_display_model_performance_3(self, mock_logging): - """ - No metrics given in ProjectReport - """ - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - y_test=self.df["y"], - ) - report.display_model_performance() - mock_logging.info.assert_called_once() - - @patch("shapash.report.project_report.logging") - def test_display_model_performance_4(self, mock_logging): - """ - Test use of proba values. - """ - report = ProjectReport( - explainer=self.xpl, - project_info_file=os.path.join(current_path, "../../data/metadata.yaml"), - y_test=self.df["y"], - config=dict(metrics=[{"path": "sklearn.metrics.log_loss", "use_proba_values": True}]), - ) - report.display_model_performance() - self.assertEqual(mock_logging.call_count, 0) diff --git a/tests/unit_tests/report/test_report_generation.py b/tests/unit_tests/report/test_report_generation.py new file mode 100644 index 00000000..98d469b0 --- /dev/null +++ b/tests/unit_tests/report/test_report_generation.py @@ -0,0 +1,378 @@ +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import panel as pn +import pandas as pd +import plotly.graph_objects as go + +from shapash.report.blocks import ReportBlockMixin, block +from shapash.report.panel_support import apply_report_css + + +def dummy_metric(y_true, y_pred): + return 0.75 + + +class _DummyModel: + def __init__(self): + self.alpha = 0.1 + self.depth = 4 + + def predict(self, x): + return np.zeros(len(x)) + + +class _DummyPlot: + def __init__(self): + self._style_dict = {"dummy": "style"} + + def correlations_plot(self, *args, **kwargs): + return go.Figure(go.Scatter(x=[1, 2], y=[2, 1])) + + def features_importance(self, *args, **kwargs): + return go.Figure(go.Bar(x=["age", "income"], y=[0.7, 0.3])) + + def contribution_plot(self, *args, **kwargs): + return go.Figure(go.Bar(x=["age"], y=[1.0])) + + def interactions_plot(self, *args, **kwargs): + return go.Figure(go.Scatter(x=[1, 2], y=[3, 4])) + + def top_interactions_plot(self, *args, **kwargs): + return go.Figure(go.Scatter(x=[2, 3], y=[4, 5])) + + def _select_indices_interactions_plot(self, selection=None, max_points=200): + return [0, 1], None + + +class _DummyExplainer: + def __init__(self, x_init): + self.x_init = x_init + self.x_encoded = x_init + self.y_pred = [1, 0, 1] + self.model = _DummyModel() + self.preprocessing = None + self.postprocessing = None + self.features_dict = {"age": "Age", "income": "Income"} + self.inv_features_dict = {"Age": "age", "Income": "income"} + self.colors_dict = { + "report_feature_distribution": {"train": "#f4c000", "test": "#2255aa"}, + "default": "#2255aa", + } + self.plot = _DummyPlot() + self.columns_dict = {0: "age", 1: "income"} + self._case = "classification" + self.y_target = [1, 0, 1] + self.proba_values = None + + def get_interaction_values(self, selection=None): + return np.array([[0.0, 0.5], [0.5, 0.0]]) + + def check_label_name(self, label): + return 1, "class_1", "class_1" + + def predict_proba(self): + self.proba_values = np.array([[0.1, 0.9], [0.8, 0.2], [0.2, 0.8]]) + + +def _build_runtime() -> ReportBlockMixin: + x_train = pd.DataFrame({"age": [20, 30, 40], "income": [100, 200, 150]}) + x_test = pd.DataFrame({"age": [21, 31, 41], "income": [110, 210, 160]}) + y_train = pd.Series([0, 1, 1], name="target") + y_test = pd.Series([1, 0, 1], name="target") + explainer = _DummyExplainer(x_test) + return ReportBlockMixin(explainer=explainer, x_train=x_train, y_train=y_train, y_test=y_test, max_points=10) + + +class TestSmartReportPanel(unittest.TestCase): + + def test_report_css_text_loads_stylesheet_content(self): + css_path = Path(__file__).resolve().parents[3] / "shapash" / "report" / "assets" / "report_styles.css" + css = css_path.read_text(encoding="utf-8") + + self.assertIn(".kv-table", css) + self.assertIn("@media (max-width: 1200px)", css) + + def test_apply_report_css_registers_styles_once(self): + css_path = Path(__file__).resolve().parents[3] / "shapash" / "report" / "assets" / "report_styles.css" + css = css_path.read_text(encoding="utf-8") + + apply_report_css() + first_count = pn.config.raw_css.count(css) + + apply_report_css() + second_count = pn.config.raw_css.count(css) + + self.assertEqual(first_count, 1) + self.assertEqual(second_count, 1) + + +class _DummyBlocks(ReportBlockMixin): + @block + def block_demo(self, title: str = "Demo"): + return [pn.pane.Markdown("Body")] + + @block + def block_dynamic_title(self, title: str = ""): + return "Resolved title", [pn.pane.Markdown("Dynamic body")] + + @block + def block_scalar_body(self, title: str = "Scalar"): + return "plain text" + + @block + def block_table(self, title: str = "Table"): + return [pn.pane.DataFrame(pd.DataFrame({"a": [1], "b": [2]}))] + + @block + def block_badge_row(self, title: str = "Badges"): + return [pn.Row(pn.pane.Markdown("One"), pn.pane.Markdown("Two"))] + + @block + def block_select_allowed(self, title: str = "Selector"): + return [pn.widgets.Select(name="Feature", options=["a", "b"], value="a")] + + @block + def block_plotly_allowed(self, title: str = "Plotly"): + fig = go.Figure(go.Scatter(x=[1, 2], y=[3, 4])) + return [pn.pane.Plotly(fig)] + + @block + def block_bind_allowed(self, title: str = "Bind"): + selector = pn.widgets.Select(name="Feature", options=["a", "b"], value="a") + selected_panel = pn.panel(pn.bind(lambda selected: pn.pane.Markdown(selected), selector)) + return [selector, selected_panel] + + @block + def block_panel_type_not_allowed(self, title: str = "HTML"): + return [pn.pane.HTML("html")] + + @block + def block_non_panel_type_not_allowed(self, title: str = "Object"): + return [object()] + + +class TestBlockDecorator(unittest.TestCase): + def test_block_decorator_wraps_with_title_from_signature(self): + runtime = _DummyBlocks() + + result = runtime.block_demo() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[0], pn.pane.Markdown) + self.assertIn("Demo", result.objects[0].object) + + def test_block_decorator_supports_dynamic_title_tuple(self): + runtime = _DummyBlocks() + + result = runtime.block_dynamic_title() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[0], pn.pane.Markdown) + self.assertIn("Resolved title", result.objects[0].object) + + def test_block_decorator_coerces_scalar_body_to_markdown(self): + runtime = _DummyBlocks() + + result = runtime.block_scalar_body() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[1], pn.pane.Markdown) + self.assertIn("plain text", result.objects[1].object) + + def test_block_decorator_auto_stylizes_body_by_type(self): + runtime = _DummyBlocks() + + text_result = runtime.block_demo() + table_result = runtime.block_table() + + self.assertIn("content-block", text_result.objects[1].css_classes) + self.assertIn("kv-table", table_result.objects[1].css_classes) + + def test_block_decorator_auto_styles_badge_rows(self): + runtime = _DummyBlocks() + + result = runtime.block_badge_row() + + badge_row = result.objects[1] + self.assertIsInstance(badge_row, pn.Row) + self.assertIn("badge-pill", badge_row.objects[0].css_classes) + self.assertIn("badge-pill", badge_row.objects[1].css_classes) + + def test_block_decorator_allows_select_and_plotly(self): + runtime = _DummyBlocks() + + select_result = runtime.block_select_allowed() + plotly_result = runtime.block_plotly_allowed() + + self.assertIsInstance(select_result.objects[1], pn.widgets.Select) + self.assertIsInstance(plotly_result.objects[1], pn.pane.Plotly) + + def test_block_decorator_allows_bind_param_function(self): + runtime = _DummyBlocks() + + result = runtime.block_bind_allowed() + + self.assertIsInstance(result.objects[1], pn.widgets.Select) + self.assertEqual(type(result.objects[2]).__name__, "ParamFunction") + + def test_block_decorator_rejects_panel_type_without_style_definition(self): + runtime = _DummyBlocks() + + with self.assertRaises(TypeError) as context: + runtime.block_panel_type_not_allowed() + + self.assertIn("Unsupported Panel object type returned", str(context.exception)) + self.assertIn("Allowed Panel return types", str(context.exception)) + + def test_block_decorator_rejects_non_panel_return_type(self): + runtime = _DummyBlocks() + + with self.assertRaises(TypeError) as context: + runtime.block_non_panel_type_not_allowed() + + self.assertIn("Unsupported block return type", str(context.exception)) + + +class TestReportBlockMixinBuiltins(unittest.TestCase): + def test_block_text_accepts_dict_content(self): + runtime = _build_runtime() + + result = runtime.block_text(title="Info", content={"project": "shapash", "version": "1.0"}) + + self.assertIsInstance(result, pn.Column) + self.assertIn("Info", result.objects[0].object) + self.assertIn("**project**", result.objects[1].object) + + def test_block_global_analysis_renders_stats_table(self): + runtime = _build_runtime() + fake_stats = {"Rows": 3, "Columns": 2} + + with patch("shapash.report.blocks.perform_global_dataframe_analysis", return_value=fake_stats), patch( + "shapash.report.blocks.stats_to_table", return_value=pd.DataFrame({"Prediction dataset": [3]}) + ): + result = runtime.block_global_analysis(title="Global") + + self.assertIsInstance(result, pn.Column) + self.assertIn("Global", result.objects[0].object) + self.assertIsInstance(result.objects[1], pn.pane.DataFrame) + + def test_block_model_analysis_renders_metadata(self): + runtime = _build_runtime() + + with patch("shapash.report.blocks.importlib.metadata.version", return_value="9.9.9"): + result = runtime.block_model_analysis() + + self.assertIsInstance(result, pn.Column) + self.assertIn("Model information", result.objects[0].object) + self.assertIn("**Model used**", result.objects[1].object) + + def test_block_performance_metrics_builds_badges(self): + runtime = _build_runtime() + + result = runtime.block_performance_metrics( + title="Perf", metrics=[{"path": f"{__name__}.dummy_metric", "name": "Dummy metric"}] + ) + + self.assertIsInstance(result, pn.Column) + self.assertIn("Perf", result.objects[0].object) + row = result.objects[1] + self.assertIsInstance(row, pn.Row) + self.assertIn("Dummy metric", row.objects[0].object) + + def test_block_feature_distribution_uses_feature_label_when_title_is_none(self): + runtime = _build_runtime() + + with patch("shapash.report.blocks.plot_distribution", return_value=go.Figure(go.Scatter(x=[1], y=[1]))): + result = runtime.block_feature_distribution(feature="age", title=None) + + self.assertIsInstance(result, pn.Column) + self.assertIn("Age", result.objects[0].object) + self.assertIsInstance(result.objects[1], pn.pane.Plotly) + + def test_block_correlations_and_feature_importance_return_plotly_panes(self): + runtime = _build_runtime() + + corr_result = runtime.block_correlations_plot(title="Corr") + fi_result = runtime.block_feature_importance(title="FI") + + self.assertIsInstance(corr_result.objects[1], pn.pane.Plotly) + self.assertIsInstance(fi_result.objects[1], pn.pane.Plotly) + + def test_block_contribution_plot_single_and_all_features(self): + runtime = _build_runtime() + + single_result = runtime.block_contribution_plot(feature="age", title=None) + all_result = runtime.block_contribution_plot(include_all_features=True, title="All") + + self.assertIsInstance(single_result, pn.Column) + self.assertIn("Age", single_result.objects[0].object) + self.assertIsInstance(single_result.objects[1], pn.pane.Plotly) + self.assertIsInstance(all_result.objects[1], pn.widgets.Select) + self.assertEqual(type(all_result.objects[2]).__name__, "ParamFunction") + + def test_block_interactions_plot_default_pair_uses_resolved_labels(self): + runtime = _build_runtime() + + with patch( + "shapash.report.blocks.compute_sorted_variables_interactions_list_indices", return_value=[(0, 1)] + ): + result = runtime.block_interactions_plot(title=None) + + self.assertIsInstance(result, pn.Column) + self.assertIn("Age / Income", result.objects[0].object) + self.assertIsInstance(result.objects[1], pn.pane.Plotly) + + def test_block_top_interactions_plot_renders_plotly(self): + runtime = _build_runtime() + + result = runtime.block_top_interactions_plot(title="Top interactions", nb_top_interaction=3) + + self.assertIsInstance(result, pn.Column) + self.assertIn("Top interactions", result.objects[0].object) + self.assertIsInstance(result.objects[1], pn.pane.Plotly) + + def test_block_target_distribution_and_analysis_render(self): + runtime = _build_runtime() + fake_fig = go.Figure(go.Scatter(x=[1], y=[1])) + fake_univariate = {"target": {"count": 3, "na_count": 0}} + + with patch("shapash.report.blocks.plot_distribution", return_value=fake_fig), patch( + "shapash.report.blocks.compute_col_types", return_value={"target": "numeric"} + ), patch("shapash.report.blocks.perform_univariate_dataframe_analysis", return_value=fake_univariate): + dist_result = runtime.block_target_distribution(title=None) + analysis_result = runtime.block_target_analysis(title="Target") + + self.assertIsInstance(dist_result.objects[1], pn.pane.Plotly) + self.assertIsInstance(analysis_result, pn.Column) + self.assertIn("Target", analysis_result.objects[0].object) + self.assertIsInstance(analysis_result.objects[2], pn.Row) + + def test_block_confusion_lift_and_univariate_render(self): + runtime = _build_runtime() + fake_fig = go.Figure(go.Scatter(x=[0, 1], y=[1, 0])) + fake_univariate = { + "age": {"count": 3, "na_count": 0}, + "income": {"count": 3, "na_count": 0}, + "data_train_test": {"count": 6}, + } + + with patch("shapash.report.blocks.plot_confusion_matrix", return_value=fake_fig), patch( + "shapash.report.blocks.plot_lift_curve", return_value=fake_fig + ), patch("shapash.report.blocks.compute_col_types", return_value={"age": "numeric", "income": "numeric"}), patch( + "shapash.report.blocks.perform_univariate_dataframe_analysis", return_value=fake_univariate + ), patch("shapash.report.blocks.plot_distribution", return_value=fake_fig): + confusion_result = runtime.block_confusion_matrix(title="CM") + lift_result = runtime.block_lift_curve(title="Lift") + univariate_result = runtime.block_univariate_analysis() + + self.assertIsInstance(confusion_result.objects[1], pn.pane.Plotly) + self.assertIsInstance(lift_result.objects[1], pn.pane.Plotly) + self.assertIsNotNone(runtime.explainer.proba_values) + self.assertIsInstance(univariate_result.objects[1], pn.widgets.Select) + self.assertEqual(type(univariate_result.objects[2]).__name__, "ParamFunction") diff --git a/tests/unit_tests/report/test_smart_report_panel.py b/tests/unit_tests/report/test_smart_report_panel.py new file mode 100644 index 00000000..30304f5c --- /dev/null +++ b/tests/unit_tests/report/test_smart_report_panel.py @@ -0,0 +1,181 @@ +import unittest +from pathlib import Path +import tempfile + +import panel as pn +import pandas as pd +import plotly.graph_objects as go + +from shapash.report.blocks import ReportBlockMixin, block +from shapash.report.panel_support import apply_report_css + + +class TestSmartReportPanel(unittest.TestCase): + def test_panel_plotly_pane_is_responsive(self): + fig = go.Figure(go.Scatter(x=[1, 2], y=[3, 4])) + + pane = pn.pane.Plotly(fig, config={"responsive": True}, sizing_mode="stretch_width") + + self.assertIsInstance(pane, pn.pane.Plotly) + self.assertEqual(pane.object, fig) + self.assertEqual(pane.sizing_mode, "stretch_width") + + def test_report_css_text_loads_stylesheet_content(self): + css_path = Path(__file__).resolve().parents[3] / "shapash" / "report" / "assets" / "report_styles.css" + css = css_path.read_text(encoding="utf-8") + + self.assertIn(".kv-table", css) + self.assertIn("@media (max-width: 1200px)", css) + + def test_apply_report_css_registers_styles_once(self): + css_path = Path(__file__).resolve().parents[3] / "shapash" / "report" / "assets" / "report_styles.css" + css = css_path.read_text(encoding="utf-8") + + apply_report_css() + first_count = pn.config.raw_css.count(css) + + apply_report_css() + second_count = pn.config.raw_css.count(css) + + self.assertEqual(first_count, 1) + self.assertEqual(second_count, 1) + + def test_apply_report_css_accepts_custom_css_file(self): + marker_css = ".custom-report-marker{outline:1px solid #f00;}" + with tempfile.TemporaryDirectory() as tmp_dir: + custom_css_path = Path(tmp_dir) / "custom.css" + custom_css_path.write_text(marker_css, encoding="utf-8") + + apply_report_css(custom_css="custom.css", base_dir=tmp_dir) + + self.assertIn(marker_css, pn.config.raw_css) + + +class _DummyBlocks(ReportBlockMixin): + @block + def block_demo(self, title: str = "Demo"): + return [pn.pane.Markdown("Body")] + + @block + def block_dynamic_title(self, title: str = ""): + return "Resolved title", [pn.pane.Markdown("Dynamic body")] + + @block + def block_scalar_body(self, title: str = "Scalar"): + return "plain text" + + @block + def block_table(self, title: str = "Table"): + return [pn.pane.DataFrame(pd.DataFrame({"a": [1], "b": [2]}))] + + @block + def block_badge_row(self, title: str = "Badges"): + return [pn.Row(pn.pane.Markdown("One"), pn.pane.Markdown("Two"))] + + @block + def block_select_allowed(self, title: str = "Selector"): + return [pn.widgets.Select(name="Feature", options=["a", "b"], value="a")] + + @block + def block_plotly_allowed(self, title: str = "Plotly"): + fig = go.Figure(go.Scatter(x=[1, 2], y=[3, 4])) + return [pn.pane.Plotly(fig)] + + @block + def block_bind_allowed(self, title: str = "Bind"): + selector = pn.widgets.Select(name="Feature", options=["a", "b"], value="a") + selected_panel = pn.panel(pn.bind(lambda selected: pn.pane.Markdown(selected), selector)) + return [selector, selected_panel] + + @block + def block_panel_type_not_allowed(self, title: str = "HTML"): + return [pn.pane.HTML("html")] + + @block + def block_non_panel_type_not_allowed(self, title: str = "Object"): + return [object()] + + +class TestBlockDecorator(unittest.TestCase): + def test_block_decorator_wraps_with_title_from_signature(self): + runtime = _DummyBlocks() + + result = runtime.block_demo() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[0], pn.pane.Markdown) + self.assertIn("Demo", result.objects[0].object) + + def test_block_decorator_supports_dynamic_title_tuple(self): + runtime = _DummyBlocks() + + result = runtime.block_dynamic_title() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[0], pn.pane.Markdown) + self.assertIn("Resolved title", result.objects[0].object) + + def test_block_decorator_coerces_scalar_body_to_markdown(self): + runtime = _DummyBlocks() + + result = runtime.block_scalar_body() + + self.assertIsInstance(result, pn.Column) + self.assertEqual(len(result.objects), 2) + self.assertIsInstance(result.objects[1], pn.pane.Markdown) + self.assertIn("plain text", result.objects[1].object) + + def test_block_decorator_auto_stylizes_body_by_type(self): + runtime = _DummyBlocks() + + text_result = runtime.block_demo() + table_result = runtime.block_table() + + self.assertIn("content-block", text_result.objects[1].css_classes) + self.assertIn("kv-table", table_result.objects[1].css_classes) + + def test_block_decorator_auto_styles_badge_rows(self): + runtime = _DummyBlocks() + + result = runtime.block_badge_row() + + badge_row = result.objects[1] + self.assertIsInstance(badge_row, pn.Row) + self.assertIn("badge-pill", badge_row.objects[0].css_classes) + self.assertIn("badge-pill", badge_row.objects[1].css_classes) + + def test_block_decorator_allows_select_and_plotly(self): + runtime = _DummyBlocks() + + select_result = runtime.block_select_allowed() + plotly_result = runtime.block_plotly_allowed() + + self.assertIsInstance(select_result.objects[1], pn.widgets.Select) + self.assertIsInstance(plotly_result.objects[1], pn.pane.Plotly) + + def test_block_decorator_allows_bind_param_function(self): + runtime = _DummyBlocks() + + result = runtime.block_bind_allowed() + + self.assertIsInstance(result.objects[1], pn.widgets.Select) + self.assertEqual(type(result.objects[2]).__name__, "ParamFunction") + + def test_block_decorator_rejects_panel_type_without_style_definition(self): + runtime = _DummyBlocks() + + with self.assertRaises(TypeError) as context: + runtime.block_panel_type_not_allowed() + + self.assertIn("Unsupported Panel object type returned", str(context.exception)) + self.assertIn("Allowed Panel return types", str(context.exception)) + + def test_block_decorator_rejects_non_panel_return_type(self): + runtime = _DummyBlocks() + + with self.assertRaises(TypeError) as context: + runtime.block_non_panel_type_not_allowed() + + self.assertIn("Unsupported block return type", str(context.exception)) diff --git a/tutorial/generate_report/config/custom_report.css b/tutorial/generate_report/config/custom_report.css new file mode 100644 index 00000000..df1c4093 --- /dev/null +++ b/tutorial/generate_report/config/custom_report.css @@ -0,0 +1,275 @@ +:root { + --shapash-yellow: #f4c000; + --shapash-black: #343736; +} + +.main-report { + padding: 24px 32px; + align-items: flex-start; + gap: 20px; + overflow: visible !important; +} + +.report-sidebar { + align-self: flex-start; + position: sticky; + top: 16px; + z-index: 30; + max-height: calc(100vh - 32px); + overflow: hidden; +} + +.report-content { + flex: 1 1 auto; + min-width: 0; +} + +/* Generic key/value and dataframe tables */ +.kv-table, +table.dataframe { + width: 100%; + border-collapse: separate; + border-spacing: 0; + margin: 12px 0 24px; + background: #fff; + border: 1px solid #ececec; + border-radius: 12px; + overflow: hidden; + box-shadow: 0 8px 24px rgba(0, 0, 0, 0.04); +} + +.kv-table th, +.kv-table td, +table.dataframe th, +table.dataframe td { + text-align: center; + vertical-align: middle; +} + +.shapash-callout { + padding: 14px 20px; + border-left: 4px solid var(--shapash-yellow); +} + +.badge-pill { + border: 1px solid #eeeeee; + border-radius: 999px; + padding: 6px 12px; + display: inline-block; +} + +.badge-pill-gold { + border-color: var(--shapash-yellow); +} + +.badge-pill-blue { + border-color: #2255aa; +} + +.badge-pill-gray { + border-color: #eeeeee; +} + +.badge-pill-orange { + border-color: var(--shapash-yellow); +} + +.project-info-grid { + display: grid !important; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + align-items: stretch; +} + +.project-info-card { + display: flex; + flex-direction: column; + height: 100%; + min-width: 0; +} + +.project-info-card .kv-table { + flex: 1 1 auto; + margin-bottom: 0; +} + +.project-info-card .kv-table table.dataframe { + height: 100%; +} + +.fit-content-table { + width: fit-content; + max-width: 100%; + overflow-x: auto; +} + +.fit-content-table table.dataframe { + width: max-content; + table-layout: auto; +} + +.report-nav { + position: static; + z-index: 20; + display: flex; + flex-direction: column; + align-items: stretch; + flex-wrap: nowrap; + gap: calc(8px * var(--nav-scale, 1)); + margin: 0; + padding: calc(10px * var(--nav-scale, 1)) calc(12px * var(--nav-scale, 1)); + border: 1px solid #ececec; + border-radius: 12px; + background: rgba(255, 255, 255, 0.96); + backdrop-filter: blur(4px); + box-shadow: 0 6px 18px rgba(0, 0, 0, 0.06); + height: calc(100vh - 32px); + overflow-y: auto; + overflow-x: hidden; + -ms-overflow-style: none; + scrollbar-width: none; +} + +.report-nav::-webkit-scrollbar { + width: 0; + height: 0; +} + +.nav-logo { + display: flex; + align-items: flex-start; + justify-content: flex-start; + padding: calc(4px * var(--nav-scale, 1)); + margin-bottom: calc(6px * var(--nav-scale, 1)); +} + +.nav-logo img { + display: block; + width: min(70px, 100%); + height: auto; +} + +.nav-current { + display: flex; + flex-direction: column; + gap: 2px; + padding: calc(8px * var(--nav-scale, 1)); + border-radius: 10px; + border: 1px solid #f2d878; + background: #fff8dc; + margin-bottom: calc(4px * var(--nav-scale, 1)); +} + +.nav-current-label { + font-size: calc(0.72rem * var(--nav-scale, 1)); + text-transform: uppercase; + letter-spacing: 0.04em; + color: #7a6a2f; +} + +.nav-current-value { + font-size: calc(0.9rem * var(--nav-scale, 1)); + font-weight: 700; +} + +.nav-group { + display: flex; + flex-direction: column; + align-items: stretch; + gap: calc(8px * var(--nav-scale, 1)); + flex-wrap: nowrap; + padding: calc(4px * var(--nav-scale, 1)) 0; +} + +.nav-group-children { + display: flex; + flex-direction: column; + align-items: stretch; + gap: calc(6px * var(--nav-scale, 1)); + flex-wrap: nowrap; + padding-left: calc(10px * var(--nav-scale, 1)); +} + +.nav-item { + display: block; + padding: calc(6px * var(--nav-scale, 1)) calc(10px * var(--nav-scale, 1)); + border-radius: 8px; + border: 1px solid #dddddd; + color: var(--shapash-black); + text-decoration: none; + font-size: calc(0.9rem * var(--nav-scale, 1)); + line-height: 1.2; + background: #fff; +} + +.nav-group-title { + border-color: #d4d4d4; + font-weight: 700; +} + +.nav-child { + border-style: dashed; + font-size: calc(0.84rem * var(--nav-scale, 1)); +} + +.nav-item:hover, +.nav-item.active { + border-color: var(--shapash-yellow); + background: #fff9e6; +} + +.nav-item.active { + box-shadow: inset 3px 0 0 var(--shapash-yellow); + font-weight: 700; +} + +.scroll-anchor { + display: block; + position: relative; + top: -10px; + visibility: hidden; +} + +/* Responsive adjustments */ +@media (max-width: 1200px) { + + .main-report { + padding: 16px; + gap: 12px; + } + + .kv-table, + table.dataframe { + display: block; + overflow-x: auto; + -webkit-overflow-scrolling: touch; + } + + .kv-val, + .content-block { + overflow-wrap: anywhere; + word-break: break-word; + } + + .project-info-grid { + grid-template-columns: 1fr; + } + + .project-info-card { + width: 100%; + height: auto; + } + + .report-nav { + position: static; + padding: 8px; + height: auto; + } + + .report-sidebar { + position: static; + top: auto; + max-height: none; + overflow: visible; + } +} diff --git a/tutorial/generate_report/config/custom_report_classification_titanic.yml b/tutorial/generate_report/config/custom_report_classification_titanic.yml new file mode 100644 index 00000000..5caf1f0f --- /dev/null +++ b/tutorial/generate_report/config/custom_report_classification_titanic.yml @@ -0,0 +1,87 @@ +# custom_report_classification_titanic.yml +# Custom smart report configuration for Titanic binary classification. + +sections: + - type: header + params: + title: "Titanic survival custom report" + subtitle: > + This report combines built-in explainability sections and custom + user blocks dedicated to classification error analysis. + + - type: model_analysis + params: + title: "Model analysis" + + - type: group + params: + title: "Dataset analysis" + blocks: + - type: global_analysis + params: + title: "Global analysis" + + - type: univariate_analysis + params: + title: "Univariate analysis" + + - type: target_analysis + params: + title: "Target analysis" + show_train: true + + - type: group + params: + title: "Classification explainability" + blocks: + - type: feature_importance + params: + title: "Global feature importance" + label: 1 + + - type: contribution_plot + params: + title: "Features contribution plots" + include_all_features: true + label: 1 + + - type: misclassification_focus + params: + title: "Most confident misclassifications" + top_k: 12 + + - type: group + params: + title: "Performance" + blocks: + - type: confusion_matrix + params: + title: "Confusion matrix" + + - type: lift_curve + params: + title: "Lift curve" + label: 1 + + - type: prediction_error_summary + params: + title: "Prediction error summary" + + - type: performance_metrics + params: + title: "Metrics" + metrics: + - path: "sklearn.metrics.accuracy_score" + name: "Accuracy" + - path: "sklearn.metrics.precision_score" + name: "Precision" + - path: "sklearn.metrics.recall_score" + name: "Recall" + - path: "sklearn.metrics.f1_score" + name: "F1" + + - type: callout + params: + body: > + Custom blocks focus on high-confidence mistakes to prioritize the + most actionable explainability investigations. diff --git a/tutorial/generate_report/config/custom_report_regression_house_prices.yml b/tutorial/generate_report/config/custom_report_regression_house_prices.yml new file mode 100644 index 00000000..379441c5 --- /dev/null +++ b/tutorial/generate_report/config/custom_report_regression_house_prices.yml @@ -0,0 +1,130 @@ +# custom_report_regression_house_prices.yml +# Custom smart report configuration for house prices regression. + +sections: + - type: header + params: + title: "House prices custom report" + subtitle: > + This report combines built-in blocks with user-defined regression + diagnostics for explainability-oriented error analysis. + + - type: group + params: + title: "Project information" + blocks: + # `text` displays a free markdown text block. + - type: text + params: + title: "General information" + content: + version: 0.7 + name: House Prices Prediction Project + purpose: Predicting the sale price of houses + date: auto + contributors: Yann Golhen, Sebastien Bidault, Thomas Bouche, Guillaume Vignal, Thibaud Real + description: This work is a data science project that tries to predict the sale of houses based on 79 explanatory variables. It was designed inside the data science team at X. and improved since the beggining of the project in 2019. The model was put into production since February 2021. + source code: https://github.com/MAIF/shapash/tree/master/tutorial + Git commit: 1ff46e83beafba8949a7f3b7de27586acd6ae99e + - type: text + params: + title: "Dataset information" + content: + path: https://www.kaggle.com/c/house-prices-advanced-regression-techniques/ + origin: The Assessor’s Office + description: the sale of individual residential property in Ames, Iowa + depth: from 2006 to 2010 + perimeter: only residential sales + target variable: SalePrice + target description: The property's sale price in dollars + - type: text + params: + title: "Data preparation" + content: + variable filtering: All variables that required special knowledge or previous calculations for their use were removed + individual filtering: only the most recent sales data on any property were kept (for houses that were sold multiple times during this period) + missing values: were replaced by 0 + Feature engineering: No feature was created. All features are directly taken from the kaggle dataset. Categorical features were transformed using an ordinal encoder. + Path to script: https://github.com/MAIF/shapash/tree/master/tutorial/ + - type: text + params: + title: "Model training" + content: + Used algorithm: We used a RandomForestRegressor algorithm (scikit-learn) but this model could be challenged with other interesting models such as XGBRegressor, Neural Networks, ... + Parameters choice: We did not perform any hyperparameter optimisation and chose to use `n_estimators=50`. Future works should be planned to perform gridsearch optimizations + Metrics: Mean Squared Error metric + Validation strategy: We splitted our data into train (75%) and test (25%) + Path to script: https://github.com/MAIF/shapash/tree/master/tutorial/ + + - type: model_analysis + params: + title: "Model analysis" + + - type: group + params: + title: "Dataset analysis" + blocks: + - type: global_analysis + params: + title: "Global analysis" + + - type: univariate_analysis + params: + title: "Univariate analysis" + + - type: target_analysis + params: + title: "Target analysis" + show_train: true + + - type: group + params: + title: "Explainability" + blocks: + - type: feature_importance + params: + title: "Global feature importance" + + - type: contribution_plot + params: + title: "Features contribution plots" + include_all_features: true + + # - type: top_interactions_plot + # params: + # title: "Top interactions plot" + # nb_top_interaction: 5 + + - type: residual_error_summary + params: + title: "Residual error summary" + + - type: largest_errors_focus + params: + title: "Largest absolute errors" + top_k: 12 + + - type: group + params: + title: "Performance" + blocks: + - type: target_distribution + params: + title: "Target distribution" + + - type: performance_metrics + params: + title: "Metrics" + metrics: + - path: "sklearn.metrics.mean_absolute_error" + name: "Mean absolute error" + - path: "sklearn.metrics.mean_squared_error" + name: "Mean squared error" + - path: "sklearn.metrics.r2_score" + name: "R2 score" + + - type: callout + params: + body: > + Custom blocks focus on the biggest residuals to prioritize model + debugging and feature-level explainability checks. diff --git a/tutorial/generate_report/shapash_classification_report_example.py b/tutorial/generate_report/shapash_classification_report_example.py new file mode 100644 index 00000000..795be9d0 --- /dev/null +++ b/tutorial/generate_report/shapash_classification_report_example.py @@ -0,0 +1,192 @@ +""" +Generate a Titanic survival classification report with the smart_report implementation. + +The script supports both built-in and custom report layouts: +- default_report_classification_titanic.yml +- custom_report_classification_titanic.yml +""" + +import argparse +import os +import sys + +import pandas as pd +from category_encoders import OrdinalEncoder +from sklearn.ensemble import RandomForestClassifier +from sklearn.metrics import accuracy_score, precision_score, recall_score +from sklearn.model_selection import train_test_split + +sys.path.insert(0, "..") + +from shapash import SmartExplainer +from shapash.data.data_loader import data_loading +from shapash.report.blocks import ReportBlockMixin, block + + +class CustomClassificationReportBlocks(ReportBlockMixin): + """User-defined blocks for a Titanic classification explainability report.""" + + @block + def block_prediction_error_summary(self, title: str = "Prediction error summary"): + """Summarize global classification errors with confusion counts and key metrics.""" + if self.y_test is None or self.y_pred is None: + raise ValueError("prediction_error_summary block requires y_test and y_pred.") + + y_true = pd.Series(self.y_test).reset_index(drop=True) + y_pred = pd.Series(self.y_pred).reset_index(drop=True) + + tp = int(((y_true == 1) & (y_pred == 1)).sum()) + tn = int(((y_true == 0) & (y_pred == 0)).sum()) + fp = int(((y_true == 0) & (y_pred == 1)).sum()) + fn = int(((y_true == 1) & (y_pred == 0)).sum()) + + metrics_df = pd.DataFrame( + [ + ["Accuracy", f"{accuracy_score(y_true, y_pred):.3f}"], + ["Precision (Survived)", f"{precision_score(y_true, y_pred, zero_division=0):.3f}"], + ["Recall (Survived)", f"{recall_score(y_true, y_pred, zero_division=0):.3f}"], + ["False positives", fp], + ["False negatives", fn], + ["True positives", tp], + ["True negatives", tn], + ], + columns=["Metric", "Value"], + ) + + summary = ( + "This block highlights where the classifier makes mistakes. " + "False positives are non-survivors predicted as survivors, while false negatives " + "are survivors predicted as non-survivors." + ) + + return title, [summary, metrics_df] + + @block + def block_misclassification_focus( + self, + title: str = "Most confident misclassifications", + top_k: int = 10, + ): + """Show the wrong predictions with highest model confidence to guide error analysis.""" + explainer = self._require_explainer("misclassification_focus") + if self.y_test is None or self.y_pred is None: + raise ValueError("misclassification_focus block requires y_test and y_pred.") + + if explainer.proba_values is None: + explainer.predict_proba() + + y_true = pd.Series(self.y_test, index=explainer.x_init.index, name="true") + y_pred = pd.Series(self.y_pred, index=explainer.x_init.index, name="pred") + + if explainer.proba_values.shape[1] < 2: + raise ValueError("misclassification_focus block requires binary class probabilities.") + + proba_survived = explainer.proba_values.iloc[:, 1].rename("proba_survived") + + analysis = pd.concat([y_true, y_pred, proba_survived, explainer.x_init], axis=1) + wrong = analysis[analysis["true"] != analysis["pred"]].copy() + + if wrong.empty: + return title, ["No misclassification found on the evaluated dataset."] + + wrong["wrong_confidence"] = wrong.apply( + lambda row: row["proba_survived"] if row["pred"] == 1 else 1 - row["proba_survived"], axis=1 + ) + wrong = wrong.sort_values("wrong_confidence", ascending=False).head(top_k) + + selected_cols = ["true", "pred", "proba_survived", "wrong_confidence"] + contextual_cols = [col for col in ["Pclass", "Sex", "Age", "Fare", "Embarked", "Title"] if col in wrong] + display_df = wrong[selected_cols + contextual_cols].copy() + display_df = display_df.rename(columns={"true": "True", "pred": "Pred", "proba_survived": "P(Survived)"}) + display_df["P(Survived)"] = display_df["P(Survived)"].map(lambda x: round(float(x), 3)) + display_df["wrong_confidence"] = display_df["wrong_confidence"].map(lambda x: round(float(x), 3)) + + text = ( + "Rows below are the most confident wrong predictions. " + "They are useful to inspect possible data drift, noise, or feature blind spots." + ) + table = display_df.reset_index(drop=False) + return title, [text, table] + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate Titanic classification report examples.") + parser.add_argument( + "--report-mode", + choices=["default", "custom"], + default="default", + help="Choose report layout: default built-in blocks or custom blocks.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + + titanic_df, titanic_dict = data_loading("titanic") + + y_df = titanic_df["Survived"] + x_df = titanic_df[titanic_df.columns.difference(["Survived"])].copy() + + # Ensure non-numeric columns are treated as categorical before encoding. + for col in x_df.columns: + if not pd.api.types.is_numeric_dtype(x_df[col]): + x_df[col] = x_df[col].astype(object) + + categorical_features = [ + col + for col in x_df.columns + if pd.api.types.is_object_dtype(x_df[col]) or pd.api.types.is_string_dtype(x_df[col]) + ] + + encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(x_df) + + x_df = encoder.transform(x_df) + + xtrain, xtest, ytrain, ytest = train_test_split( + x_df, + y_df, + train_size=0.75, + random_state=1, + stratify=y_df, + ) + + classifier = RandomForestClassifier(n_estimators=200, random_state=1).fit(xtrain, ytrain) + + # Keep y_pred as dataframe to match SmartExplainer report expectations. + y_pred = pd.DataFrame(classifier.predict(xtest), columns=["pred"], index=xtest.index) + + cur_dir = os.path.dirname(os.path.abspath(__file__)) + + xpl = SmartExplainer( + model=classifier, + preprocessing=encoder, # Optional: compile step can use inverse_transform method + features_dict=titanic_dict, + label_dict={0: "Did not survive", 1: "Survived"}, + ) + + # Compile once before report generation. + xpl.compile(x=xtest, y_pred=y_pred, y_target=ytest) + + if args.report_mode == "custom": + output_file = os.path.join(cur_dir, "output", "classification_report_custom.html") + report_config_file = os.path.join(cur_dir, "config", "custom_report_classification_titanic.yml") + block_instance = CustomClassificationReportBlocks( + explainer=xpl, + x_train=xtrain, + y_train=ytrain, + y_test=ytest, + ) + xpl.generate_report( + output_file=output_file, + yaml_path=report_config_file, + block_instance=block_instance, + ) + else: + output_file = os.path.join(cur_dir, "output", "classification_report.html") + xpl.generate_report( + output_file=output_file, + x_train=xtrain, + y_train=ytrain, + y_test=ytest, + ) diff --git a/tutorial/generate_report/shapash_regression_report_example.py b/tutorial/generate_report/shapash_regression_report_example.py new file mode 100644 index 00000000..85fd6728 --- /dev/null +++ b/tutorial/generate_report/shapash_regression_report_example.py @@ -0,0 +1,175 @@ +""" +Generate a house prices regression report with the smart_report implementation. + +The script supports both built-in and custom report layouts: +- default_report_regression_house_prices.yml +- custom_report_regression_house_prices.yml +""" + +import argparse +import os +import sys + +import pandas as pd +from category_encoders import OrdinalEncoder +from sklearn.ensemble import RandomForestRegressor +from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score +from sklearn.model_selection import train_test_split + +sys.path.insert(0, "..") + +from shapash import SmartExplainer +from shapash.data.data_loader import data_loading +from shapash.report.blocks import ReportBlockMixin, block + + +class CustomRegressionReportBlocks(ReportBlockMixin): + """User-defined blocks for a house prices regression explainability report.""" + + @block + def block_residual_error_summary(self, title: str = "Residual error summary"): + """Summarize residual dispersion and global regression performance indicators.""" + if self.y_test is None or self.y_pred is None: + raise ValueError("residual_error_summary block requires y_test and y_pred.") + + y_true = pd.Series(self.y_test).reset_index(drop=True) + y_pred = pd.Series(self.y_pred).reset_index(drop=True) + residuals = y_true - y_pred + abs_residuals = residuals.abs() + + summary_df = pd.DataFrame( + [ + ["MAE", f"{mean_absolute_error(y_true, y_pred):,.2f}"], + ["MSE", f"{mean_squared_error(y_true, y_pred):,.2f}"], + ["R2", f"{r2_score(y_true, y_pred):.3f}"], + ["Residual mean", f"{residuals.mean():,.2f}"], + ["Residual std", f"{residuals.std():,.2f}"], + ["Median absolute error", f"{abs_residuals.median():,.2f}"], + ["95th pct absolute error", f"{abs_residuals.quantile(0.95):,.2f}"], + ], + columns=["Metric", "Value"], + ) + + explanation = ( + "This section summarizes global regression error levels. " + "A strong gap between median and 95th percentile absolute error usually highlights " + "a subset of difficult cases worth deeper explainability analysis." + ) + return title, [explanation, summary_df] + + @block + def block_largest_errors_focus(self, title: str = "Largest absolute errors", top_k: int = 10): + """Display samples with the largest absolute errors to prioritize local explainability reviews.""" + explainer = self._require_explainer("largest_errors_focus") + if self.y_test is None or self.y_pred is None: + raise ValueError("largest_errors_focus block requires y_test and y_pred.") + + y_true = pd.Series(self.y_test, index=explainer.x_init.index, name="true") + y_pred = pd.Series(self.y_pred, index=explainer.x_init.index, name="pred") + details = pd.concat([y_true, y_pred, explainer.x_init], axis=1) + details["residual"] = details["true"] - details["pred"] + details["abs_error"] = details["residual"].abs() + + focus = details.sort_values("abs_error", ascending=False).head(top_k).copy() + if focus.empty: + return title, ["No rows available to compute largest errors."] + + focus = focus.rename(columns={"true": "True", "pred": "Pred", "residual": "Residual", "abs_error": "AbsError"}) + + # Keep key business columns first when available. + preferred = ["OverallQual", "GrLivArea", "TotalBsmtSF", "GarageArea", "Neighborhood"] + context_cols = [c for c in preferred if c in focus.columns] + leading = ["True", "Pred", "Residual", "AbsError"] + trailing = [c for c in focus.columns if c not in leading + context_cols] + ordered_cols = leading + context_cols + trailing + + focus = focus[ordered_cols] + for col in ["True", "Pred", "Residual", "AbsError"]: + focus[col] = focus[col].map(lambda x: round(float(x), 2)) + + info = ( + "Rows below are the largest absolute errors. " + "They are ideal candidates for local contribution plots and feature-level investigation." + ) + table = focus.reset_index(drop=False) + return title, [info, table] + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate house prices regression report examples.") + parser.add_argument( + "--report-mode", + choices=["default", "custom"], + default="default", + help="Choose report layout: default built-in blocks or custom blocks.", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = _parse_args() + + house_df, house_dict = data_loading("house_prices") + + y_df = house_df["SalePrice"] + x_df = house_df[house_df.columns.difference(["SalePrice"])].copy() + + # Ensure non-numeric columns are treated as categorical before encoding. + for col in x_df.columns: + if not pd.api.types.is_numeric_dtype(x_df[col]): + x_df[col] = x_df[col].astype(object) + + categorical_features = [ + col + for col in x_df.columns + if pd.api.types.is_object_dtype(x_df[col]) or pd.api.types.is_string_dtype(x_df[col]) + ] + + encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(x_df) + x_df = encoder.transform(x_df) + + xtrain, xtest, ytrain, ytest = train_test_split( + x_df, + y_df, + train_size=0.75, + random_state=1, + ) + + regressor = RandomForestRegressor(n_estimators=200, random_state=1).fit(xtrain, ytrain) + + # Keep y_pred as dataframe to match SmartExplainer report expectations. + y_pred = pd.DataFrame(regressor.predict(xtest), columns=["pred"], index=xtest.index) + + cur_dir = os.path.dirname(os.path.abspath(__file__)) + + xpl = SmartExplainer( + model=regressor, + preprocessing=encoder, + features_dict=house_dict, + ) + + # Compile once before report generation. + xpl.compile(x=xtest, y_pred=y_pred, y_target=ytest) + + if args.report_mode == "custom": + output_file = os.path.join(cur_dir, "output", "regression_report_custom.html") + report_config_file = os.path.join(cur_dir, "config", "custom_report_regression_house_prices.yml") + block_instance = CustomRegressionReportBlocks( + explainer=xpl, + x_train=xtrain, + y_train=ytrain, + y_test=ytest, + ) + xpl.generate_report( + output_file=output_file, + yaml_path=report_config_file, + block_instance=block_instance, + ) + else: + output_file = os.path.join(cur_dir, "output", "regression_report.html") + xpl.generate_report( + output_file=output_file, + x_train=xtrain, + y_train=ytrain, + y_test=ytest, + ) diff --git a/tutorial/generate_report/shapash_report_example.py b/tutorial/generate_report/shapash_report_example.py deleted file mode 100644 index 79fe6cf9..00000000 --- a/tutorial/generate_report/shapash_report_example.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -This script can be used to generate the report example. -For more information, please refer to the tutorial 'tuto-shapash-report01.ipynb' -that generates the same report. -""" -import os -import sys - -import pandas as pd -from category_encoders import OrdinalEncoder -from sklearn.ensemble import RandomForestRegressor -from sklearn.model_selection import train_test_split - -sys.path.insert(0, "..") - -from shapash import SmartExplainer -from shapash.data.data_loader import data_loading -from shapash.utils.dtypes import text_like_columns - -if __name__ == "__main__": - house_df, house_dict = data_loading("house_prices") - y_df = house_df["SalePrice"] - X_df = house_df[house_df.columns.difference(["SalePrice"])] - - categorical_features = text_like_columns(X_df, strict_object=False) - - encoder = OrdinalEncoder(cols=categorical_features, handle_unknown="return_nan", return_df=True).fit(X_df) - - X_df = encoder.transform(X_df) - - Xtrain, Xtest, ytrain, ytest = train_test_split(X_df, y_df, train_size=0.75, random_state=1) - - regressor = RandomForestRegressor(n_estimators=50).fit(Xtrain, ytrain) - - y_pred = pd.DataFrame(regressor.predict(Xtest), columns=["pred"], index=Xtest.index) - - cur_dir = os.path.dirname(os.path.abspath(__file__)) - - xpl = SmartExplainer( - model=regressor, - preprocessing=encoder, # Optional: compile step can use inverse_transform method - features_dict=house_dict, - ) - xpl.compile(x=Xtest, y_pred=y_pred, y_target=ytest) - - xpl.generate_report( - output_file=os.path.join(cur_dir, "output", "report.html"), - project_info_file=os.path.join(cur_dir, "utils", "project_info.yml"), - x_train=Xtrain, - y_train=ytrain, - y_test=ytest, - title_story="House prices report", - title_description="""This document is a data science report of the kaggle house prices tutorial project. - It was generated using the Shapash library.""", - metrics=[ - { - "path": "sklearn.metrics.mean_absolute_error", - "name": "Mean absolute error", - }, - { - "path": "sklearn.metrics.mean_squared_error", - "name": "Mean squared error", - }, - ], - ) diff --git a/tutorial/generate_report/tuto-shapash-regression-report.ipynb b/tutorial/generate_report/tuto-shapash-regression-report.ipynb new file mode 100644 index 00000000..ca6b2bee --- /dev/null +++ b/tutorial/generate_report/tuto-shapash-regression-report.ipynb @@ -0,0 +1,447 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "6a51631c", + "metadata": {}, + "source": [ + "# Tutorial: Build a Shapash Regression Report\n", + "\n", + "This notebook walks through the full construction of a **Shapash regression report** step by step.\n", + "\n", + "We will start from a simple `RandomForestRegressor`, compile a `SmartExplainer`, generate a base report from the default regression YAML template, then extend the report with custom blocks.\n", + "\n", + "The tutorial is designed for users discovering the product, so each step includes comments explaining why it is needed and how to adapt it to your own project.\n", + "\n", + "At the end of the notebook, you will have:\n", + "- a trained regression model\n", + "- a compiled `SmartExplainer`\n", + "- a base HTML report generated from `default_report_regression_house_prices.yml`\n", + "- a custom HTML report generated from `custom_report_regression_house_prices.yml` with user-defined blocks" + ] + }, + { + "cell_type": "markdown", + "id": "72a3528d", + "metadata": {}, + "source": [ + "## 1. Imports and Working Directories\n", + "\n", + "We first import the libraries needed for data preparation, model training, explainability, and report generation.\n", + "\n", + "The small directory helper below makes the notebook robust whether it is executed from the repository root or directly from the `tutorial/generate_report` folder." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3de4750f", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "from category_encoders import OrdinalEncoder\n", + "from sklearn.ensemble import RandomForestRegressor\n", + "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "from shapash import SmartExplainer\n", + "from shapash.data.data_loader import data_loading\n", + "from shapash.report import ReportTemplate, export_report_yml\n", + "from shapash.report.blocks import ReportBlockMixin, block\n", + "\n", + "BASE_DIR = Path.cwd()\n", + "if not (BASE_DIR / \"config\").exists():\n", + " BASE_DIR = BASE_DIR / \"tutorial\" / \"generate_report\"\n", + "\n", + "CONFIG_DIR = BASE_DIR / \"config\"\n", + "OUTPUT_DIR = BASE_DIR / \"output\"\n", + "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(f\"Base directory: {BASE_DIR}\")\n", + "print(f\"Config directory: {CONFIG_DIR}\")\n", + "print(f\"Output directory: {OUTPUT_DIR}\")" + ] + }, + { + "cell_type": "markdown", + "id": "e5cb3da3", + "metadata": {}, + "source": [ + "## 2. Load the Dataset\n", + "\n", + "We use the built-in **House Prices** dataset shipped with Shapash tutorials.\n", + "\n", + "The target is `SalePrice`. All other columns are used as input features." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09ae8b21", + "metadata": {}, + "outputs": [], + "source": [ + "house_df, house_dict = data_loading(\"house_prices\")\n", + "\n", + "y_df = house_df[\"SalePrice\"]\n", + "x_df = house_df[house_df.columns.difference([\"SalePrice\"])].copy()\n", + "\n", + "print(f\"Dataset shape: {house_df.shape}\")\n", + "print(f\"Number of features: {x_df.shape[1]}\")\n", + "house_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "79140a47", + "metadata": {}, + "source": [ + "## 3. Prepare the Data and Train a Simple Regression Model\n", + "\n", + "To keep the example simple, we use an `OrdinalEncoder` for categorical variables and then train a `RandomForestRegressor`.\n", + "\n", + "This is intentionally a lightweight baseline model: the focus of this notebook is the **report generation workflow**, not model optimization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbdbe53f", + "metadata": {}, + "outputs": [], + "source": [ + "# Convert non-numeric columns to object so the encoder treats them as categorical features.\n", + "for col in x_df.columns:\n", + " if not pd.api.types.is_numeric_dtype(x_df[col]):\n", + " x_df[col] = x_df[col].astype(object)\n", + "\n", + "categorical_features = [\n", + " col\n", + " for col in x_df.columns\n", + " if pd.api.types.is_object_dtype(x_df[col]) or pd.api.types.is_string_dtype(x_df[col])\n", + "]\n", + "\n", + "encoder = OrdinalEncoder(\n", + " cols=categorical_features,\n", + " handle_unknown=\"return_nan\",\n", + " return_df=True,\n", + ").fit(x_df)\n", + "\n", + "x_df_encoded = encoder.transform(x_df)\n", + "\n", + "xtrain, xtest, ytrain, ytest = train_test_split(\n", + " x_df_encoded,\n", + " y_df,\n", + " train_size=0.75,\n", + " random_state=1,\n", + ")\n", + "\n", + "regressor = RandomForestRegressor(\n", + " n_estimators=200,\n", + " random_state=1,\n", + ").fit(xtrain, ytrain)\n", + "\n", + "# Keep predictions as a DataFrame to match report expectations.\n", + "y_pred = pd.DataFrame(regressor.predict(xtest), columns=[\"pred\"], index=xtest.index)\n", + "\n", + "print(f\"Train shape: {xtrain.shape}\")\n", + "print(f\"Test shape: {xtest.shape}\")\n", + "print(f\"MAE: {mean_absolute_error(ytest, y_pred.iloc[:, 0]):,.2f}\")\n", + "print(f\"MSE: {mean_squared_error(ytest, y_pred.iloc[:, 0]):,.2f}\")\n", + "print(f\"R2: {r2_score(ytest, y_pred.iloc[:, 0]):.3f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "57086a35", + "metadata": {}, + "source": [ + "## 4. Instantiate and Compile the SmartExplainer\n", + "\n", + "`SmartExplainer` is the central Shapash object used to compute explainability artifacts and later generate the report.\n", + "\n", + "Compilation links the model outputs, the dataset to explain, and the human-readable feature labels." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2741aad", + "metadata": {}, + "outputs": [], + "source": [ + "xpl = SmartExplainer(\n", + " model=regressor,\n", + " preprocessing=encoder,\n", + " features_dict=house_dict,\n", + ")\n", + "\n", + "# Compile once before generating any report.\n", + "xpl.compile(x=xtest, y_pred=y_pred, y_target=ytest)\n", + "\n", + "xpl" + ] + }, + { + "cell_type": "markdown", + "id": "adaa75f7", + "metadata": {}, + "source": [ + "## 5. Generate a Base Regression Report\n", + "\n", + "We now generate a first report using the default regression template.\n", + "\n", + "This gives a complete report using **built-in blocks only**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a7376cf", + "metadata": {}, + "outputs": [], + "source": [ + "base_report_path = OUTPUT_DIR / \"default_regression_report.html\"\n", + "\n", + "xpl.generate_report(\n", + " output_file=str(base_report_path),\n", + " x_train=xtrain,\n", + " y_train=ytrain,\n", + " y_test=ytest,\n", + ")\n", + "\n", + "print(f\"Base report generated: {base_report_path}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ad00e4f9", + "metadata": {}, + "source": [ + "## 6. Inspect and Adapt the YAML Configuration\n", + "\n", + "Shapash ships with default regression and classification report templates.\n", + "\n", + "Using `export_report_yml`, you can copy a template into your own working directory and then adapt it to your project. This is the recommended starting point for most users.\n", + "\n", + "Before generating the report, open the exported YAML file and adapt it to your own context.\n", + "\n", + "Typical customizations include:\n", + "- changing the report title and subtitle\n", + "- updating project information and dataset information\n", + "- documenting your data preparation choices\n", + "- editing the list of blocks and their parameters\n", + "- choosing the metrics to display\n", + "\n", + "The next cell prints the template content so you can inspect it directly from the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c2ea635", + "metadata": {}, + "outputs": [], + "source": [ + "default_template_path = CONFIG_DIR / \"default_report_regression_tutorial.yml\"\n", + "\n", + "# Export the built-in default regression template to a local file.\n", + "export_report_yml(ReportTemplate.DEFAULT_REGRESSION, output_path=str(default_template_path))\n", + "\n", + "print(f\"Default template exported to: {default_template_path}\")\n", + "\n", + "# Read the YAML template as plain text so users can review it before editing.\n", + "print(default_template_path.read_text(encoding=\"utf-8\"))" + ] + }, + { + "cell_type": "markdown", + "id": "3d41d7f3", + "metadata": {}, + "source": [ + "## 7. Create a Class with Custom Report Blocks\n", + "\n", + "One of the strengths of the Shapash reporting system is that you can extend it with your own blocks.\n", + "\n", + "A custom block is simply a method named `block_` inside a class inheriting from `ReportBlockMixin`.\n", + "\n", + "These blocks can use the explainer data, the predictions, the targets, and any additional logic you want to expose in the report.\n", + "\n", + "Decorated methods can return either ``(title, body)`` or a bare body value.\n", + "\n", + "The body may be a single supported item or a list of supported items.\n", + "Each item can be a string, a pandas ``DataFrame``, a Plotly figure, or a Panel viewable.\n", + "Tuples inside the body are rendered as horizontal rows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e942d6d", + "metadata": {}, + "outputs": [], + "source": [ + "class CustomRegressionReportBlocks(ReportBlockMixin):\n", + " \"\"\"User-defined blocks for a house prices regression explainability report.\"\"\"\n", + "\n", + " @block\n", + " def block_residual_error_summary(self, title: str = \"Residual error summary\"):\n", + " \"\"\"Summarize residual dispersion and global regression performance indicators.\"\"\"\n", + " if self.y_test is None or self.y_pred is None:\n", + " raise ValueError(\"residual_error_summary block requires y_test and y_pred.\")\n", + "\n", + " # Convert arrays to aligned pandas Series for simple metrics computation.\n", + " y_true = pd.Series(self.y_test).reset_index(drop=True)\n", + " y_pred_series = pd.Series(self.y_pred).reset_index(drop=True)\n", + " residuals = y_true - y_pred_series\n", + " abs_residuals = residuals.abs()\n", + "\n", + " summary_df = pd.DataFrame(\n", + " [\n", + " [\"MAE\", f\"{mean_absolute_error(y_true, y_pred_series):,.2f}\"],\n", + " [\"MSE\", f\"{mean_squared_error(y_true, y_pred_series):,.2f}\"],\n", + " [\"R2\", f\"{r2_score(y_true, y_pred_series):.3f}\"],\n", + " [\"Residual mean\", f\"{residuals.mean():,.2f}\"],\n", + " [\"Residual std\", f\"{residuals.std():,.2f}\"],\n", + " [\"Median absolute error\", f\"{abs_residuals.median():,.2f}\"],\n", + " [\"95th pct absolute error\", f\"{abs_residuals.quantile(0.95):,.2f}\"],\n", + " ],\n", + " columns=[\"Metric\", \"Value\"],\n", + " )\n", + "\n", + " explanation = (\n", + " \"This section summarizes global regression error levels. \"\n", + " \"A strong gap between median and 95th percentile absolute error usually highlights \"\n", + " \"a subset of difficult cases worth deeper explainability analysis.\"\n", + " )\n", + " return title, [explanation, summary_df]\n", + "\n", + " @block\n", + " def block_largest_errors_focus(self, title: str = \"Largest absolute errors\", top_k: int = 10):\n", + " \"\"\"Display samples with the largest absolute errors to prioritize local explainability reviews.\"\"\"\n", + " explainer = self._require_explainer(\"largest_errors_focus\")\n", + " if self.y_test is None or self.y_pred is None:\n", + " raise ValueError(\"largest_errors_focus block requires y_test and y_pred.\")\n", + "\n", + " # Rebuild a detailed table mixing predictions, residuals, and business context columns.\n", + " y_true = pd.Series(self.y_test, index=explainer.x_init.index, name=\"true\")\n", + " y_pred_series = pd.Series(self.y_pred, index=explainer.x_init.index, name=\"pred\")\n", + " details = pd.concat([y_true, y_pred_series, explainer.x_init], axis=1)\n", + " details[\"residual\"] = details[\"true\"] - details[\"pred\"]\n", + " details[\"abs_error\"] = details[\"residual\"].abs()\n", + "\n", + " focus = details.sort_values(\"abs_error\", ascending=False).head(top_k).copy()\n", + " if focus.empty:\n", + " return title, [\"No rows available to compute largest errors.\"]\n", + "\n", + " focus = focus.rename(\n", + " columns={\"true\": \"True\", \"pred\": \"Pred\", \"residual\": \"Residual\", \"abs_error\": \"AbsError\"}\n", + " )\n", + "\n", + " # Keep a few business-relevant columns first when they are available.\n", + " preferred = [\"OverallQual\", \"GrLivArea\", \"TotalBsmtSF\", \"GarageArea\", \"Neighborhood\"]\n", + " context_cols = [c for c in preferred if c in focus.columns]\n", + " leading = [\"True\", \"Pred\", \"Residual\", \"AbsError\"]\n", + " trailing = [c for c in focus.columns if c not in leading + context_cols]\n", + " focus = focus[leading + context_cols + trailing]\n", + "\n", + " for col in [\"True\", \"Pred\", \"Residual\", \"AbsError\"]:\n", + " focus[col] = focus[col].map(lambda x: round(float(x), 2))\n", + "\n", + " info = (\n", + " \"Rows below are the largest absolute errors. \"\n", + " \"They are ideal candidates for local contribution plots and feature-level investigation.\"\n", + " )\n", + " table = focus.reset_index(drop=False)\n", + " return title, [info, table]" + ] + }, + { + "cell_type": "markdown", + "id": "9d64d0f8", + "metadata": {}, + "source": [ + "## 8. Generate a Custom Regression Report\n", + "\n", + "We now use the custom YAML layout already written for this tutorial and the custom block class defined above.\n", + "\n", + "This second report combines built-in Shapash sections with project-specific regression diagnostics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ac526f3", + "metadata": {}, + "outputs": [], + "source": [ + "custom_report_path = OUTPUT_DIR / \"regression_report_custom.html\"\n", + "custom_config_path = CONFIG_DIR / \"custom_report_regression_house_prices.yml\"\n", + "\n", + "custom_blocks = CustomRegressionReportBlocks(\n", + " explainer=xpl,\n", + " x_train=xtrain,\n", + " y_train=ytrain,\n", + " y_test=ytest,\n", + ")\n", + "\n", + "xpl.generate_report(\n", + " output_file=str(custom_report_path),\n", + " yaml_path=str(custom_config_path),\n", + " block_instance=custom_blocks,\n", + ")\n", + "\n", + "print(f\"Custom report generated: {custom_report_path}\")" + ] + }, + { + "cell_type": "markdown", + "id": "f5c1802d", + "metadata": {}, + "source": [ + "## 9. What to Customize Next\n", + "\n", + "Once this notebook works end to end, the usual next step is to adapt it to your own project.\n", + "\n", + "In practice, you will typically modify:\n", + "- the dataset loading logic\n", + "- the preprocessing pipeline\n", + "- the trained model\n", + "- the default or custom YAML configuration\n", + "- the custom report blocks needed by your use case\n", + "\n", + "Open the generated HTML files in your browser to review the result and iterate on the YAML and custom blocks." + ] + }, + { + "cell_type": "markdown", + "id": "d08ecf1f", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv (3.12.8.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorial/generate_report/tuto-shapash-report01.ipynb b/tutorial/generate_report/tuto-shapash-report01.ipynb deleted file mode 100644 index 6b85bff6..00000000 --- a/tutorial/generate_report/tuto-shapash-report01.ipynb +++ /dev/null @@ -1,386 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "furnished-present", - "metadata": {}, - "source": [ - "# Shapash Report\n", - "\n", - "> The Shapash Report feature allows data scientists to deliver to anyone who is interested in their project **a document that freezes different aspects of their work as a basis of an audit report**. This document can be easily shared across teams and does not require anything else than a working internet connexion.\n", - "\n", - "The shapash `generate_report` method allows to generate a report of your project. \n", - "The result is a standalone HTML file that does not require any external dependency or server to work. \n", - "The only requirement for the document to display properly is an active internet connexion. \n", - "\n", - "The report contains the following information :\n", - "1. General information about the project\n", - "2. Description of the dataset used\n", - "3. Documentation about data preparation and feature engineering\n", - "4. Details about your model used (library, parameters...)\n", - "5. Exploration of the data with a focus on the difference between train and test sets\n", - "6. Global explainability of the model\n", - "7. Model performance\n", - "\n", - "> The first three points are generated using a YML file that the user should fill. An example is available [here](https://github.com/MAIF/shapash/blob/master/tutorial/report/utils/project_info.yml).\n", - "\n", - "This tutorial presents an example of how one can generate the Shapash Report. \n", - "\n", - "Content:\n", - "- Set up an example project\n", - "- Create and fill your project information that will be displayed in the report\n", - "- Generate the base Shapash Report\n", - "- *Go further*: Generate a custom report\n", - "\n", - "Data from Kaggle [House Prices](https://www.kaggle.com/c/house-prices-advanced-regression-techniques/data)\n", - "\n", - "> Note : you may need to download the HTML report locally and open it in your browser otherwise it may not show properly." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "automatic-exemption", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from shapash.utils.dtypes import text_like_columns\n", - "from category_encoders import OrdinalEncoder\n", - "from sklearn.ensemble import RandomForestRegressor\n", - "from sklearn.model_selection import train_test_split" - ] - }, - { - "cell_type": "markdown", - "id": "familiar-charm", - "metadata": {}, - "source": [ - "## Building Supervized Model " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "subtle-sheet", - "metadata": {}, - "outputs": [], - "source": [ - "from shapash.data.data_loader import data_loading\n", - "house_df, house_dict = data_loading('house_prices')\n", - "y_df=house_df['SalePrice']\n", - "X_df=house_df[house_df.columns.difference(['SalePrice'])]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "colored-indie", - "metadata": {}, - "outputs": [], - "source": [ - "from category_encoders import OrdinalEncoder\n", - "\n", - "categorical_features = text_like_columns(X_df, strict_object=False)\n", - "\n", - "encoder = OrdinalEncoder(\n", - " cols=categorical_features,\n", - " handle_unknown='return_nan',\n", - " return_df=True).fit(X_df)\n", - "\n", - "X_df = encoder.transform(X_df)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "hispanic-leave", - "metadata": {}, - "outputs": [], - "source": [ - "Xtrain, Xtest, ytrain, ytest = train_test_split(X_df, y_df, train_size=0.75, random_state=1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "increased-shuttle", - "metadata": {}, - "outputs": [], - "source": [ - "regressor = RandomForestRegressor(n_estimators=50).fit(Xtrain, ytrain)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "floral-cleaners", - "metadata": {}, - "outputs": [], - "source": [ - "y_pred = pd.DataFrame(regressor.predict(Xtest),columns=['pred'], index=Xtest.index)" - ] - }, - { - "cell_type": "markdown", - "id": "satisfactory-baptist", - "metadata": {}, - "source": [ - "## Fill your project information\n", - "\n", - "**The next step is to create a YML file containing information about your project.** \n", - "\n", - "We will use the example file available [here](https://github.com/MAIF/shapash/blob/master/tutorial/report/utils/project_info.yml). \n", - "**You are welcome to use this file as a template for your own report.** \n", - "\n", - "We display the information contained in the YML file below :" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "searching-northern", - "metadata": {}, - "outputs": [], - "source": [ - "import yaml\n", - "\n", - "with open(r'utils/project_info.yml') as file:\n", - " project_info = yaml.full_load(file)\n", - "\n", - "print(yaml.dump(project_info, sort_keys=False))" - ] - }, - { - "cell_type": "markdown", - "id": "northern-bahrain", - "metadata": {}, - "source": [ - "---\n", - "**If you want to create your own custom file :**\n", - "\n", - "The keys of the YML file are the titles of the different sections in the report. \n", - "The YML file must then respect the following format:\n", - "\n", - "```yaml\n", - "Title of section 1: \n", - " property1 name: property1 value \n", - " property2 name: property2 value \n", - " ...\n", - "Title of section 2: \n", - " property1 name: property1 value \n", - " ...\n", - "```\n", - "> Note that the **date** can be computed automatically using the *auto* property value (see example above)" - ] - }, - { - "cell_type": "markdown", - "id": "optimum-description", - "metadata": {}, - "source": [ - "## Generate your report" - ] - }, - { - "cell_type": "markdown", - "id": "suited-honor", - "metadata": {}, - "source": [ - "### Declare and compile SmartExplainer object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "automated-hundred", - "metadata": {}, - "outputs": [], - "source": [ - "from shapash import SmartExplainer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "consecutive-venice", - "metadata": {}, - "outputs": [], - "source": [ - "xpl = SmartExplainer(\n", - " model=regressor,\n", - " preprocessing=encoder, # Optional: compile step can use inverse_transform method\n", - " features_dict=house_dict # optional parameter, specifies label for features name \n", - ") " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "rising-person", - "metadata": {}, - "outputs": [], - "source": [ - "xpl.compile(x=Xtest, y_pred=y_pred, y_target=ytest)" - ] - }, - { - "cell_type": "markdown", - "id": "focal-liberty", - "metadata": {}, - "source": [ - "At this step the model can be checked and inspected using different methods of the SmartExplainer object we just created. \n", - "\n", - "Please refer to the other tutorials for more information." - ] - }, - { - "cell_type": "markdown", - "id": "talented-shadow", - "metadata": {}, - "source": [ - "### Generate the base Shapash Report\n", - "\n", - "Next we can generate the report using the `generate_report` method of our SmartExplainer object.\n", - "\n", - "We need to pass `x_train`, `y_train` and `y_test` parameters in order to explore the data used when training the model.\n", - "\n", - "Please refer to the documentation for a full description of the parameters.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "returning-effort", - "metadata": {}, - "outputs": [], - "source": [ - "xpl.generate_report(\n", - " output_file='output/report.html', \n", - " project_info_file='utils/project_info.yml',\n", - " x_train=Xtrain,\n", - " y_train=ytrain,\n", - " y_test=ytest,\n", - " title_story=\"House prices report\",\n", - " title_description=\"\"\"This document is a data science report of the kaggle house prices tutorial project. \n", - " It was generated using the Shapash library.\"\"\",\n", - " metrics=[\n", - " {\n", - " 'path': 'sklearn.metrics.mean_absolute_error',\n", - " 'name': 'Mean absolute error', \n", - " },\n", - " {\n", - " 'path': 'sklearn.metrics.mean_squared_error',\n", - " 'name': 'Mean squared error',\n", - " }\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "central-karma", - "metadata": {}, - "source": [ - "> Note: You might want to specify the jupyter kernel used when generating the report.\n", - "You should consider using the `kernel_name` parameter to indicate what kernel to use." - ] - }, - { - "cell_type": "markdown", - "id": "elementary-sympathy", - "metadata": {}, - "source": [ - "## Customize your own report" - ] - }, - { - "cell_type": "markdown", - "id": "grateful-terror", - "metadata": {}, - "source": [ - "Now let's customize our report by adding some new sections.\n", - "\n", - "To do so :\n", - "- First, **copy the base report notebook** you can find [here](https://github.com/MAIF/shapash/blob/master/shapash/report/base_report.ipynb). This is the notebook that is used to generate the shapash report. It is executed and then converted to an HTML file. Only the output of each cell is kept and the code is deleted.\n", - "- Then, delete or add cells depending on what you want to change.\n", - "- Finally, add the parameter `notebook_path=\"path/to/your/custom/report.ipynb\"` in the `generate_report` method.\n", - "\n", - "> **Tip** : You can use the `working_dir` parameter to easily work inside your custom notebook before using the `generate_report` method. This way you can load the parameters used inside the notebook by papermill. Replace the `dir_path` inside your custom notebook with your own `working_dir` where are saved the different instances used.\n", - "\n", - "For our simple example, we created [this notebook](https://github.com/MAIF/shapash/blob/master/tutorial/report/utils/custom_report.ipynb). \n", - "- We removed the multivariate analysis using the `report.display_dataset_analysis(multivariate_analysis=False)` (see notebook utils/custom_report.ipynb for more information)\n", - "- It includes new sections **Relashionship with target variable** and **Relashionship between training variables** in which we included new simple graphs for this example. \n", - "- We also added new cells at the end of the **metrics** section.\n", - "\n", - "Next, we use this notebook to generate our new custom report :" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "played-vegetation", - "metadata": {}, - "outputs": [], - "source": [ - "xpl.generate_report(\n", - " output_file='output/custom_report.html', \n", - " project_info_file='utils/project_info.yml',\n", - " x_train=Xtrain,\n", - " y_train=ytrain,\n", - " y_test=ytest,\n", - " title_story=\"House prices report\",\n", - " title_description=\"\"\"This document is a data science report of the kaggle house prices tutorial project. \n", - " It was generated using the Shapash library.\"\"\",\n", - " metrics=[\n", - " {\n", - " 'path': 'sklearn.metrics.mean_absolute_error',\n", - " 'name': 'Mean absolute error', \n", - " },\n", - " {\n", - " 'path': 'sklearn.metrics.mean_squared_error',\n", - " 'name': 'Mean squared error',\n", - " }\n", - " ],\n", - " working_dir='working',\n", - " notebook_path=\"utils/custom_report.ipynb\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "direct-cheese", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "hide_input": false, - "kernelspec": { - "display_name": "Python 3.9.13", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - }, - "vscode": { - "interpreter": { - "hash": "6dbaec60c0b0d722a3fa908c2fd7b738d946da6332c67fea5eea602801fdaf43" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tutorial/generate_report/utils/custom_report.ipynb b/tutorial/generate_report/utils/custom_report.ipynb deleted file mode 100644 index 29838ec0..00000000 --- a/tutorial/generate_report/utils/custom_report.ipynb +++ /dev/null @@ -1,290 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "threatened-gamma", - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "# These parameter are replaced by papermill during execution but can be used to work interactively on your report\n", - "# You need to use the generate_report once with the parameter working_dir='../working' \n", - "# to use the following values. This way the objects used below are created in the directory.\n", - "dir_path = '../working' \n", - "project_info_file = '../utils/project_info.yml'\n", - "config = dict(\n", - " title_story=\"House prices report\",\n", - " title_description=\"\"\"This document is a data science report of the kaggle house prices tutorial project. \n", - " It was generated using the Shapash library.\"\"\",\n", - " metrics=[\n", - " {\n", - " 'path': 'sklearn.metrics.mean_absolute_error',\n", - " 'name': 'Mean absolute error', \n", - " },\n", - " {\n", - " 'path': 'sklearn.metrics.mean_squared_error',\n", - " 'name': 'Mean squared error',\n", - " }\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "taken-tomorrow", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import pandas as pd\n", - "from shapash import SmartExplainer\n", - "from shapash.report.project_report import ProjectReport\n", - "from shapash.report.common import load_saved_df\n", - "\n", - "xpl = SmartExplainer.load(os.path.join(dir_path, 'smart_explainer.pickle'))\n", - "\n", - "x_train = load_saved_df(os.path.join(dir_path, 'x_train.csv'))\n", - "y_train = load_saved_df(os.path.join(dir_path, 'y_train.csv'))\n", - "y_test = load_saved_df(os.path.join(dir_path, 'y_test.csv'))\n", - "\n", - "report = ProjectReport(\n", - " explainer=xpl, \n", - " project_info_file=project_info_file, \n", - " x_train=x_train, \n", - " y_train=y_train,\n", - " y_test=y_test, \n", - " config=config\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "peaceful-frame", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_title_description()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "decreased-philadelphia", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_project_information()" - ] - }, - { - "cell_type": "markdown", - "id": "fourth-confusion", - "metadata": {}, - "source": [ - "## Model information" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "union-person", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_model_analysis()" - ] - }, - { - "cell_type": "markdown", - "id": "regional-centre", - "metadata": {}, - "source": [ - "## Dataset analysis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "rational-breakfast", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_dataset_analysis(multivariate_analysis=False)" - ] - }, - { - "cell_type": "markdown", - "id": "fitted-uncle", - "metadata": {}, - "source": [ - "### Relashionship with target variable" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "front-employment", - "metadata": {}, - "outputs": [], - "source": [ - "import seaborn as sns\n", - "import matplotlib.pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "collectible-upgrade", - "metadata": {}, - "outputs": [], - "source": [ - "df_train = report.x_train_pre\n", - "y_train = report.y_train\n", - "df_train['SalePrice'] = y_train" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "monthly-reply", - "metadata": {}, - "outputs": [], - "source": [ - "f, ax = plt.subplots(figsize=(8, 6))\n", - "fig = sns.boxplot(x='OverallQual', y=\"SalePrice\", data=df_train)\n", - "fig.axis(ymin=0, ymax=800000)\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "subtle-amazon", - "metadata": {}, - "source": [ - "### Relashionship between training variables" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "packed-vermont", - "metadata": {}, - "outputs": [], - "source": [ - "corr_matrix = df_train.corr()\n", - "f, ax = plt.subplots(figsize=(16, 12))\n", - "sns.heatmap(corr_matrix, vmax=.8, square=True, cmap=\"YlGnBu\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "romance-division", - "metadata": {}, - "source": [ - "## Model explainability" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "accessible-favorite", - "metadata": {}, - "outputs": [], - "source": [ - "# Note : Plotly graphs may not show correctly in notebook but still work in html output file.\n", - "report.display_model_explainability()" - ] - }, - { - "cell_type": "markdown", - "id": "unknown-transaction", - "metadata": {}, - "source": [ - "## Model performance" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ignored-career", - "metadata": {}, - "outputs": [], - "source": [ - "report.display_model_performance()" - ] - }, - { - "cell_type": "markdown", - "id": "noble-seafood", - "metadata": {}, - "source": [ - "**The graph below represents y_pred vs y_test :**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "filled-challenge", - "metadata": {}, - "outputs": [], - "source": [ - "y_test = report.y_test\n", - "y_pred = report.y_pred\n", - "\n", - "sns.scatterplot(x=y_test, y=y_pred)\n", - "plt.xlabel('y_test')\n", - "plt.ylabel('y_pred')\n", - "plt.title('y_pred vs y_test')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "efficient-badge", - "metadata": {}, - "source": [ - "You can add as many graphs, text, or other cells as you want.\n", - "\n", - "The code will not be displayed. Only the markdown and output of the cells will be shown on the generated html file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "passive-peoples", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "celltoolbar": "Tags", - "hide_input": false, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tutorial/generate_report/utils/project_info.yml b/tutorial/generate_report/utils/project_info.yml deleted file mode 100644 index 2cd415c7..00000000 --- a/tutorial/generate_report/utils/project_info.yml +++ /dev/null @@ -1,32 +0,0 @@ -General information: - version: 0.7 - name: House Prices Prediction Project - purpose: Predicting the sale price of houses - date: auto - contributors: Yann Golhen, Sebastien Bidault, Thomas Bouche, Guillaume Vignal, Thibaud Real - description: This work is a data science project that tries to predict the sale of houses based on 79 explanatory variables. It was designed inside the data science team at X. and improved since the beggining of the project in 2019. The model was put into production since February 2021. - source code: https://github.com/MAIF/shapash/tree/master/tutorial - Git commit: 1ff46e83beafba8949a7f3b7de27586acd6ae99e - -Dataset information: - path: https://www.kaggle.com/c/house-prices-advanced-regression-techniques/ - origin: The Assessor’s Office - description: the sale of individual residential property in Ames, Iowa - depth: from 2006 to 2010 - perimeter: only residential sales - target variable: SalePrice - target description: The property's sale price in dollars - -Data Preparation: - variable filetring: All variables that required special knowledge or previous calculations for their use were removed - individual filtering: only the most recent sales data on any property were kept (for houses that were sold multiple times during this period) - missing values: were replaced by 0 - Feature engineering: No feature was created. All features are directly taken from the kaggle dataset. Categorical features were transformed using an ordinal encoder. - Path to script: https://github.com/MAIF/shapash/tree/master/tutorial/ - -Model training: - Used algorithm: We used a RandomForestRegressor algorithm (scikit-learn) but this model could be challenged with other interesting models such as XGBRegressor, Neural Networks, ... - Parameters choice: We did not perform any hyperparameter optimisation and chose to use `n_estimators=50`. Future works should be planned to perform gridsearch optimizations - Metrics: Mean Squared Error metric - Validation strategy: We splitted our data into train (75%) and test (25%) - Path to script: https://github.com/MAIF/shapash/tree/master/tutorial/