diff --git a/Cargo.lock b/Cargo.lock index 597378b51d..36b7a8e2a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -330,7 +330,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -341,7 +341,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2315,7 +2315,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2527,7 +2527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3599,7 +3599,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.5", "system-configuration", "tokio", "tower-service", @@ -5346,7 +5346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5679,7 +5679,7 @@ dependencies = [ [[package]] name = "plotly" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#60a4ffc07830ce429de0498e4a7c83671486a298" +source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#c071112240c8d8a6eb9717853668cd110bdc5129" dependencies = [ "askama", "async-trait", @@ -5702,7 +5702,7 @@ dependencies = [ [[package]] name = "plotly_derive" version = "0.14.1" -source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#60a4ffc07830ce429de0498e4a7c83671486a298" +source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#c071112240c8d8a6eb9717853668cd110bdc5129" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5713,7 +5713,7 @@ dependencies = [ [[package]] name = "plotly_static" version = "0.1.0" -source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#60a4ffc07830ce429de0498e4a7c83671486a298" +source = "git+https://github.com/dathere/plotly?branch=deps%2Fwebdriver-downloader-0.17%2Bfunnel#c071112240c8d8a6eb9717853668cd110bdc5129" dependencies = [ "anyhow", "base64", @@ -6904,7 +6904,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.5", "thiserror 2.0.19", "tokio", "tracing", @@ -6943,9 +6943,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7755,7 +7755,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7814,7 +7814,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8390,7 +8390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8515,7 +8515,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8766,10 +8766,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8782,7 +8782,7 @@ dependencies = [ "parking_lot", "rustix", "signal-hook 0.3.18", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8818,7 +8818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10075,7 +10075,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2005073916..50e43b3d27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -472,6 +472,8 @@ whatlang = { git = "https://github.com/jqnatividad/whatlang-rs", branch = "bump- # - new trace types Splom, Parcats (PR #425) — used by qsv `viz splom`/`viz parcats` # - webdriver-downloader 0.17 (PR #427) — used by qsv `viz_static` for headless Chromium/Firefox export # - Funnel trace + layout funnelmode (PR #432) — used by qsv `viz smart` pipeline funnels +# - Waterfall trace (PR #432) — used by qsv `viz smart` when a declared pipeline's stages do +# NOT nest, so the panel bridges the signed gaps instead of asserting containment # When #421/#423/#425/#428/#432 merge upstream, repoint this to a plotly.rs release/main rev. plotly = { git = "https://github.com/dathere/plotly", branch = "deps/webdriver-downloader-0.17+funnel" } diff --git a/docs/help/viz.md b/docs/help/viz.md index d9cc07344d..f35b6b16f6 100644 --- a/docs/help/viz.md +++ b/docs/help/viz.md @@ -109,18 +109,26 @@ that contains the value, and gains a "vs target" DELTA when it supplies an with no headline measure. (Overall dataset completeness - the share of non-empty cells - is a quiet "Completeness:" line in the header metadata table, not a KPI tile.) -- pipeline funnel, when the dictionary DECLARES one (see --dictionary). Which +- pipeline panel, when the dictionary DECLARES one (see --dictionary). Which columns are process stages, and in which direction, is semantics rather than a statistic - no column-name vocabulary settles it and no statistic does -either - so a funnel is drawn ONLY from an explicit declaration, never +either - so the panel is drawn ONLY from an explicit declaration, never guessed. Both encodings are supported: stages held in separate measure columns, and stages held as values of one category column. Costs one extra data pass over the declared stages only. +The declaration fixes WHICH columns and in WHAT order; the numbers decide the +FORM. A funnel's band widths are a containment claim, so one is drawn only +while the stage totals never grow. If any stage outruns the one before it, the +same declaration is drawn as a BRIDGE instead: the signed difference between +consecutive totals, each step labelled as the arithmetic difference it is +rather than as a flow. A funnel there would render a band wider than the one +above it, asserting the opposite of the data. The subtitle says which form was +used and why. Stage order is the declared order and is never re-sorted by size. For the column encoding, row-wise containment (does each stage nest inside the one -before it?) is MEASURED and disclosed in the subtitle, but never refuses the -panel: a pipeline whose stages overrun is a finding to name, not a reason to -show nothing. Totals sum over the rows complete across every declared stage, +before it?) is MEASURED and disclosed in the subtitle - separately from the +form, since rows can overrun while the totals still shrink, or nest while the +totals grow. Totals sum over the rows complete across every declared stage, so they do NOT match `stats.sum`; the subtitle always discloses that denominator. See also the standalone `qsv viz funnel` chart type, which takes its stage order from the file and needs no dictionary. @@ -447,7 +455,7 @@ qsv viz --help |  `‑‑no‑other`  | flag | Omit the "Other (N)" aggregate bar from frequency bar charts. It collects the categories beyond --limit (N = how many distinct categories were rolled up) and is shown by default. When just ONE category is left over, it is charted under its own name instead of as an opaque "Other (1)" bar. | | |  `‑‑smarter`  | flag | Before building the dashboard, run `qsv moarstats --advanced` to enrich the stats cache with distribution-shape statistics (bimodality, entropy, skewness, outlier share, Gini). This unlocks histograms for bimodal columns, frequency bars for concentrated high-cardinality columns, skew/outlier hints on box panels, and Lorenz curves for the most unequal additive measures (high Gini). Costs one extra pass over the data and writes .stats.csv, its sidecars, and an .idx index (like running `qsv moarstats` manually). On geocode-enabled builds it also enriches map point hovers with the US FIPS code and annotates the spatial-extent summary with the country's continent; the county is always shown in map hovers, with or without --smarter. Only affects `smart`. Applied only with default parsing; inputs using --no-headers or a custom --delimiter fall back to the standard dashboard. | | |  `‑‑hierarchy‑style`  | string | For `smart`, the chart used for the categorical part-to-whole hierarchy panel (built when 2+ low-cardinality dimensions exist). One of: auto (default), treemap, sunburst, icicle. auto follows best practice — a treemap for a shallow 2-level hierarchy (accurate size comparison) and a sunburst for a deep 3-level one (parent child structure); icicle is an opt-in level-aligned alternative. Only affects `smart`. | | -|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved schema to fine-tune). A "gauge_range" of [min, max] on a continuous numeric measure renders its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent); qsv keeps it only when the observed data lies within the range, so a mis-scaled range can't draw a misleading dial, and "infer" emits it for canonical-scale measures. A "target" number on a measure renders a "vs target" DELTA against that goal (value minus target) - a GOAL you supply, never a fabricated prior-period baseline, so "infer" never emits it; hand-author it. The dictionary is also the ONLY source of the pipeline funnel panel, declared in the dataset-level "x-qsv" object as a "relationships" entry with "kind": "pipeline". Two encodings, both hand-editable: stages as COLUMNS - "members" lists the stage columns in process order, WIDEST/UPSTREAM FIRST (note this is the opposite direction from a "kind":"ordered" group, which ascends), e.g. {"kind":"pipeline", "members":["planned_amt","committed_amt","spent_amt"]} stages as ROW VALUES - "stage_column" names the category column, "stages" lists its values in process order, and an optional "value_column" names the measure to sum per stage (omit it to count rows), e.g. {"kind":"pipeline","members":["stage","revenue"], "stage_column":"stage", "stages":["Impression","Click","Lead","Conversion"], "value_column":"revenue"} Declared order is authoritative and is never re-sorted by size, so a stage that outruns its predecessor stays visible instead of being quietly reordered away. A declaration naming a missing column, or a stage that is an average/rate rather than a summable amount, is skipped with a note rather than erroring. Only affects `smart`. | | +|  `‑‑dictionary`  | string | Use a describegpt Data Dictionary to guide panel selection from each field's semantic role/concept (falling back to its content type) instead of relying on column statistics alone: dimensions and numeric codes (ward, census_tract, zone) become bars, measures get box/correlation/trend panels, date/datetime columns feed the time-series panel (not noisy frequency bars), identifiers / PII / free-text are skipped, and lat/lon feed the map. Field labels are shown as panel subtitles beneath the field-name titles. Columns the dictionary cannot classify still use the statistical heuristic. is one of: "infer" to run describegpt on the input now (with description, infer-content-type, two-pass and jsonschema output; requires an LLM configured) and use its output; or a path to an existing describegpt dictionary file (jsonschema or json). With "infer", the generated dictionary is saved beside the input as .schema.json so you can fine-tune it; if that file already exists, it is reused as-is (skipping the LLM) - edit it to fine-tune, or delete it to force a fresh re-infer. Generation/read failures soft-fall back to the stats-only dashboard. The dictionary also drives the KPI overview row via two optional per-field hints in a property's "x-qsv" object (edit them in the saved schema to fine-tune). A "gauge_range" of [min, max] on a continuous numeric measure renders its KPI tile as a GAUGE on that canonical scale (e.g. [0,1] for a ratio, [0,100] for a percent); qsv keeps it only when the observed data lies within the range, so a mis-scaled range can't draw a misleading dial, and "infer" emits it for canonical-scale measures. A "target" number on a measure renders a "vs target" DELTA against that goal (value minus target) - a GOAL you supply, never a fabricated prior-period baseline, so "infer" never emits it; hand-author it. The dictionary is also the ONLY source of the pipeline panel (drawn as a funnel, or as a bridge when the stage totals do not nest), declared in the dataset-level "x-qsv" object as a "relationships" entry with "kind": "pipeline". Two encodings, both hand-editable: stages as COLUMNS - "members" lists the stage columns in process order, WIDEST/UPSTREAM FIRST (note this is the opposite direction from a "kind":"ordered" group, which ascends), e.g. {"kind":"pipeline", "members":["planned_amt","committed_amt","spent_amt"]} stages as ROW VALUES - "stage_column" names the category column, "stages" lists its values in process order, and an optional "value_column" names the measure to sum per stage (omit it to count rows), e.g. {"kind":"pipeline","members":["stage","revenue"], "stage_column":"stage", "stages":["Impression","Click","Lead","Conversion"], "value_column":"revenue"} Declared order is authoritative and is never re-sorted by size, so a stage that outruns its predecessor stays visible instead of being quietly reordered away - it switches the panel to a bridge rather than drawing a funnel that widens. A declaration naming a missing column, or a stage that is an average/rate rather than a summable amount, is skipped with a note rather than erroring. Only affects `smart`. | | |  `‑‑dictionary‑context`  | string | Path to a file with extra context about the dataset (a glossary, README, data dictionary, PDF, etc.) forwarded to describegpt as --context-file when `--dictionary infer` generates the dictionary. Better context yields better role/concept/label/grain tags, hence a better dashboard. Ignored unless `--dictionary infer` is used (it does not apply when reading an existing dictionary file). Only affects `smart`. | | |  `‑‑dict‑info`  | flag | When a usable Data Dictionary is available (per --dictionary), add a "Data Dictionary" link beneath the dashboard title and an info icon on each panel title: hovering shows that column's dictionary description; clicking opens a human-friendly rendering of the dictionary in a side drawer NEXT TO the plots (embedded in the dashboard file - no extra file is written), scrolled to and highlighting that column's entry. The drawer is open by default on load and can be dismissed with its close button or Esc. The dictionary page carries a role-tinted table of contents and per-column "View chart" links back to the panels, plus a row of download buttons: the dictionary itself as JSON Schema, the frequency counts the dashboard actually charted, and every generated sidecar this run read (the stats cache and its metadata, the frequency cache when it was reused, and the bivariate stats CSV when freshly written). A sidecar qsv wrote but viz never read - the human-readable .stats.csv - is NOT offered, since nothing can show it describes the same computation the dashboard used. Every file is BUNDLED into the HTML, so anyone you send the dashboard to can download them with no access to your machine; absolute local paths are stripped from the embedded metadata so sharing a dashboard doesn't disclose your directory layout. Sidecars over 4 MB are skipped with a note. The drawer's popout button opens the same document in its own browser tab instead (needs a browser that allows user-initiated pop-ups). HTML output only; ignored with a note when no dictionary is available or when exporting an image. Only affects `smart`. | | |  `‑‑dataset‑pid`  | string | A persistent identifier (PID) for the dataset - typically a full URL such as a DOI () or other citable link. When set, a "PID" row is added to the metadata table at the top of the dashboard. http(s) and mailto values become a clickable link (opened in a new tab); any other scheme is shown as plain text rather than linked. HTML output only. Only affects `smart`. | | diff --git a/examples/viz/README.md b/examples/viz/README.md index baed8435b0..883a2f3749 100644 --- a/examples/viz/README.md +++ b/examples/viz/README.md @@ -17,7 +17,7 @@ qsv and running [`gen_gallery.py`](gen_gallery.py) from the repo root — `python3 examples/viz/gen_gallery.py`. Individual `qsv viz` outputs are instead fully self-contained (plotly embedded), so they work offline. -The fourteen **smart dashboards** are embedded as ` @@ -43,66 +43,68 @@

qsv viz gallery (An Interactive Viz is worth a Million Rows)

-o smart_dashboard_kpi_gauges_target_delta.html
smart dashboard (--smarter)Same auto-profiler with `--smarter`, which runs `qsv moarstats --advanced` itself to enrich the stats cache in one step: the bimodal monthly_spend column renders as a histogram (a box plot would hide its two peaks), and the skewed account_age_days box is annotated with its skew direction and outlier share.
qsv viz smart customer_spend.csv --smarter --max-charts 8 -o smart_dashboard_smarter.html
smart dashboard (--smarter, Gini/Lorenz inequality + log-skew boxes)Medicare payments to individual practitioners (CMS — a 30k sample), with medical_payment and total_patients per provider. These are additive amounts across comparable units, so --smarter recognizes them as inequality measures and adds a Lorenz curve for each (Gini 0.65 / 0.61) — the further the curve bows below the diagonal, the more concentrated the payments. The distribution boxes are skew-aware: a heavily right-skewed money column would squash a linear box into a sliver against its largest values, so viz smart draws it on a log axis instead, keeping the median and quartiles legible.
qsv viz smart cms_medicare_providers.csv --smarter -o smart_dashboard_smarter_gini_lorenz_inequality_log_skew_boxes.html
-
smart dashboard (--smarter, zero-inflated capital pipeline)NYC Capital Projects Database (12,587 projects, sourced from Checkbook NYC), the dataset this dashboard's inequality work was designed against. Its money columns are both extremely concentrated (Gini 0.93–0.96) and heavily zero-inflated, which the Lorenz panels state outright: flat run = 60% zeros, not small values. That long flat opening run is not a mass of small projects — it is the projects with nothing committed or spent yet, a funding-pipeline stage rather than a have-not population. Every Lorenz panel also carries the unit caveat, because a Gini across units as unlike a subway extension and a playground resurfacing is close to tautological and must not be read as inequity.

It also carries the pipeline funnel, declared by nyc_capital_projects_dict.schema.json as an x-qsv.relationships entry of kind: "pipeline". Which columns are stages, and in which direction, is semantics rather than a statistic, so a funnel is only ever drawn from an explicit declaration — never guessed from column names.

Read the subtitle: these stages do not nest. Spent totals 2.9× committed, because the three are independent aggregates on different bases — totalplannedcommit is allocated in the Capital Commitment Plan, while the other two are sums within the City's budget. Rather than refuse the panel, the funnel draws and names the violation: Spent exceeds Committed in 46% of rows. Declared order is never re-sorted by size, so the overrun stays visible instead of being quietly tidied away.
qsv viz smart nyc_capital_projects.csv --smarter --dictionary nyc_capital_projects_dict.schema.json \
+
smart dashboard (dictionary-declared pipeline funnel)A synthetic product-onboarding funnel — visits → signups → activated → subscribed, one row per channel per week. The four stage columns are declared as a pipeline in onboarding_funnel_dict.schema.json (x-qsv.relationships, kind: "pipeline"); which columns are stages, and in which direction, is semantics rather than a statistic, so the panel is only ever drawn from an explicit declaration — never guessed from column names.

Here the stages genuinely nest: every row is a strict subset of the one before it, and so are the totals. That is what earns a funnel, whose band widths are a containment claim — plotly computes the stage-to-stage conversion from the bar values themselves (100% → 41% → 54% → 46%), so the percentages can never drift from the bands. Compare the next figure, where the same kind of declaration over non-nesting totals is drawn as a bridge instead.
qsv viz smart onboarding_funnel.csv --dictionary onboarding_funnel_dict.schema.json \
+  -o smart_dashboard_dictionary_declared_pipeline_funnel.html

+
smart dashboard (--smarter, zero-inflated capital pipeline)NYC Capital Projects Database (12,587 projects, sourced from Checkbook NYC), the dataset this dashboard's inequality work was designed against. Its money columns are both extremely concentrated (Gini 0.93–0.96) and heavily zero-inflated, which the Lorenz panels state outright: flat run = 60% zeros, not small values. That long flat opening run is not a mass of small projects — it is the projects with nothing committed or spent yet, a funding-pipeline stage rather than a have-not population. Every Lorenz panel also carries the unit caveat, because a Gini across units as unlike a subway extension and a playground resurfacing is close to tautological and must not be read as inequity.

It also carries the pipeline bridge, declared by nyc_capital_projects_dict.schema.json as an x-qsv.relationships entry of kind: "pipeline". Which columns are stages, and in which direction, is semantics rather than a statistic, so the panel is only ever drawn from an explicit declaration — never guessed from column names.

The declaration says these columns are one pipeline; it cannot say that they nest, and that is measured. Here they do not: spent totals 2.9× committed, because the three are independent aggregates on different bases — totalplannedcommit is allocated in the Capital Commitment Plan, while the other two are sums within the City's budget. A funnel's band widths are a containment claim, so drawing one would render a band wider than the one above it — an hourglass asserting the opposite of the numbers, which no caption can retract. So viz smart keeps the declaration and changes the form: it bridges the signed gap between consecutive totals, naming each step as the arithmetic difference it is (Committed − Planned), never as a flow. Declared order is still never re-sorted by size, and the subtitle says why the form changed: stages do not nest — bridged, not funnelled.
qsv viz smart nyc_capital_projects.csv --smarter --dictionary nyc_capital_projects_dict.schema.json \
   -o smart_dashboard_smarter_zero_inflated_capital_pipeline.html

 
barRevenue by region (aggregated sum).
qsv viz bar sales_sample.csv --x region --y revenue --agg sum \
-  -o bar.html

+  -o bar.html
bar (animated slider)Revenue by product category, animated over an ordinal column with --slider: each distinct satisfaction rating (1–5) becomes an animation frame, with a ▶ Play/⏸ Pause button and a scrub slider to step through them (Gapminder-style). Axis ranges are pinned across frames so the bars stay comparable frame to frame instead of rescaling. --slider also works on line and scatter, may be split into animated traces with --series, and can accumulate with --slider-cumulative.
qsv viz bar sales_sample.csv --x product_category --y revenue \
-  --agg sum --slider satisfaction -o bar_animated_slider.html
-
lineClosing price over time.
qsv viz line stock_prices.csv --x date --y close -o line.html
-
scatterUnits sold vs revenue.
qsv viz scatter sales_sample.csv --x units_sold --y revenue -o scatter.html
+ --agg sum --slider satisfaction -o bar_animated_slider.html
+
lineClosing price over time.
qsv viz line stock_prices.csv --x date --y close -o line.html
+
scatterUnits sold vs revenue.
qsv viz scatter sales_sample.csv --x units_sold --y revenue -o scatter.html
scatter (bubble)Units vs revenue; marker size = shipping cost, color = profit margin %.
qsv viz scatter sales_sample.csv --x units_sold --y revenue --size shipping_cost \
-  --color profit_margin_pct -o scatter_bubble.html
+ --color profit_margin_pct -o scatter_bubble.html
scatter3dUnits vs revenue vs shipping cost in 3D; marker color = profit margin %.
qsv viz scatter3d sales_sample.csv --x units_sold --y revenue \
-  --z shipping_cost --color profit_margin_pct -o scatter3d.html
-
histogramDistribution of unit price.
qsv viz histogram sales_sample.csv --x unit_price -o histogram.html
-
boxSpread of revenue (Tukey whiskers; points beyond the fences shown as outliers).
qsv viz box sales_sample.csv --y revenue -o box.html
+ --z shipping_cost --color profit_margin_pct -o scatter3d.html
+
histogramDistribution of unit price.
qsv viz histogram sales_sample.csv --x unit_price -o histogram.html
+
boxSpread of revenue (Tukey whiskers; points beyond the fences shown as outliers).
qsv viz box sales_sample.csv --y revenue -o box.html
box (grouped)Revenue spread per region — real Tukey whiskers + every (jittered) point overlaid (--box-points all).
qsv viz box sales_sample.csv --y revenue --x region --box-points all \
-  -o box_grouped.html
-
violinRevenue distribution per region — a KDE density silhouette around an inner quartile box + mean line, revealing shape (modes, shoulders) a box hides. viz smart auto-picks this for columns in the bimodality ambiguity band (--violin auto).
qsv viz violin sales_sample.csv --y revenue --x region -o violin.html
+ -o box_grouped.html
+
violinRevenue distribution per region — a KDE density silhouette around an inner quartile box + mean line, revealing shape (modes, shoulders) a box hides. viz smart auto-picks this for columns in the bimodality ambiguity band (--violin auto).
qsv viz violin sales_sample.csv --y revenue --x region -o violin.html
pie (donut)Revenue share by product category.
qsv viz pie sales_sample.csv --x product_category --y revenue \
-  --donut -o pie_donut.html
-
funnelStage-by-stage drop-off for a signup pipeline encoded as rows (one row per stage per channel, summed per stage). Stages keep the order they first appear in the file, so the data defines the pipeline and nothing is inferred; each band is labelled with its conversion from the previous stage. This is the row-shaped counterpart to the column-shaped pipeline viz smart detects automatically.
qsv viz funnel signup_funnel.csv --x stage --y users -o funnel.html
-
heatmap (correlation)Pearson correlation matrix over numeric columns.
qsv viz heatmap sales_sample.csv -o heatmap_correlation.html
+ --donut -o pie_donut.html
+
funnelStage-by-stage drop-off for a signup pipeline encoded as rows (one row per stage per channel, summed per stage). Stages keep the order they first appear in the file, so the data defines the pipeline and nothing is inferred; each band is labelled with its conversion from the previous stage. This is the row-shaped counterpart to the column-shaped pipeline viz smart detects automatically.
qsv viz funnel signup_funnel.csv --x stage --y users -o funnel.html
+
heatmap (correlation)Pearson correlation matrix over numeric columns.
qsv viz heatmap sales_sample.csv -o heatmap_correlation.html
scatter (correlated pair)The most strongly correlated numeric pair (discount_pct vs profit_margin_pct, r=-0.99). viz smart auto-adds this as a drill-down beside the correlation heatmap.
qsv viz scatter sales_sample.csv --x discount_pct --y profit_margin_pct \
-  -o scatter_correlated_pair.html
+ -o scatter_correlated_pair.html
contour2D density of units sold vs revenue (binned into a 20x20 grid). viz smart uses this instead of the pair scatter for large datasets, where a scatter would overplot.
qsv viz contour sales_sample.csv --x units_sold --y revenue --bins 20 \
-  -o contour.html
+ -o contour.html
heatmap (pivot)Region x category grid of revenue.
qsv viz heatmap sales_sample.csv --x region --y product_category \
-  --z revenue -o heatmap_pivot.html
+ --z revenue -o heatmap_pivot.html
candlestickOHLC price action.
qsv viz candlestick stock_prices.csv --x date --ohlc-open open \
-  --high high --low low --close close -o candlestick.html
+ --high high --low low --close close -o candlestick.html
ohlcOpen-high-low-close bars.
qsv viz ohlc stock_prices.csv --x date --ohlc-open open --high high \
-  --low low --close close -o ohlc.html
+ --low low --close close -o ohlc.html
radarMulti-axis brand comparison (per-axis mean per series).
qsv viz radar product_ratings.csv --cols battery,camera,performance,display,value,design \
-  --series brand -o radar.html
+ --series brand -o radar.html
sankeyWeb session funnel (duplicate edges aggregated).
qsv viz sankey web_flows.csv --source source --target target \
-  --value sessions -o sankey.html
+ --value sessions -o sankey.html
treemapPart-to-whole spend by plan then region, sized by summed monthly_spend. Rounded tiles + white separators come from the treemap-specific marker; non-numeric/negative measure cells are rejected so proportions can't silently misstate.
qsv viz treemap customer_spend.csv --cols plan,region --value monthly_spend \
-  --agg sum -o treemap.html
+ --agg sum -o treemap.html
sunburstThree-level hierarchy (region -> product_category -> payment_method) as concentric rings, sized by row count; inner rings are parents, outer rings their children. Opens at two rings (maxdepth) so labels stay legible instead of crowding a ~100-sector outer ring; click a sector to drill in and the deeper ring's labels grow back. Hover always shows value + percent.
qsv viz sunburst sales_sample.csv --cols region,product_category,payment_method \
-  -o sunburst.html
+ -o sunburst.html
icicleSame three-level hierarchy (region -> product_category -> payment_method) as a rectangular icicle: parents on the left, children fanning right, each rectangle sized by row count. The flat left-to-right layout keeps deep labels readable where a sunburst's outer ring would crowd; click a rectangle to zoom into that branch. Hover shows label + value + percent of parent.
qsv viz icicle sales_sample.csv --cols region,product_category,payment_method \
-  -o icicle.html
+ -o icicle.html
splomScatter-plot matrix of four numeric columns (units_sold, revenue, discount_pct, profit_margin_pct): every pairwise scatter in an N x N grid with shared axes, so correlation structure is legible at a glance. This is a standalone chart command — the viz smart dashboard has no SPLOM panel; it covers the same ground with its correlation heatmap plus the strongest-pair scatter and 3D scatter drill-downs.
qsv viz splom sales_sample.csv --cols units_sold,revenue,discount_pct,profit_margin_pct \
-  -o splom.html
+ -o splom.html
parcatsParallel-categories flow over three categorical columns (region -> product_category -> payment_method): each ribbon is a category combination, sized by how many rows share it, so co-occurrence between the dimensions is visible without implying a part-to-whole nesting. Ribbons are colored by their first-axis category and bundled (like a Sankey), and each axis opens ordered by frequency with a ⇅ category order toggle that flips every axis between frequency and alphabetical order. viz smart auto-adds this panel for 3-4 associated many-to-many categoricals (and suppresses the hierarchy on the same columns); genuine rollup trees still auto-select a treemap/sunburst.
qsv viz parcats sales_sample.csv --cols region,product_category,payment_method \
-  -o parcats.html
+ -o parcats.html
mapEarthquake points on token-free OpenStreetMap tiles; marker color = magnitude, size = depth.
qsv viz map quakes.csv --lat lat --lon lon --color magnitude \
-  --size depth_km -o map.html
+ --size depth_km -o map.html
map (density)DensityMap heatmap of the same points on a light Carto basemap, weighted by magnitude (via --color) so stronger quakes glow hotter — hovering a point shows its magnitude, not just the coordinates.
qsv viz map quakes.csv --lat lat --lon lon --density --color magnitude \
-  --style carto-positron -o map_density.html
+ --style carto-positron -o map_density.html
geoSame earthquakes on an offline natural-earth projection (no tiles, no token); marker color = magnitude. viz smart auto-uses this projection for global-extent coordinates.
qsv viz geo quakes.csv --lat lat --lon lon --color magnitude \
-  --projection natural-earth -o geo.html
+ --projection natural-earth -o geo.html
geo (animated slider)An animated point map: --slider steps through the region column, revealing the world's seismicity continent by continent, each region in its own color via --series. With --slider-cumulative the points accumulate as the animation plays (Play/Pause + a scrub slider). Unlike the MapLibre tile map, scattergeo animates natively, so --slider is supported on viz geo — use it instead of viz map for animated point maps.
qsv viz geo quakes.csv --lat lat --lon lon --slider region --series region \
-  --slider-cumulative --projection natural-earth -o geo_animated_slider.html
+ --slider-cumulative --projection natural-earth -o geo_animated_slider.html
choroplethFilled-region map coloring countries by GDP, matched by ISO-3 code on a token-free projection basemap. Use --location-mode usa-states / country-names / geojson-id for other region keys, --map for a MapLibre tile basemap, or --geocode to derive codes from lat/lon or place names.
qsv viz choropleth country_stats.csv --locations iso3 --value gdp_usd_tn \
-  --color-scale viridis -o choropleth.html
+ --color-scale viridis -o choropleth.html
choropleth (US states)Same chart, --location-mode usa-states: state codes matched to Plotly's built-in US-state geometry on the token-free albers-usa projection (CONUS + Alaska/Hawaii insets) — no GeoJSON needed. States are colored by renewable-electricity share.
qsv viz choropleth us_state_stats.csv --locations state --value renewable_electricity_pct \
-  --location-mode usa-states -o choropleth_us_states.html
+ --location-mode usa-states -o choropleth_us_states.html
choropleth (MapLibre + GeoJSON)--map draws the filled regions on an interactive MapLibre tile basemap (token-free carto-positron) instead of a projection. The regions come from a custom GeoJSON (--geojson local file or URL) matched to the data by --feature-id-key — here the near-rectangular western states, colored by installed wind capacity. The view auto-centers and zooms to the GeoJSON extent (shown full-width so the computed zoom frames the regions as the CLI does — a tile map's zoom is fixed, so a narrow grid cell would crop it).
qsv viz choropleth western_states.csv --locations state --value wind_capacity_gw \
   --geojson western_states.geojson --feature-id-key id --map --style carto-positron \
-  -o choropleth_maplibre_geojson.html
+ -o choropleth_maplibre_geojson.html
smart dashboard (time-series)Auto dashboard for stock_prices: a time-series trend panel (the first numeric column over the date) leads; the strongest-correlated pair drill-down (open vs close) is shown as a static scatter — that relationship is a near-perfect line whose 2-D shape doesn't evolve, so the judicious animation gate withholds the (uninformative) animated version — alongside box-plot summaries of the OHLC columns.
qsv viz smart stock_prices.csv --max-charts 8 -o smart_dashboard_time_series.html
smart dashboard (per-US-state choropleth)`viz smart` reverse-geocodes each point; because every city resolves to a US state, it adds a per-US-state choropleth (cities-per-state, albers-usa) beside the point map, alongside the usual box plots, frequency bars and the strongest-pair scatter. (The point map's spatial extent caption counts the data's bounding-box corners, which spill into neighboring countries and ocean — the choropleth instead resolves each city to its own state.) No flags, no LLM — the state fill is derived purely from the lat/lon columns.
qsv viz smart us_cities.csv -o smart_dashboard_per_us_state_choropleth.html
smart dashboard (--dictionary infer, treemap)Auto dashboard for customer_spend with a describegpt-inferred Data Dictionary (--dictionary infer) guiding panel selection & field labels. Two categorical dimensions (plan, region) form a shallow part-to-whole hierarchy, auto-rendered as a TREEMAP (area = size). Requires a local LLM; the committed HTML is reused on regen.
qsv viz smart customer_spend.csv --dictionary infer -o smart_dashboard_dictionary_infer_treemap.html
@@ -126,9 +128,8 @@

qsv viz gallery (An Interactive Viz is worth a Million Rows)

--feature-id-key properties.nameBoston 311 (2025) smart visual data dictionary screenshot diff --git a/examples/viz/gen_gallery.py b/examples/viz/gen_gallery.py index 4011c12c1b..ef5b02f814 100755 --- a/examples/viz/gen_gallery.py +++ b/examples/viz/gen_gallery.py @@ -22,7 +22,10 @@ Set QSV_BIN to point at a specific binary; otherwise target/{debug,release}/qsv or a `qsv` on PATH is used. Re-run and commit gallery.html if the diff is what -you expect. The per-figure commands below are mirrored in README.md. +you expect. README.md documents `viz` usage in tutorial form; its commands are +# illustrative and deliberately NOT a verbatim mirror of the gallery's argv (the gallery +# pins things like --max-charts that would only distract a reader). What README.md IS +# required to keep in step with this file is checked by `check_readme_claims` below. """ import json import os @@ -57,6 +60,7 @@ "smart dashboard (animated geo, world events)": "smart_world_events.html", "smart dashboard (Gapminder bubble, regions growth)": "smart_regions_growth.html", "smart dashboard (--smarter, Gini/Lorenz inequality + log-skew boxes)": "smart_cms_medicare.html", + "smart dashboard (dictionary-declared pipeline funnel)": "smart_onboarding_funnel.html", "smart dashboard (--smarter, zero-inflated capital pipeline)": "smart_cpdb.html", } @@ -477,6 +481,21 @@ "against its largest values, so viz smart draws it on a log axis instead, " "keeping the median and quartiles legible.", True, ["smart", "cms_medicare_providers.csv", "--smarter"]), + ("smart dashboard (dictionary-declared pipeline funnel)", + "A synthetic product-onboarding funnel — visits → signups → activated " + "→ subscribed, one row per channel per week. The four stage columns are declared as a " + "pipeline in onboarding_funnel_dict.schema.json " + "(x-qsv.relationships, kind: \"pipeline\"); which columns are " + "stages, and in which direction, is semantics rather than a statistic, so the panel is only " + "ever drawn from an explicit declaration — never guessed from column names.

" + "Here the stages genuinely nest: every row is a strict subset of the one before it, " + "and so are the totals. That is what earns a funnel, whose band widths are a " + "containment claim — plotly computes the stage-to-stage conversion from the bar values " + "themselves (100% → 41% → 54% → 46%), so the percentages can " + "never drift from the bands. Compare the next figure, where the same kind of declaration " + "over non-nesting totals is drawn as a bridge instead.", + True, ["smart", "onboarding_funnel.csv", + "--dictionary", "onboarding_funnel_dict.schema.json"]), ("smart dashboard (--smarter, zero-inflated capital pipeline)", "NYC Capital Projects Database (12,587 projects, sourced from Checkbook NYC), the " "dataset this dashboard's inequality work was designed against. Its money columns are both " @@ -486,17 +505,22 @@ "committed or spent yet, a funding-pipeline stage rather than a have-not population. " "Every Lorenz panel also carries the unit caveat, because a Gini across units as unlike a " "subway extension and a playground resurfacing is close to tautological and must not be read " - "as inequity.

It also carries the pipeline funnel, declared by " + "as inequity.

It also carries the pipeline bridge, declared by " "nyc_capital_projects_dict.schema.json as an x-qsv.relationships " "entry of kind: \"pipeline\". Which columns are stages, and in which direction, " - "is semantics rather than a statistic, so a funnel is only ever drawn from an explicit " - "declaration — never guessed from column names.

Read the subtitle: these stages " - "do not nest. Spent totals 2.9× committed, because the three are independent " + "is semantics rather than a statistic, so the panel is only ever drawn from an explicit " + "declaration — never guessed from column names.

The declaration says these " + "columns are one pipeline; it cannot say that they nest, and that is measured. Here " + "they do not: spent totals 2.9× committed, because the three are independent " "aggregates on different bases — totalplannedcommit is allocated in the " - "Capital Commitment Plan, while the other two are sums within the City's budget. Rather than " - "refuse the panel, the funnel draws and names the violation: " - "Spent exceeds Committed in 46% of rows. Declared order is never re-sorted by " - "size, so the overrun stays visible instead of being quietly tidied away.", + "Capital Commitment Plan, while the other two are sums within the City's budget. A funnel's " + "band widths are a containment claim, so drawing one would render a band wider than " + "the one above it — an hourglass asserting the opposite of the numbers, which no " + "caption can retract. So viz smart keeps the declaration and changes the " + "form: it bridges the signed gap between consecutive totals, naming each step as the " + "arithmetic difference it is (Committed − Planned), never as a flow. " + "Declared order is still never re-sorted by size, and the subtitle says why the form " + "changed: stages do not nest — bridged, not funnelled.", True, ["smart", "nyc_capital_projects.csv", "--smarter", "--dictionary", "nyc_capital_projects_dict.schema.json"]), ("bar", "Revenue by region (aggregated sum).", @@ -1130,14 +1154,104 @@ def cleanup_sidecars(): # writes a `*.stats.bivariate.csv` sidecar (a separate moarstats output, so it doesn't # match the `.stats.csv` substring check below). Don't leave any of these in the tree # (the committed datasets ship without them). + # + # Called at BOTH ends of main(). Sweeping afterwards keeps the tree clean; sweeping first is + # what keeps the gallery reproducible, because `viz smart` REUSES an existing stats cache + # rather than recomputing it. A cache left behind by unrelated manual work -- notably a + # `--smarter` run, whose moarstats enrichment adds outlier annotations to panel titles -- is + # otherwise silently inherited by a figure whose own command never asked for it. That + # produced a committed dashboard whose caption did not match the command printed beside it. + removed = 0 for f in os.listdir(VIZ_DIR): if (".stats.csv" in f or ".stats.jsonl" in f or ".stats.bivariate.csv" in f or f.endswith(".idx")): os.unlink(os.path.join(VIZ_DIR, f)) + removed += 1 + return removed + + +# Counting words as README.md writes them. Only the range the gallery can plausibly reach. +_NUM_WORDS = { + 1: "one", 2: "two", 3: "three", 4: "four", 5: "five", 6: "six", 7: "seven", + 8: "eight", 9: "nine", 10: "ten", 11: "eleven", 12: "twelve", 13: "thirteen", + 14: "fourteen", 15: "fifteen", 16: "sixteen", 17: "seventeen", 18: "eighteen", + 19: "nineteen", 20: "twenty", 21: "twenty-one", 22: "twenty-two", + 23: "twenty-three", 24: "twenty-four", +} + + +def check_readme_claims(): + """Fail closed on the README drift this script can actually prove. + + Adding a figure silently invalidates prose counts and leaves its input undocumented; that + has happened repeatedly, most recently a dashboard count that was corrected to the number of + `smart_*.html` files on disk when the sentence is about EMBEDDED iframes -- two of those files + are screenshot link-outs, so the "fix" was wrong in the other direction. + + Only claims with a mechanical source of truth are checked. Command text is deliberately NOT + compared: README.md is a tutorial and its invocations differ from the gallery's argv on + purpose. Runs AFTER the gallery is written, so a new figure still regenerates -- the exit + code is what tells you the docs need a follow-up. + """ + readme_path = os.path.join(VIZ_DIR, "README.md") + if not os.path.exists(readme_path): + return + with open(readme_path, encoding="utf-8") as fh: + readme = fh.read() + + problems = [] + + def expect_phrase(count, phrase_fmt, what): + word = _NUM_WORDS.get(count, str(count)) + if phrase_fmt.format(n=word) not in readme: + problems.append( + f"{what}: expected README.md to say " + f"{phrase_fmt.format(n=word)!r} (there are {count})" + ) + + expect_phrase( + len(SMART_IFRAME), + "The {n} **smart dashboards** are embedded", + "embedded-dashboard count", + ) + expect_phrase( + len(SCREENSHOTS), + "closes with {n} clickable **screenshot link-outs**", + "screenshot link-out count", + ) + + # every input actually fed to qsv should be findable in the README's tables. That includes + # the screenshot link-outs built with `args`: those run qsv exactly like a figure does. The + # `cmd`-only entries are display strings for datasets too large to commit, so their inputs + # are deliberately not required here. + fed_to_qsv = [fig[3] for fig in FIGURES] + fed_to_qsv += [shot["args"] for shot in SCREENSHOTS if "args" in shot] + inputs = { + arg + for argv in fed_to_qsv + for arg in argv + if arg.endswith((".csv", ".tsv", ".schema.json", ".geojson")) + } + for asset in sorted(inputs): + if f"`{asset}`" not in readme: + problems.append(f"undocumented gallery input: `{asset}` is not mentioned in README.md") + + if problems: + raise SystemExit( + "gallery written, but README.md is now out of step with gen_gallery.py:\n " + + "\n ".join(problems) + ) def main(): qsv = find_qsv() + # before anything runs qsv: a stale cache from unrelated work would otherwise be reused and + # silently change a figure (see `cleanup_sidecars`). + stale = cleanup_sidecars() + if stale: + sys.stderr.write( + f"swept {stale} pre-existing qsv sidecar(s) so figures regenerate from source\n" + ) # QSV_VIZ_REGEN_LLM opts into regenerating the `--dictionary infer` dashboards live (needs a # local LLM up); otherwise their committed HTML is reused so a normal run stays LLM-free. # Only an explicit truthy value enables it, so QSV_VIZ_REGEN_LLM=0/false/off (or empty) stays @@ -1389,6 +1503,8 @@ def _emitted_cdn_tag(): cleanup_sidecars() sys.stderr.write(f"wrote {GALLERY} ({len(body)} bytes, {len(figs)} figures)\n") + check_readme_claims() + if __name__ == "__main__": main() diff --git a/examples/viz/onboarding_funnel.csv b/examples/viz/onboarding_funnel.csv new file mode 100644 index 0000000000..6cd551730c --- /dev/null +++ b/examples/viz/onboarding_funnel.csv @@ -0,0 +1,61 @@ +channel,week,visits,signups,activated,subscribed +Organic Search,01,11629,5336,2592,1205 +Paid Social,01,6350,1758,722,208 +Email,01,3581,2130,1592,868 +Referral,01,2878,1281,746,343 +Partner,01,1695,625,278,105 +Organic Search,02,11397,4933,2462,1042 +Paid Social,02,6285,1739,658,173 +Email,02,4183,2594,1721,996 +Referral,02,2987,1585,891,405 +Partner,02,1843,679,306,124 +Organic Search,03,11933,5398,2913,1434 +Paid Social,03,8006,2238,823,263 +Email,03,3659,1952,1429,795 +Referral,03,2805,1225,806,387 +Partner,03,1996,731,331,138 +Organic Search,04,10607,4341,2612,1195 +Paid Social,04,6609,1686,624,200 +Email,04,3653,2088,1547,851 +Referral,04,3031,1542,1029,504 +Partner,04,1811,696,319,113 +Organic Search,05,10763,4053,2221,991 +Paid Social,05,6729,1795,693,197 +Email,05,4906,3098,1885,992 +Referral,05,2362,1142,696,343 +Partner,05,1958,734,353,125 +Organic Search,06,11256,4952,2658,1185 +Paid Social,06,7248,1976,816,237 +Email,06,4661,2666,1678,970 +Referral,06,3081,1509,985,479 +Partner,06,1963,716,314,107 +Organic Search,07,9835,3827,1973,940 +Paid Social,07,6600,1855,677,200 +Email,07,3709,2131,1312,682 +Referral,07,2151,1154,727,376 +Partner,07,2117,782,343,140 +Organic Search,08,7851,3539,1982,864 +Paid Social,08,8795,2549,968,306 +Email,08,3296,1861,1117,600 +Referral,08,2184,1011,614,348 +Partner,08,1941,606,312,126 +Organic Search,09,10318,4199,2056,867 +Paid Social,09,8180,1995,810,227 +Email,09,3582,2058,1229,611 +Referral,09,2103,1143,618,312 +Partner,09,2235,779,376,155 +Organic Search,10,10070,4262,2172,847 +Paid Social,10,6533,1894,638,185 +Email,10,4443,2816,1706,881 +Referral,10,2223,1035,650,299 +Partner,10,1976,653,329,113 +Organic Search,11,11134,4821,2354,1067 +Paid Social,11,8645,2324,816,253 +Email,11,4177,2526,1678,1016 +Referral,11,2324,1150,780,350 +Partner,11,1712,577,250,92 +Organic Search,12,10577,3831,2337,987 +Paid Social,12,6559,1778,749,209 +Email,12,4968,3121,2086,1047 +Referral,12,2577,1206,747,425 +Partner,12,1898,599,252,87 diff --git a/examples/viz/onboarding_funnel_dict.schema.json b/examples/viz/onboarding_funnel_dict.schema.json new file mode 100644 index 0000000000..a278a7562a --- /dev/null +++ b/examples/viz/onboarding_funnel_dict.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Data Dictionary for onboarding_funnel.csv", + "description": "A synthetic product-onboarding funnel: one row per acquisition channel per week, with the four stage counts a visitor passes through. Each stage is a strict subset of the one before it, in every row.", + "type": "object", + "properties": { + "channel": { + "type": "string", + "title": "Channel", + "description": "Acquisition channel the visitor arrived from.", + "x-qsv": { + "qsv_type": "String", + "role": "dimension", + "concept": "category.type" + } + }, + "week": { + "type": "string", + "title": "Week", + "description": "Reporting week, zero-padded within the year.", + "x-qsv": { + "qsv_type": "String", + "role": "dimension", + "concept": "time.period" + } + }, + "visits": { + "type": "integer", + "title": "Visits", + "description": "Sessions that reached the product site.", + "x-qsv": { + "qsv_type": "Integer", + "role": "measure", + "concept": "measure.count" + } + }, + "signups": { + "type": "integer", + "title": "Signups", + "description": "Visits that created an account.", + "x-qsv": { + "qsv_type": "Integer", + "role": "measure", + "concept": "measure.count" + } + }, + "activated": { + "type": "integer", + "title": "Activated", + "description": "Signups that completed onboarding.", + "x-qsv": { + "qsv_type": "Integer", + "role": "measure", + "concept": "measure.count" + } + }, + "subscribed": { + "type": "integer", + "title": "Subscribed", + "description": "Activated users that started a paid plan.", + "x-qsv": { + "qsv_type": "Integer", + "role": "measure", + "concept": "measure.count" + } + } + }, + "x-qsv": { + "grain": "one row = one channel-week", + "relationships": [ + { + "kind": "pipeline", + "members": [ + "visits", + "signups", + "activated", + "subscribed" + ] + } + ] + } +} diff --git a/examples/viz/smart_allegheny_dogs.html b/examples/viz/smart_allegheny_dogs.html index 31a6e8573e..13c34cdb1e 100644 --- a/examples/viz/smart_allegheny_dogs.html +++ b/examples/viz/smart_allegheny_dogs.html @@ -49,7 +49,7 @@

allegheny_dog_licenses.csv — data overview

Rows:50,013 Columns:8 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -72,7 +72,7 @@

allegheny_dog_licenses.csv — data overview

@@ -80,7 +80,7 @@

allegheny_dog_licenses.csv — data overview

@@ -88,7 +88,7 @@

allegheny_dog_licenses.csv — data overview

@@ -96,7 +96,7 @@

allegheny_dog_licenses.csv — data overview

@@ -104,7 +104,7 @@

allegheny_dog_licenses.csv — data overview

@@ -112,7 +112,7 @@

allegheny_dog_licenses.csv — data overview

@@ -652,7 +652,7 @@

Data Dictionary — allegheny_dog_licenses.csv — data overview

⤓ Export JSONSchemaschema ⤓ allegheny_dog_licenses.stats.csv.data.jsonlstats -⤓ allegheny_dog_licenses.stats.csv.jsonstats meta +⤓ allegheny_dog_licenses.stats.csv.jsonstats meta ⤓ allegheny_dog_licenses.viz-frequency.csvcharted freq ⤓ allegheny_dog_licenses.stats.bivariate.csvbivariate
@@ -815,7 +815,7 @@

ValidDate

function qsvOpenDictTab(anchor) { var src = document.getElementById("qsv-dict-src"); if (!src) return false; - var w = window.open("", "qsv_dict_57da7b37"); + var w = window.open("", "qsv_dict_77a37454"); if (!w) return false; var needsWrite = true; // reading a foreign document (user navigated the tab elsewhere) throws: rewrite in that case. diff --git a/examples/viz/smart_cms_medicare.html b/examples/viz/smart_cms_medicare.html index d40c9160db..387ae519f7 100644 --- a/examples/viz/smart_cms_medicare.html +++ b/examples/viz/smart_cms_medicare.html @@ -46,13 +46,13 @@ Rows:30,007 Columns:4 Completeness:97.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
diff --git a/examples/viz/smart_cpdb.html b/examples/viz/smart_cpdb.html index 3593f2fba5..b43ca6167b 100644 --- a/examples/viz/smart_cpdb.html +++ b/examples/viz/smart_cpdb.html @@ -48,7 +48,7 @@

nyc_capital_projects.csv — data overview

Rows:12,587 Columns:7 Completeness:98.8% -Compiled:2026-07-26 03:38 UTC +Compiled:2026-07-26 15:26 UTC
@@ -63,7 +63,7 @@

nyc_capital_projects.csv — data overview

@@ -79,7 +79,7 @@

nyc_capital_projects.csv — data overview

@@ -87,7 +87,7 @@

nyc_capital_projects.csv — data overview

@@ -95,7 +95,7 @@

nyc_capital_projects.csv — data overview

@@ -103,7 +103,7 @@

nyc_capital_projects.csv — data overview

@@ -111,7 +111,7 @@

nyc_capital_projects.csv — data overview

@@ -119,7 +119,7 @@

nyc_capital_projects.csv — data overview

@@ -127,7 +127,7 @@

nyc_capital_projects.csv — data overview

@@ -135,7 +135,7 @@

nyc_capital_projects.csv — data overview

@@ -143,7 +143,7 @@

nyc_capital_projects.csv — data overview

@@ -151,7 +151,7 @@

nyc_capital_projects.csv — data overview

@@ -159,7 +159,7 @@

nyc_capital_projects.csv — data overview

diff --git a/examples/viz/smart_geo_outliers.html b/examples/viz/smart_geo_outliers.html index 109ad52097..531d1fb372 100644 --- a/examples/viz/smart_geo_outliers.html +++ b/examples/viz/smart_geo_outliers.html @@ -48,7 +48,7 @@

delivery_stops.csv — data overview

Rows:200 Columns:10 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -80,7 +80,7 @@

delivery_stops.csv — data overview

@@ -88,7 +88,7 @@

delivery_stops.csv — data overview

@@ -96,7 +96,7 @@

delivery_stops.csv — data overview

@@ -104,7 +104,7 @@

delivery_stops.csv — data overview

@@ -112,7 +112,7 @@

delivery_stops.csv — data overview

@@ -120,7 +120,7 @@

delivery_stops.csv — data overview

@@ -128,7 +128,7 @@

delivery_stops.csv — data overview

@@ -136,7 +136,7 @@

delivery_stops.csv — data overview

@@ -144,7 +144,7 @@

delivery_stops.csv — data overview

@@ -152,7 +152,7 @@

delivery_stops.csv — data overview

diff --git a/examples/viz/smart_nyc311.html b/examples/viz/smart_nyc311.html index 90e0fba101..77ddbce8f2 100644 --- a/examples/viz/smart_nyc311.html +++ b/examples/viz/smart_nyc311.html @@ -49,7 +49,7 @@

nyc_311.csv — data overview

Rows:10,000 Columns:41 Completeness:68.4% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -81,7 +81,7 @@

nyc_311.csv — data overview

@@ -105,7 +105,7 @@

nyc_311.csv — data overview

@@ -121,7 +121,7 @@

nyc_311.csv — data overview

@@ -129,7 +129,7 @@

nyc_311.csv — data overview

@@ -137,7 +137,7 @@

nyc_311.csv — data overview

@@ -145,7 +145,7 @@

nyc_311.csv — data overview

@@ -153,7 +153,7 @@

nyc_311.csv — data overview

@@ -161,7 +161,7 @@

nyc_311.csv — data overview

@@ -169,7 +169,7 @@

nyc_311.csv — data overview

@@ -177,7 +177,7 @@

nyc_311.csv — data overview

@@ -185,7 +185,7 @@

nyc_311.csv — data overview

@@ -193,7 +193,7 @@

nyc_311.csv — data overview

@@ -201,7 +201,7 @@

nyc_311.csv — data overview

@@ -209,7 +209,7 @@

nyc_311.csv — data overview

@@ -217,7 +217,7 @@

nyc_311.csv — data overview

@@ -225,7 +225,7 @@

nyc_311.csv — data overview

@@ -233,7 +233,7 @@

nyc_311.csv — data overview

@@ -241,7 +241,7 @@

nyc_311.csv — data overview

@@ -249,7 +249,7 @@

nyc_311.csv — data overview

@@ -257,7 +257,7 @@

nyc_311.csv — data overview

@@ -265,7 +265,7 @@

nyc_311.csv — data overview

@@ -273,7 +273,7 @@

nyc_311.csv — data overview

@@ -281,7 +281,7 @@

nyc_311.csv — data overview

@@ -289,7 +289,7 @@

nyc_311.csv — data overview

@@ -297,7 +297,7 @@

nyc_311.csv — data overview

@@ -305,7 +305,7 @@

nyc_311.csv — data overview

@@ -313,7 +313,7 @@

nyc_311.csv — data overview

@@ -321,7 +321,7 @@

nyc_311.csv — data overview

@@ -329,7 +329,7 @@

nyc_311.csv — data overview

@@ -337,7 +337,7 @@

nyc_311.csv — data overview

@@ -345,7 +345,7 @@

nyc_311.csv — data overview

@@ -353,7 +353,7 @@

nyc_311.csv — data overview

@@ -361,7 +361,7 @@

nyc_311.csv — data overview

@@ -901,7 +901,7 @@

Data Dictionary — nyc_311.csv — data overview

⤓ Export JSONSchemaschema ⤓ nyc_311.stats.csv.data.jsonlstats -⤓ nyc_311.stats.csv.jsonstats meta +⤓ nyc_311.stats.csv.jsonstats meta ⤓ nyc_311.viz-frequency.csvcharted freq ⤓ nyc_311.stats.bivariate.csvbivariate
@@ -1605,7 +1605,7 @@

Location

function qsvOpenDictTab(anchor) { var src = document.getElementById("qsv-dict-src"); if (!src) return false; - var w = window.open("", "qsv_dict_81c75350"); + var w = window.open("", "qsv_dict_bef5b877"); if (!w) return false; var needsWrite = true; // reading a foreign document (user navigated the tab elsewhere) throws: rewrite in that case. diff --git a/examples/viz/smart_onboarding_funnel.html b/examples/viz/smart_onboarding_funnel.html new file mode 100644 index 0000000000..e5256775c3 --- /dev/null +++ b/examples/viz/smart_onboarding_funnel.html @@ -0,0 +1,658 @@ + + + + + +onboarding_funnel.csv — data overview + + + + + + + + + + + + + +
Rows:60
Columns:6
Completeness:100.0%
Compiled:2026-07-26 15:58 UTC
+
+
+
+ +
+
+ + + + + + + + diff --git a/examples/viz/smart_regions_growth.html b/examples/viz/smart_regions_growth.html index 021c73382c..9171b88ecc 100644 --- a/examples/viz/smart_regions_growth.html +++ b/examples/viz/smart_regions_growth.html @@ -48,7 +48,7 @@

regions_growth.csv — data overview

Rows:844 Columns:5 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -71,7 +71,7 @@

regions_growth.csv — data overview

@@ -79,7 +79,7 @@

regions_growth.csv — data overview

@@ -87,7 +87,7 @@

regions_growth.csv — data overview

@@ -103,7 +103,7 @@

regions_growth.csv — data overview

@@ -111,7 +111,7 @@

regions_growth.csv — data overview

@@ -127,7 +127,7 @@

regions_growth.csv — data overview

@@ -135,7 +135,7 @@

regions_growth.csv — data overview

@@ -143,7 +143,7 @@

regions_growth.csv — data overview

@@ -151,7 +151,7 @@

regions_growth.csv — data overview

@@ -159,7 +159,7 @@

regions_growth.csv — data overview

diff --git a/examples/viz/smart_sales.html b/examples/viz/smart_sales.html index 19b36a0a57..18e6512013 100644 --- a/examples/viz/smart_sales.html +++ b/examples/viz/smart_sales.html @@ -48,7 +48,7 @@

sales_sample.csv — data overview

Rows:500 Columns:13 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -71,7 +71,7 @@

sales_sample.csv — data overview

@@ -79,7 +79,7 @@

sales_sample.csv — data overview

@@ -87,7 +87,7 @@

sales_sample.csv — data overview

@@ -95,7 +95,7 @@

sales_sample.csv — data overview

@@ -103,7 +103,7 @@

sales_sample.csv — data overview

@@ -111,7 +111,7 @@

sales_sample.csv — data overview

@@ -119,7 +119,7 @@

sales_sample.csv — data overview

diff --git a/examples/viz/smart_sales_kpi.html b/examples/viz/smart_sales_kpi.html index 1075f62b9d..85060862de 100644 --- a/examples/viz/smart_sales_kpi.html +++ b/examples/viz/smart_sales_kpi.html @@ -48,7 +48,7 @@

sales_sample.csv — data overview

Rows:500 Columns:13 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -63,7 +63,7 @@

sales_sample.csv — data overview

@@ -71,7 +71,7 @@

sales_sample.csv — data overview

@@ -79,7 +79,7 @@

sales_sample.csv — data overview

@@ -87,7 +87,7 @@

sales_sample.csv — data overview

@@ -95,7 +95,7 @@

sales_sample.csv — data overview

@@ -103,7 +103,7 @@

sales_sample.csv — data overview

@@ -111,7 +111,7 @@

sales_sample.csv — data overview

@@ -119,7 +119,7 @@

sales_sample.csv — data overview

@@ -127,7 +127,7 @@

sales_sample.csv — data overview

@@ -135,7 +135,7 @@

sales_sample.csv — data overview

@@ -143,7 +143,7 @@

sales_sample.csv — data overview

@@ -151,7 +151,7 @@

sales_sample.csv — data overview

@@ -159,7 +159,7 @@

sales_sample.csv — data overview

@@ -167,7 +167,7 @@

sales_sample.csv — data overview

diff --git a/examples/viz/smart_smarter.html b/examples/viz/smart_smarter.html index 78d4901e28..c61a424a61 100644 --- a/examples/viz/smart_smarter.html +++ b/examples/viz/smart_smarter.html @@ -48,7 +48,7 @@

customer_spend.csv — data overview

Rows:300 Columns:5 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -71,7 +71,7 @@

customer_spend.csv — data overview

@@ -79,7 +79,7 @@

customer_spend.csv — data overview

@@ -87,7 +87,7 @@

customer_spend.csv — data overview

@@ -95,7 +95,7 @@

customer_spend.csv — data overview

@@ -103,7 +103,7 @@

customer_spend.csv — data overview

diff --git a/examples/viz/smart_timeseries.html b/examples/viz/smart_timeseries.html index 224f5e49bf..eb1f653b43 100644 --- a/examples/viz/smart_timeseries.html +++ b/examples/viz/smart_timeseries.html @@ -48,7 +48,7 @@

stock_prices.csv — data overview

Rows:205 Columns:6 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -71,7 +71,7 @@

stock_prices.csv — data overview

@@ -79,7 +79,7 @@

stock_prices.csv — data overview

@@ -87,7 +87,7 @@

stock_prices.csv — data overview

@@ -95,7 +95,7 @@

stock_prices.csv — data overview

@@ -103,7 +103,7 @@

stock_prices.csv — data overview

@@ -111,7 +111,7 @@

stock_prices.csv — data overview

@@ -119,7 +119,7 @@

stock_prices.csv — data overview

diff --git a/examples/viz/smart_us_choropleth.html b/examples/viz/smart_us_choropleth.html index 41ac66c02e..fe183e7baa 100644 --- a/examples/viz/smart_us_choropleth.html +++ b/examples/viz/smart_us_choropleth.html @@ -48,7 +48,7 @@

us_cities.csv — data overview

Rows:332 Columns:6 Completeness:100.0% -Compiled:2026-07-25 05:12 UTC +Compiled:2026-07-26 14:39 UTC
@@ -80,7 +80,7 @@

us_cities.csv — data overview

@@ -88,7 +88,7 @@

us_cities.csv — data overview

@@ -96,7 +96,7 @@

us_cities.csv — data overview

@@ -104,7 +104,7 @@

us_cities.csv — data overview

@@ -112,7 +112,7 @@

us_cities.csv — data overview

diff --git a/examples/viz/smart_world_events.html b/examples/viz/smart_world_events.html index ef9a1a216f..62f7f75e47 100644 --- a/examples/viz/smart_world_events.html +++ b/examples/viz/smart_world_events.html @@ -48,7 +48,7 @@

world_events_dated.csv — data overview

Rows:264 Columns:5 Completeness:100.0% -Compiled:2026-07-26 03:25 UTC +Compiled:2026-07-26 14:39 UTC
@@ -87,7 +87,7 @@

world_events_dated.csv — data overview

@@ -103,7 +103,7 @@

world_events_dated.csv — data overview

diff --git a/src/cmd/viz.rs b/src/cmd/viz.rs index 936bdd48bc..07585a3c50 100644 --- a/src/cmd/viz.rs +++ b/src/cmd/viz.rs @@ -94,18 +94,26 @@ auto-picks panels, so no --x/--y is needed: with no headline measure. (Overall dataset completeness - the share of non-empty cells - is a quiet "Completeness:" line in the header metadata table, not a KPI tile.) - - pipeline funnel, when the dictionary DECLARES one (see --dictionary). Which + - pipeline panel, when the dictionary DECLARES one (see --dictionary). Which columns are process stages, and in which direction, is semantics rather than a statistic - no column-name vocabulary settles it and no statistic does - either - so a funnel is drawn ONLY from an explicit declaration, never + either - so the panel is drawn ONLY from an explicit declaration, never guessed. Both encodings are supported: stages held in separate measure columns, and stages held as values of one category column. Costs one extra data pass over the declared stages only. + The declaration fixes WHICH columns and in WHAT order; the numbers decide the + FORM. A funnel's band widths are a containment claim, so one is drawn only + while the stage totals never grow. If any stage outruns the one before it, the + same declaration is drawn as a BRIDGE instead: the signed difference between + consecutive totals, each step labelled as the arithmetic difference it is + rather than as a flow. A funnel there would render a band wider than the one + above it, asserting the opposite of the data. The subtitle says which form was + used and why. Stage order is the declared order and is never re-sorted by size. For the column encoding, row-wise containment (does each stage nest inside the one - before it?) is MEASURED and disclosed in the subtitle, but never refuses the - panel: a pipeline whose stages overrun is a finding to name, not a reason to - show nothing. Totals sum over the rows complete across every declared stage, + before it?) is MEASURED and disclosed in the subtitle - separately from the + form, since rows can overrun while the totals still shrink, or nest while the + totals grow. Totals sum over the rows complete across every declared stage, so they do NOT match `stats.sum`; the subtitle always discloses that denominator. See also the standalone `qsv viz funnel` chart type, which takes its stage order from the file and needs no dictionary. @@ -542,8 +550,9 @@ smart options: A "target" number on a measure renders a "vs target" DELTA against that goal (value minus target) - a GOAL you supply, never a fabricated prior-period baseline, so "infer" never emits it; hand-author it. - The dictionary is also the ONLY source of the pipeline funnel panel, - declared in the dataset-level "x-qsv" object as a "relationships" entry + The dictionary is also the ONLY source of the pipeline panel (drawn as + a funnel, or as a bridge when the stage totals do not nest), declared in + the dataset-level "x-qsv" object as a "relationships" entry with "kind": "pipeline". Two encodings, both hand-editable: stages as COLUMNS - "members" lists the stage columns in process order, WIDEST/UPSTREAM FIRST (note this is the opposite direction @@ -560,7 +569,8 @@ smart options: "value_column":"revenue"} Declared order is authoritative and is never re-sorted by size, so a stage that outruns its predecessor stays visible instead of being - quietly reordered away. A declaration naming a missing column, or a + quietly reordered away - it switches the panel to a bridge rather than + drawing a funnel that widens. A declaration naming a missing column, or a stage that is an average/rate rather than a summable amount, is skipped with a note rather than erroring. Only affects `smart`. @@ -749,14 +759,14 @@ use plotly::{ Bar, BoxPlot, Candlestick, Choropleth, ChoroplethMap, Configuration, Contour, DensityMap, Funnel, HeatMap, Histogram, Icicle, Indicator, Ohlc, Parcats, Pie, Plot, Sankey, Scatter, Scatter3D, ScatterGeo, ScatterMap, ScatterPolar, Splom, Sunburst, Trace, Traces, Treemap, - Violin, + Violin, Waterfall, box_plot::{BoxPoints, QuartileMethod}, choropleth::{LocationMode, Marker as ChoroplethMarker}, color::NamedColor, common::{ - Anchor, ColorBar, ColorScale, ColorScalePalette, Domain, ErrorData, ErrorType, Fill, Font, - HoverInfo, Line, Marker, Mode, Orientation, Pattern, PatternShape, TextPosition, TickMode, - Title, + Anchor, ColorBar, ColorScale, ColorScalePalette, Domain, ErrorData, ErrorType, + ExponentFormat, Fill, Font, HoverInfo, Line, Marker, Mode, Orientation, Pattern, + PatternShape, TextPosition, TickMode, Title, }, funnel::Connector as FunnelConnector, indicator::{Delta, Gauge, GaugeAxis, Mode as IndicatorMode, Number}, @@ -774,6 +784,7 @@ use plotly::{ traces::{icicle::BranchValues as IcicleBranchValues, scatter_map::Cluster}, treemap::{BranchValues, Marker as TreemapMarker, Pad}, violin::{MeanLine, SpanMode, ViolinBox, ViolinPoints}, + waterfall::{Marker as WaterfallMarker, Measure, MeasureStyle}, }; use rayon::prelude::*; use serde::Deserialize; @@ -1429,6 +1440,11 @@ const OTHER_TEXT: &str = "Other"; /// Muted grey for the aggregate `(NULL)` / `Other (N)` frequency bars so they read as summary /// buckets, visually distinct from the palette-colored real categories. const MUTED_COLOR: &str = "#999999"; +/// Bridge delta colours (issue #4222). A bridge's step bars are the only place in the dashboard +/// where a bar's SIGN is the message, so they get their own semantics rather than the panel hue: +/// a shortfall against the previous stage and an overrun beyond it must not look alike. +const BRIDGE_DOWN_COLOR: &str = "#c0504d"; +const BRIDGE_UP_COLOR: &str = "#4f8a5b"; /// Faint grey wash used as the fill of a box/violin distribution panel when it renders on a /// log value axis (see `log_distribution_body_cue`). @@ -12536,6 +12552,7 @@ enum PanelKind { reached: Vec, n_complete: usize, shape: FunnelShape, + form: PipelineForm, }, /// 2D density contour of the most strongly correlated numeric pair — used INSTEAD of /// `ScatterPair` for large datasets (>= `SMART_CONTOUR_MIN_POINTS`), where a scatter overplots. @@ -12968,6 +12985,45 @@ struct DictRow { target: Option, } +/// Which form a declared pipeline is drawn as (issue #4222). +/// +/// The dictionary declares the SEMANTICS — that these columns are one ordered pipeline — and that +/// declaration is always honoured. It does not, and cannot, declare that the stages actually nest: +/// that is a measurable property of the data, and it decides the FORM. +/// +/// A funnel is a containment chart; its band widths ARE the claim that each stage is a subset of +/// the one above. Drawing one for stages that grow renders a band wider than its predecessor — an +/// hourglass that asserts the opposite of what the numbers say, which no subtitle can retract, +/// because the geometry is read before the caption is. +#[derive(Clone, Debug, PartialEq, Eq)] +enum PipelineForm { + /// Stage totals never increase: the containment claim holds, so the funnel is honest. + Funnel, + /// At least one stage total exceeds its predecessor. The panel bridges the signed differences + /// between consecutive totals instead — same declared order, same numbers, but the form + /// asserts arithmetic rather than containment. + Bridge, +} + +impl PipelineForm { + /// Funnel only while the declared totals are monotonically non-increasing. + /// + /// Deliberately NOT the per-row containment share: that measures how many individual rows + /// break the nesting and is reported in the subtitle, whereas this decides what the picture + /// claims. A pipeline can have violating rows that still net out to a shrinking total (a + /// funnel remains truthful), or perfectly nesting rows whose totals grow (it does not). + fn for_totals(totals: &[f64]) -> Self { + if totals.windows(2).all(|w| { + let [a, b] = w else { return true }; + b <= a + }) { + Self::Funnel + } else { + Self::Bridge + } + } +} + /// How a built funnel's numbers should be read — the render arm's hover text depends on it. /// /// Kept as data on the panel rather than re-derived, because `reached`/`n_complete` are @@ -15759,6 +15815,22 @@ fn read_stage_totals( Ok((totals, counts, matched)) } +/// Render a completeness share for the funnel subtitle, never rounding UP to 100%. +/// +/// The clause this feeds exists specifically to warn that the funnel's totals will not reconcile +/// with `stats.sum`. Plain `{:.0}` defeats that: a dataset dropping 21 of 12,587 rows is 99.83% +/// complete, which renders as "100% of rows" — asserting the opposite of the disclosure's whole +/// purpose, while a nine-figure sum quietly goes missing. So anything short of genuinely complete +/// is FLOORED to one decimal; only an exact 1.0 prints a bare "100". +fn complete_pct_str(complete_frac: f64) -> String { + if complete_frac >= 1.0 { + return "100".to_string(); + } + // floor, not round: 99.96% must not become "100.0%" + let tenths = (complete_frac * 1000.0).floor() / 10.0; + format!("{tenths:.1}") +} + /// The caveat line beneath a funnel panel's title (issue #4222). /// /// Clause 1 is UNCONDITIONAL, following `lorenz_caveat`'s philosophy. The funnel's totals are @@ -15776,11 +15848,12 @@ fn funnel_subtitle( violations: &[f64], n_complete: usize, complete_frac: f64, + form: &PipelineForm, ) -> Option { let mut parts: Vec = vec![format!( - "n = {} complete cases ({:.0}% of rows)", + "n = {} complete cases ({}% of rows)", HumanCount(n_complete as u64), - complete_frac * 100.0 + complete_pct_str(complete_frac) )]; // worst per-row violation, if any is worth naming @@ -15798,6 +15871,12 @@ fn funnel_subtitle( )); } + // On a bridge, name the form choice: a reader who expected the declared "pipeline" to be a + // funnel is owed the reason it is not one, in the same line that reports the violation. + if matches!(form, PipelineForm::Bridge) { + parts.push("stages do not nest \u{2014} bridged, not funnelled".to_string()); + } + // a stage whose TOTAL outruns its predecessor: the bar order is kept (vocabulary order is // never sorted away), so the inverted band is the finding and needs naming. if let Some(k) = (1..totals.len()).find(|&k| totals[k] > totals[k - 1]) @@ -21830,8 +21909,8 @@ impl<'a> SmartCtx<'a> { ) -> Option { if totals.iter().any(|t| *t < 0.0) { viz_note( - "viz smart: pipeline funnel skipped \u{2014} a stage total is negative, which a \ - funnel cannot represent", + "viz smart: pipeline panel skipped \u{2014} a stage total is negative, which \ + neither a funnel nor a bridge can represent", ); return None; } @@ -21845,13 +21924,27 @@ impl<'a> SmartCtx<'a> { ); return None; } - let subtitle = funnel_subtitle(&stages, &totals, violations, n_complete, complete_frac); + // The declaration says these columns are one pipeline; the NUMBERS decide whether a + // containment form can honestly represent it. See `PipelineForm`. + let form = PipelineForm::for_totals(&totals); + let subtitle = funnel_subtitle( + &stages, + &totals, + violations, + n_complete, + complete_frac, + &form, + ); + let noun = match form { + PipelineForm::Funnel => "funnel", + PipelineForm::Bridge => "bridge", + }; let title = match &shape { FunnelShape::Columns => { - format!("Pipeline funnel: {}", labels.join(" \u{2192} ")) + format!("Pipeline {noun}: {}", labels.join(" \u{2192} ")) }, _ => format!( - "Pipeline funnel: {} ({})", + "Pipeline {noun}: {} ({})", labels.first().cloned().unwrap_or_default(), stages.join(" \u{2192} ") ), @@ -21866,6 +21959,7 @@ impl<'a> SmartCtx<'a> { reached, n_complete, shape, + form, }, ) .with_subtitle(subtitle), @@ -23156,6 +23250,146 @@ fn panel_trace( reached, n_complete, shape, + form: PipelineForm::Bridge, + } => { + // The declared stages do not nest, so the panel bridges the signed differences + // between consecutive totals instead of asserting containment (see `PipelineForm`). + // + // Drawn VERTICALLY, unlike the funnel. A waterfall is an ordinary bar-like trace, so + // category index 0 lands at the axis BOTTOM — the opposite of a funnel — and the + // arrays cannot simply be reversed to compensate, because a waterfall accumulates in + // array order and index 0 must be the `Absolute` that seeds the running total. + // Vertical sidesteps that entirely, and is the conventional orientation for a bridge. + // + // Each step is `stage[k] - stage[k-1]`, labelled as exactly that: an ARITHMETIC + // difference between two independent totals, never "converted" or "lost", because + // the whole reason this is not a funnel is that no flow between them is claimed. + let mut cats: Vec = Vec::with_capacity(stages.len() * 2 - 1); + let mut vals: Vec = Vec::with_capacity(stages.len() * 2 - 1); + let mut measures: Vec = Vec::with_capacity(stages.len() * 2 - 1); + let mut hover: Vec = Vec::with_capacity(stages.len() * 2 - 1); + + // A stage bar's hover, worded per `shape` exactly as the funnel arm words it: the + // number means something different in each encoding, so calling it "Amount" + // everywhere would label a ROW COUNT as an amount, and would drop the declared + // value-column label a row-encoded measure pipeline was given. `RowsCount` folds the + // value and the count into one line because they are the same number. + let stage_hover = |k: usize| -> String { + #[allow(clippy::cast_precision_loss)] + let pct = if *n_complete == 0 { + 0.0 + } else { + reached[k] as f64 / *n_complete as f64 * 100.0 + }; + match shape { + FunnelShape::Columns => format!( + "{}
Stage: {}
Amount: {}
Rows reached: {} of {} ({pct:.0}% of \ + complete cases)", + escape_hover(&labels[k]), + escape_hover(&stages[k]), + fmt_measure(totals[k]), + HumanCount(reached[k] as u64), + HumanCount(*n_complete as u64), + ), + FunnelShape::RowsMeasure { value_label } => format!( + "{}
Stage: {}
{}: {}
Rows in stage: {} of {} ({pct:.0}% of rows \ + in declared stages)", + escape_hover(&labels[k]), + escape_hover(&stages[k]), + escape_hover(value_label), + fmt_measure(totals[k]), + HumanCount(reached[k] as u64), + HumanCount(*n_complete as u64), + ), + FunnelShape::RowsCount => format!( + "{}
Stage: {}
Rows: {} of {} ({pct:.0}% of rows in declared stages)", + escape_hover(&labels[k]), + escape_hover(&stages[k]), + HumanCount(reached[k] as u64), + HumanCount(*n_complete as u64), + ), + } + }; + + cats.push(stages[0].clone()); + vals.push(totals[0]); + measures.push(Measure::Absolute); + hover.push(stage_hover(0)); + + for k in 1..stages.len() { + let delta = totals[k] - totals[k - 1]; + cats.push(format!("{} \u{2212} {}", stages[k], stages[k - 1])); + vals.push(delta); + measures.push(Measure::Relative); + hover.push(format!( + "{} \u{2212} {}
Difference: {}
{} is {} than {}", + escape_hover(&stages[k]), + escape_hover(&stages[k - 1]), + fmt_measure(delta), + escape_hover(&stages[k]), + if delta < 0.0 { "lower" } else { "higher" }, + escape_hover(&stages[k - 1]), + )); + + // `Total` re-derives its own height from the running total; the real amount goes + // in the slot anyway so the arrays stay parallel and the JSON self-documenting. + cats.push(stages[k].clone()); + vals.push(totals[k]); + measures.push(Measure::Total); + hover.push(stage_hover(k)); + } + + bar_max = vals + .iter() + .zip(&measures) + .filter(|(_, m)| !matches!(m, Measure::Relative)) + .map(|(v, _)| *v) + .fold(None, |acc: Option, v| { + Some(acc.map_or(v, |a| f64::max(a, v))) + }); + + // Per-bar templates, because no single one is right for every bar: a step bar's + // number is its DELTA, a stage bar's is the running TOTAL. plotly stays the + // formatter (so the labels cannot drift from the values it draws), and `.3s` matches + // the SI precision the KPI row and the axes already use — its own `textinfo` renders + // seven significant figures. + let templates: Vec = measures + .iter() + .map(|m| { + if matches!(m, Measure::Relative) { + "%{delta:.3s}".to_string() + } else { + "%{final:.3s}".to_string() + } + }) + .collect(); + let mut w = Waterfall::new(cats, vals) + .name(panel.name.clone()) + .measure(measures) + .text_template_array(templates) + .text_position(TextPosition::Outside) + .increasing( + MeasureStyle::new().marker(WaterfallMarker::new().color(BRIDGE_UP_COLOR)), + ) + .decreasing( + MeasureStyle::new().marker(WaterfallMarker::new().color(BRIDGE_DOWN_COLOR)), + ) + .totals(MeasureStyle::new().marker(WaterfallMarker::new().color(color))) + .hover_text_array(hover) + .hover_template("%{hovertext}"); + if let Some((x, y)) = &axes { + w = w.x_axis(x.clone()).y_axis(y.clone()); + } + w + }, + PanelKind::Funnel { + stages, + labels, + totals, + reached, + n_complete, + shape, + form: _, } => { // A horizontal funnel: amount on the value axis, stage on the category axis. // A funnel trace draws index 0 at the TOP and works downward — the opposite of a @@ -23494,7 +23728,11 @@ fn smart_grid_parts( .max(), // the funnel is the other horizontal panel: its category ticks are the short // canonical stage names, which still need left room reserved - PanelKind::Funnel { stages, .. } => stages + PanelKind::Funnel { + stages, + form: PipelineForm::Funnel, + .. + } => stages .iter() .map(|l| l.chars().count().min(TOPREL_LABEL_MAX_CHARS)) .max(), @@ -23848,7 +24086,13 @@ fn smart_grid_parts( let geom = geoms[n].clone(); // the Top Relationships lollipop is the one horizontal panel: value (NMI) on a zoomed x, // pair on the category y — the opposite axis roles from every other (vertical) panel. - let (x_axis, y_axis) = if let PanelKind::Funnel { stages, totals, .. } = &panel.kind { + let (x_axis, y_axis) = if let PanelKind::Funnel { + stages, + totals, + form: PipelineForm::Funnel, + .. + } = &panel.kind + { // the funnel is horizontal like the lollipop, but its value axis is anchored at // zero and its categories run TOP-DOWN, matching the upstream-first arrays the // trace feeds (see the `panel_trace` arm). @@ -23862,6 +24106,18 @@ fn smart_grid_parts( .domain(&geom.y_domain) .anchor(xref.clone()), ) + } else if let PanelKind::Funnel { totals, .. } = &panel.kind { + // a bridge is VERTICAL (see the `panel_trace` arm): categories on x, amounts on y, + // with the ordinary bar headroom so the outside delta labels are not clipped. + let max = totals.iter().copied().fold(0.0_f64, f64::max); + ( + styled_x_axis(false, false, false, theme, None) + .domain(&geom.x_domain) + .anchor(yref.clone()), + styled_y_axis(Some(max), false, theme) + .domain(&geom.y_domain) + .anchor(xref.clone()), + ) } else if let PanelKind::TopRelationships { values, labels, .. } = &panel.kind { let (floor, ceil) = lollipop_value_range(values); // bottom-to-top (weakest-first), matching the reversed y-values the trace feeds @@ -25137,7 +25393,6 @@ fn smart_inline_panel_plot( ); let is_date = matches!(panel.kind, PanelKind::TimeSeries { .. }); let is_toprel = matches!(panel.kind, PanelKind::TopRelationships { .. }); - let is_funnel = matches!(panel.kind, PanelKind::Funnel { .. }); let (trace, bar_max, log_y) = panel_trace(panel, color, freq, hist, outliers, None, theme, log_scale); @@ -25155,9 +25410,18 @@ fn smart_inline_panel_plot( (110, 90) } else if is_toprel { (TOPREL_LABEL_MAX_CHARS * CORR_LABEL_PX_PER_CHAR + 24, 30) - } else if is_funnel { - // right room as well: plotly's in-band text can overflow the widest band - (TOPREL_LABEL_MAX_CHARS * CORR_LABEL_PX_PER_CHAR + 24, 60) + } else if let PanelKind::Funnel { + stages, + form: PipelineForm::Funnel, + .. + } = &panel.kind + { + // size from the funnel's OWN ticks, not the lollipop's budget: the lollipop reserves room + // for 56-char truncated column-PAIR labels, while a funnel's ticks are short stage names + // ("Planned", "Committed"). Borrowing its constant reserved 416px for labels needing ~90, + // leaving the headline panel squeezed into the left half of a full-width cell. + // Right room as well: plotly's in-band text can overflow the widest band. + (funnel_left_margin(stages), 60) } else if log_y { (60 + LOG_AXIS_TITLE_MARGIN_PX, 30) } else { @@ -25165,7 +25429,13 @@ fn smart_inline_panel_plot( }; // the lollipop is the one horizontal panel: value (NMI) on a zoomed x, pair on the category y // — the opposite axis roles from every other (vertical) inline panel. - let (x_axis, y_axis) = if let PanelKind::Funnel { stages, totals, .. } = &panel.kind { + let (x_axis, y_axis) = if let PanelKind::Funnel { + stages, + totals, + form: PipelineForm::Funnel, + .. + } = &panel.kind + { let ticks: Vec = stages.clone(); let max = totals.iter().copied().fold(0.0_f64, f64::max); ( @@ -26977,7 +27247,12 @@ fn styled_y_axis(headroom_max: Option, log: bool, theme: Option) -> Ax a } +/// Left margin (px) reserved for a funnel panel's category ticks in the INLINE render path. +/// +/// Mirrors the per-label sizing `smart_grid_parts` already does for its shared margin, rather +/// than borrowing `TOPREL_LABEL_MAX_CHARS` — that budget is sized for the lollipop's 56-char +/// truncated column-PAIR labels, and a funnel's ticks are short stage names. +fn funnel_left_margin(stages: &[String]) -> usize { + stages + .iter() + .map(|l| l.chars().count().min(TOPREL_LABEL_MAX_CHARS)) + .max() + .map_or(60, |longest| longest * CORR_LABEL_PX_PER_CHAR + 24) +} + /// The VALUE (x) axis for a horizontal pipeline funnel (issue #4222). /// -/// Anchored at **zero**, unlike `lollipop_value_axis`'s zoomed range. A funnel's whole claim is -/// that band widths are proportional to stage amounts; a floor above zero would exaggerate the -/// taper, which is the one thing this panel must not do. The headroom above `max` leaves room for -/// plotly's in-band `textinfo` on the widest band. +/// **Symmetric about zero, because plotly centers funnel bands on the value zero** — each band +/// spans `-amount/2 ..= +amount/2`. A `0..=ceil` range therefore puts the band centre on the +/// LEFT EDGE of the plot area and draws half of every band outside it. That was survivable only +/// because the panel happened to reserve a 416px left margin wide enough to absorb the overflow; +/// sizing that margin to the actual tick labels (which is what it is for) clipped the widest band. /// -/// Never logged, for the same reason — a log value axis destroys the proportional reading. +/// So the range is half-width either side of zero: the widest band still occupies `1/1.35` of the +/// plot area, exactly as before, but centred rather than half off-screen — and the remaining +/// headroom leaves room for plotly's `textinfo` beside the widest band. +/// +/// Tick labels are hidden: with a symmetric range they would read `-100G … 100G` for amounts that +/// are never negative, and a funnel's value axis carries no information the panel doesn't already +/// state — the band text gives stage-to-stage conversion and the hover gives exact amounts. The +/// standalone `viz funnel` likewise draws no value axis. +/// +/// Never logged — a log value axis destroys the proportional reading the form depends on. fn funnel_value_axis(max: f64, theme: Option) -> Axis { - let ceil = if max > 0.0 { max * 1.35 } else { 1.0 }; + let half = if max > 0.0 { max * 1.35 / 2.0 } else { 1.0 }; let mut a = Axis::new() - .show_grid(true) - .grid_width(1) + .show_grid(false) .zero_line(false) .show_line(false) - .range(vec![0.0, ceil]); + .show_tick_labels(false) + // SI prefixes, matching the rest of the dashboard's value formatting (see `styled_y_axis`) + .exponent_format(ExponentFormat::SI) + .range(vec![-half, half]); if theme.is_none() { - a = a - .grid_color(GRID_COLOR) - .tick_color(AXIS_LINE) - .tick_font(Font::new().family(FONT_FAMILY).size(10)); + a = a.tick_color(AXIS_LINE); } a } @@ -31060,6 +31357,87 @@ mod tests { assert!((containment_fraction(&[], &[]) - 0.0).abs() < f64::EPSILON); } + #[test] + fn pipeline_form_follows_the_totals_not_the_rows() { + use PipelineForm::{Bridge, Funnel}; + // monotonically shrinking: the containment claim holds + assert_eq!(PipelineForm::for_totals(&[100.0, 60.0, 40.0]), Funnel); + // flat stages still nest (a plateau is not growth) + assert_eq!(PipelineForm::for_totals(&[100.0, 100.0, 40.0]), Funnel); + // one growing step is enough to make a funnel dishonest -- this is the CPDB shape, + // where Spent (81G) outruns Committed (28.4G) + assert_eq!( + PipelineForm::for_totals(&[191.75, 28.38, 80.98]), + Bridge, + "a stage that outruns its predecessor must not be drawn as containment" + ); + // growth anywhere counts, including the last step only + assert_eq!(PipelineForm::for_totals(&[100.0, 60.0, 61.0]), Bridge); + // degenerate inputs are funnels by default -- nothing contradicts containment + assert_eq!(PipelineForm::for_totals(&[]), Funnel); + assert_eq!(PipelineForm::for_totals(&[42.0]), Funnel); + } + + #[test] + fn complete_pct_str_never_rounds_up_to_100() { + // the real CPDB case: 12,566 of 12,587 rows. `{:.0}` rendered this as "100", asserting + // completeness while a nine-figure sum was silently dropped. + assert_eq!(complete_pct_str(12_566.0 / 12_587.0), "99.8"); + + // rounding would reach 100 well before completeness does; flooring must not + assert_eq!(complete_pct_str(0.9996), "99.9"); + assert_eq!(complete_pct_str(0.999_999), "99.9"); + + // only genuine completeness prints a bare 100 + assert_eq!(complete_pct_str(1.0), "100"); + + // and the ordinary cases still read naturally + assert_eq!(complete_pct_str(0.5), "50.0"); + assert_eq!(complete_pct_str(0.0), "0.0"); + } + + #[test] + fn funnel_left_margin_sizes_to_the_stage_labels() { + let stages: Vec = ["Planned", "Committed", "Spent"] + .iter() + .map(|s| (*s).to_string()) + .collect(); + // "Committed" is 9 chars -> 9*7 + 24. The bug was borrowing the lollipop's 56-char + // budget (56*7 + 24 = 416px) for labels that need a fraction of it. + assert_eq!(funnel_left_margin(&stages), 9 * CORR_LABEL_PX_PER_CHAR + 24); + assert!(funnel_left_margin(&stages) < TOPREL_LABEL_MAX_CHARS * CORR_LABEL_PX_PER_CHAR + 24); + + // an absurdly long stage name is still capped at the lollipop budget + let long = vec!["x".repeat(200)]; + assert_eq!( + funnel_left_margin(&long), + TOPREL_LABEL_MAX_CHARS * CORR_LABEL_PX_PER_CHAR + 24 + ); + + // no stages: fall back to the default left margin rather than zero + assert_eq!(funnel_left_margin(&[]), 60); + } + + #[test] + fn funnel_value_axis_is_symmetric_about_zero() { + // plotly centres funnel bands on the value zero, so a 0..=ceil range would draw half of + // every band outside the plot area (only survivable while an oversized left margin + // absorbed the overflow). + let axis = serde_json::to_value(funnel_value_axis(100.0, None)).unwrap(); + let range = axis["range"].as_array().unwrap(); + let lo = range[0].as_f64().unwrap(); + let hi = range[1].as_f64().unwrap(); + assert!(lo < 0.0 && hi > 0.0, "range must straddle zero: {lo}..{hi}"); + assert!( + (lo + hi).abs() < 1e-9, + "range must be symmetric: {lo}..{hi}" + ); + // the widest band still occupies 1/1.35 of the plot area + assert!((hi - 100.0 * 1.35 / 2.0).abs() < 1e-9); + // and its tick labels stay hidden: a symmetric range would read as negative amounts + assert_eq!(axis["showticklabels"], serde_json::json!(false)); + } + #[test] fn panel_title_escapes_markup_with_and_without_subtitle() { // a hostile column header must not reach plotly's markup renderer as live markup on diff --git a/tests/test_viz.rs b/tests/test_viz.rs index 88ba0e4e00..9a66fdcf24 100644 --- a/tests/test_viz.rs +++ b/tests/test_viz.rs @@ -12056,8 +12056,9 @@ fn viz_smart_builds_pipeline_funnel_from_a_dictionary() { #[test] fn viz_smart_funnel_honors_declared_order_over_magnitude() { // The declaration is authoritative: stages are NOT re-sorted by size. Reversing the members - // must reverse the bands even though that makes the funnel widen downward -- if the tool - // silently sorted, a mis-declared pipeline would look plausible and never be noticed. + // must reverse the panel's order -- if the tool silently sorted, a mis-declared pipeline + // would look plausible and never be noticed. Reversing necessarily makes the totals GROW, so + // the form becomes a bridge; the order guarantee is what is under test here, not the form. let wrk = Workdir::new("viz_smart_funnel_honors_declared_order_over_magnitude"); let html = smart_with_dict( &wrk, @@ -12065,8 +12066,12 @@ fn viz_smart_funnel_honors_declared_order_over_magnitude() { &pipeline_dict(r#"["spentamt","commitamt","totalplannedcommit"]"#), ); assert!( - html.contains(r#""y":["spentamt","commitamt","totalplannedcommit"]"#), - "declared order must survive verbatim; html: {html}" + html.contains( + "\"x\":[\"spentamt\",\"commitamt \u{2212} \ + spentamt\",\"commitamt\",\"totalplannedcommit \u{2212} \ + commitamt\",\"totalplannedcommit\"]" + ), + "declared order must survive verbatim, bridged step-by-step; html: {html}" ); } @@ -12132,11 +12137,11 @@ fn viz_smart_funnel_denominator_covers_only_the_declared_stages() { #[test] fn viz_smart_funnel_discloses_containment_violations_instead_of_refusing() { - // INVERTED by the #4222 rework. This table's `spentamt` overruns `commitamt` on ~30% of rows, - // which used to trip a 0.90 containment GATE and suppress the panel entirely. Containment is - // now a MEASUREMENT, not a gate: the funnel draws and the subtitle names the violation share. - // This is what lets the motivating NYC CPDB dataset -- three aggregates on different - // accounting bases -- render at all. + // INVERTED by the #4222 rework, then refined again. This table's `spentamt` overruns + // `commitamt` on ~30% of rows and in TOTAL, so the panel still draws -- containment is a + // measurement, not a gate -- but it draws as a BRIDGE, because a funnel's band widths are a + // containment claim these numbers contradict. The subtitle still names the violation share. + // This is the motivating NYC CPDB shape: three aggregates on different accounting bases. let wrk = Workdir::new("viz_smart_funnel_discloses_containment_violations_instead_of_refusing"); let mut rows = String::from("totalplannedcommit,commitamt,spentamt\n"); for i in 0..300 { @@ -12152,14 +12157,23 @@ fn viz_smart_funnel_discloses_containment_violations_instead_of_refusing() { &pipeline_dict(r#"["totalplannedcommit","commitamt","spentamt"]"#), ); assert!( - html.contains(r#""type":"funnel""#), - "a declared pipeline draws even when its stages do not nest; html: {html}" + html.contains(r#""type":"waterfall""#), + "a declared pipeline whose stages do not nest draws as a bridge, not a funnel; html: \ + {html}" + ); + assert!( + !html.contains(r#""type":"funnel""#), + "a funnel would assert the containment these totals contradict; html: {html}" ); assert!( html.contains("exceeds"), "the subtitle must NAME the containment violation rather than hiding the panel; html: \ {html}" ); + assert!( + html.contains("stages do not nest"), + "the subtitle must say why the form is a bridge; html: {html}" + ); } #[test] @@ -12273,11 +12287,14 @@ fn viz_smart_builds_a_row_encoded_funnel_summing_a_value_column() { let wrk = Workdir::new("viz_smart_builds_a_row_encoded_funnel_summing_a_value_column"); let mut rows = String::from("stage,revenue,region\n"); for i in 0..300 { + // per-stage spend shrinks down the pipeline, so the stage TOTALS nest and the panel + // stays a funnel -- this test is about the row encoding, not the form (see + // `viz_smart_row_pipeline_that_grows_is_bridged` for the other branch) let (stage, rev) = match i % 10 { - 0..=4 => ("Impression", 1), - 5..=7 => ("Click", 4), - 8 => ("Lead", 20), - _ => ("Conversion", 90), + 0..=4 => ("Impression", 90), + 5..=7 => ("Click", 20), + 8 => ("Lead", 4), + _ => ("Conversion", 1), }; rows.push_str(&format!("{stage},{rev},r{}\n", i % 3)); } @@ -12369,6 +12386,133 @@ fn viz_smart_row_funnel_counts_rows_when_no_value_column_is_declared() { ); } +#[test] +fn viz_smart_row_pipeline_that_grows_is_bridged() { + // The row encoding is not exempt from the form rule. A marketing pipeline summing REVENUE + // grows down the stages even though the row COUNTS shrink -- revenue at conversion is not a + // subset of revenue at impression -- so a funnel would widen downward and assert a + // containment that does not hold. The declaration is still honoured: same stages, same + // order, bridged instead. + let wrk = Workdir::new("viz_smart_row_pipeline_that_grows_is_bridged"); + let mut rows = String::from("stage,revenue,region\n"); + for i in 0..300 { + let (stage, rev) = match i % 10 { + 0..=4 => ("Impression", 1), + 5..=7 => ("Click", 4), + 8 => ("Lead", 20), + _ => ("Conversion", 90), + }; + rows.push_str(&format!("{stage},{rev},r{}\n", i % 3)); + } + wrk.create_from_string("p.csv", &rows); + wrk.create_from_string( + "d.schema.json", + r#"{ + "properties": { + "stage": {"type":"string","x-qsv":{"qsv_type":"String","role":"dimension"}}, + "revenue": {"type":"integer","x-qsv":{"qsv_type":"Integer","role":"measure"}}, + "region": {"type":"string","x-qsv":{"qsv_type":"String","role":"dimension"}} + }, + "x-qsv": { "relationships": [ + {"kind":"pipeline","members":["stage","revenue"],"stage_column":"stage", + "stages":["Impression","Click","Lead","Conversion"],"value_column":"revenue"} + ] } + }"#, + ); + let out_html = wrk.path("p.html").to_string_lossy().to_string(); + let mut cmd = wrk.command("viz"); + cmd.args(["smart", "p.csv", "-o", &out_html, "--dictionary"]) + .arg(wrk.path("d.schema.json")) + .env("QSV_VIZ_NO_COMPRESS", "1"); + wrk.assert_success(&mut cmd); + + let html = wrk.read_to_string("p.html").unwrap(); + assert!( + html.contains(r#""type":"waterfall""#), + "a row-encoded pipeline whose value grows must be bridged; html: {html}" + ); + assert!( + html.contains("Pipeline bridge:"), + "the title must say bridge, not funnel; html: {html}" + ); + assert!( + html.contains( + r#""measure":["absolute","relative","total","relative","total","relative","total"]"# + ), + "four stages bridge into seven bars: a seed, then a step and a running total each; html: \ + {html}" + ); + // the stage hover must use the DECLARED value-column label, not the column encoding's + // generic "Amount" -- the bridge borrowed that wording and mislabelled every row encoding + assert!( + html.contains("revenue: "), + "a row-encoded measure bridge must label the value with its declared value column; html: \ + {html}" + ); + assert!( + !html.contains("Amount: "), + "\"Amount\" is the COLUMN encoding's wording; html: {html}" + ); +} + +#[test] +fn viz_smart_row_count_bridge_says_rows_not_amount() { + // A count-only row pipeline whose stages GROW. `reached` IS the value here, so the funnel + // arm folds both into one "Rows: n of m" line; the bridge used to print "Amount: 100" + // alongside a duplicate "Rows in stage: 100", labelling a row count as an amount and + // stating the same number twice. + let wrk = Workdir::new("viz_smart_row_count_bridge_says_rows_not_amount"); + let mut rows = String::from("stage,region\n"); + for i in 0..300 { + let stage = if i % 6 == 0 { + "Impression" + } else if i % 6 < 3 { + "Click" + } else { + "Conversion" + }; + rows.push_str(&format!("{stage},r{}\n", i % 3)); + } + wrk.create_from_string("p.csv", &rows); + wrk.create_from_string( + "d.schema.json", + r#"{ + "properties": { + "stage": {"type":"string","x-qsv":{"qsv_type":"String","role":"dimension"}}, + "region": {"type":"string","x-qsv":{"qsv_type":"String","role":"dimension"}} + }, + "x-qsv": { "relationships": [ + {"kind":"pipeline","members":["stage"],"stage_column":"stage", + "stages":["Impression","Click","Conversion"]} + ] } + }"#, + ); + let out_html = wrk.path("p.html").to_string_lossy().to_string(); + let mut cmd = wrk.command("viz"); + cmd.args(["smart", "p.csv", "-o", &out_html, "--dictionary"]) + .arg(wrk.path("d.schema.json")) + .env("QSV_VIZ_NO_COMPRESS", "1"); + wrk.assert_success(&mut cmd); + + let html = wrk.read_to_string("p.html").unwrap(); + assert!( + html.contains(r#""type":"waterfall""#), + "growing stage counts must be bridged; html: {html}" + ); + assert!( + html.contains("Rows: "), + "a count-only bridge must call the value Rows; html: {html}" + ); + assert!( + !html.contains("Amount: "), + "a row count is not an Amount; html: {html}" + ); + assert!( + !html.contains("Rows in stage: "), + "the count line is redundant when the count IS the value; html: {html}" + ); +} + #[test] fn viz_smart_row_funnel_tolerates_stage_case_drift_and_absent_stages() { // The stage values are transcribed by an LLM from the frequency distribution, so case drift @@ -12378,7 +12522,9 @@ fn viz_smart_row_funnel_tolerates_stage_case_drift_and_absent_stages() { let wrk = Workdir::new("viz_smart_row_funnel_tolerates_stage_case_drift_and_absent_stages"); let mut rows = String::from("stage,region\n"); for i in 0..300 { - let stage = if i % 3 == 0 { "impression" } else { "click" }; + // impressions outnumber clicks so the counts nest and the panel stays a funnel: this + // test is about case-insensitive stage matching, not the form + let stage = if i % 3 == 0 { "click" } else { "impression" }; rows.push_str(&format!("{stage},r{}\n", i % 3)); } wrk.create_from_string("p.csv", &rows);