fix(hubs): repair ListCost/ContractedCost for rows without a meter/offer ID

docs-mslearn/toolkit/changelog.md

@@ -3,7 +3,7 @@ title: FinOps toolkit changelog
 description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more.
 author: MSBrett
 ms.author: brettwil
-ms.date: 08/22/2026
+ms.date: 09/01/2026
 ms.topic: reference
 ms.service: finops
 ms.subservice: finops-toolkit
@@ -38,6 +38,7 @@ The following section lists features and enhancements that are currently in deve
   - Fixed the `ContractedCost` recompute guard to compare with a null-safe tolerance instead of exact float equality, eliminating millions of no-op rewrites that polluted the `x_SourceValues` audit trail while preserving the null-cost backfill and no longer overwriting an existing cost when the unit price is missing ([#2216](https://github.com/microsoft/finops-toolkit/issues/2216)).
   - Fixed the SQL VMs without Azure Hybrid Benefit recommendation query to join on the SQL VM `virtualMachineResourceId` instead of a case-sensitive VM name match that skipped VMs with uppercase names and dropped duplicate names, and made all Azure Resource Graph join kinds explicit so no query relies on the `innerunique` default ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
   - Switched dimension enrichment in the v1_0/v1_2 ingestion transforms (`PricingUnits`, `Regions`, `ResourceTypes`, `Services`) from `join` to the broadcast-optimized `lookup` operator and deduplicated the `Services` mapping per resource type to prevent cost row fan-out ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)).
+  - Fixed `ListCost`/`ContractedCost` never being repaired for rows without a Microsoft meter or offer ID (for example, third-party Marketplace/ISV purchases), which left `ListCost` at 0 despite a real `EffectiveCost` and corrupted Effective Savings Rate reporting; also cleared the `MissingListCost` data-quality flag once a row is successfully repaired instead of leaving it flagged as broken ([#2214](https://github.com/microsoft/finops-toolkit/issues/2214), [#2235](https://github.com/microsoft/finops-toolkit/issues/2235)).
 
 ### [FinOps workbooks](workbooks/finops-workbooks-overview.md)
 

src/powershell/Tests/Unit/HubsMissingCostGate.Tests.ps1

@@ -0,0 +1,115 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+<#
+    Regression coverage for the missing-cost gate split (#2214 / #2235 / #2286):
+    the ListCost/ContractedCost repair fallback must not require a meter ID and offer ID -- only the
+    price-sheet lookup does. Rows without either (most commonly third-party Marketplace/ISV purchases,
+    which have no Microsoft retail list price by design) must still reach the repair fallback so ListCost
+    isn't left at 0 despite a real EffectiveCost, which was corrupting x_TotalSavings and Effective Savings
+    Rate reporting.
+
+    Per-row behavioral coverage lives in the executable harness
+    Tests/assets/MissingCostGateSplit.kql (PASS = 0 returned rows on any Kusto database).
+
+    v1.0 also gets the MissingListCost flag-clearing fix (defect 4), but NOT the x_SourceValues audit trail
+    (defect 3) -- v1.0's Costs_final_v1_0 schema is intentionally frozen so people can revert to legacy
+    behavior; adding a column there is out of scope permanently, not just for this change.
+#>
+
+Describe 'HubsMissingCostGate' {
+
+    BeforeDiscovery {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $scriptsPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts'
+        $guardFiles = @('IngestionSetup_v1_0.kql', 'IngestionSetup_v1_2.kql') | ForEach-Object {
+            @{ Name = $_; FullName = (Join-Path $scriptsPath $_) }
+        }
+    }
+
+    BeforeAll {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $harnessPath = Join-Path $repoRoot 'src/powershell/Tests/assets/MissingCostGateSplit.kql'
+    }
+
+    Context 'Gate split' {
+
+        It 'Should gate the cost-repair fallback on tmp_MissingCost, not tmp_MissingPrices: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match 'extend\s+tmp_MissingCost\s*=' `
+                -Because 'the cost-repair fallback must be reachable without a meter/offer ID (#2214); tmp_MissingCost is the broader gate that no longer requires them'
+        }
+
+        It 'Should narrow tmp_MissingPrices from tmp_MissingCost plus the meter/offer ID requirement: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match 'extend\s+tmp_MissingPrices\s*=\s*tmp_MissingCost\s+and\s+isnotempty\(x_SkuMeterId\)\s+and\s+isnotempty\(x_SkuOfferId\)' `
+                -Because 'only the price-sheet join needs a meter/offer ID; the cost-repair fallback below does not'
+        }
+
+        It 'Should filter into the repair block on tmp_MissingCost, not tmp_MissingPrices: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match '(?m)\|\s*where\s+tmp_MissingCost\s*$' -Because 'rows without a meter/offer ID must still enter the repair block, just skip the price-sheet join'
+        }
+
+        It 'Should restrict the price-sheet join to rows that pass tmp_MissingPrices: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match 'costsWithMissingPrices\s*\|\s*where\s+tmp_MissingPrices\s*\|\s*summarize\s+by\s+tmp_ReservationPriceLookupKey' `
+                -Because 'a row without a meter/offer ID has no valid lookup key and must not join to the price sheet'
+        }
+
+        It 'Should merge unrepaired rows back by tmp_MissingCost, not tmp_MissingPrices: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match 'union\s*\(allCosts\s*\|\s*where\s+not\(tmp_MissingCost\)\)' `
+                -Because 'rows that entered the repair block are gated by tmp_MissingCost now, so the merge-back of untouched rows must exclude the same set'
+        }
+    }
+
+    Context 'MissingListCost flag clears after repair' {
+
+        It 'Should clear MissingListCost from x_SourceChanges once ListCost is repaired: <Name>' -ForEach $guardFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match "iff\(\(isnotempty\(ListCost\) and ListCost != 0\),\s*\r?\n\s*trim_end\(',', replace_string\(replace_string\(x_SourceChanges, 'MissingListCost,', ''\), 'MissingListCost', ''\)\),\s*\r?\n\s*x_SourceChanges\)" `
+                -Because 'MissingListCost is computed before the repair runs, so a repaired row must not still report itself as broken (#2214 defect 4)'
+        }
+    }
+
+    Context 'Equivalence harness' {
+
+        It 'Should have the gate-split harness asset' {
+            Test-Path $harnessPath | Should -BeTrue
+        }
+
+        It 'Should assert both the repaired outcome and the changed-vs-old-gate outcome per row' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match '\| where new_entersRepair != expectedRepaired or \(old_entersRepair != new_entersRepair\) != expectedChanged' `
+                -Because 'the harness must return only rows that violate either the final gate outcome or the expected delta from the old gate'
+        }
+
+        It 'Should cover rows missing only one of meter ID or offer ID' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match 'only meter ID set' -Because 'the old gate required both IDs; a row with just one must still change behavior'
+            $harness | Should -Match 'only offer ID set' -Because 'the old gate required both IDs; a row with just one must still change behavior'
+        }
+
+        It 'Should cover the unused spend commitment exclusion' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match 'unused spend commitment' -Because 'unused spend commitments must stay excluded from the repair fallback under both gates'
+        }
+
+        It 'Should cover the non-Microsoft provider exclusion' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match 'non-Microsoft provider' -Because 'the gate is Microsoft-only under both old and new behavior'
+        }
+    }
+
+    Context 'v1.0 schema is frozen' {
+
+        It 'Should not add x_SourceValues to the v1.0 schema' {
+            $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+            $v10Path = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql'
+            $content = Get-Content -Path $v10Path -Raw
+            $content | Should -Not -Match 'x_SourceValues' `
+                -Because 'Costs_final_v1_0 is kept for people who want to revert to legacy behavior; its schema is intentionally frozen and must never gain new columns'
+        }
+    }
+}

src/powershell/Tests/assets/MissingCostGateSplit.kql

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//======================================================================================================================
+// Missing-cost gate split regression harness
+//
+// Verifies, per row, the behavior change shipped for #2214/#2235/#2286: the ListCost/ContractedCost repair fallback
+// used to be gated behind the same condition as the price-sheet lookup, which requires a meter ID and offer ID.
+// Rows without either -- most commonly third-party Marketplace/ISV purchases, which have no Microsoft retail list
+// price by design -- were excluded from the whole repair block and never got a repaired ListCost/ContractedCost,
+// even though a real EffectiveCost was present. That leaves ListCost at 0 despite real spend, which collapses
+// x_TotalSavings and corrupts downstream Effective Savings Rate reporting.
+//
+//   old: tmp_MissingPrices gates BOTH the price-sheet join AND the cost-repair fallback
+//        (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0)
+//            and x_EffectiveUnitPrice != 0 and not(unused spend commitment)
+//            and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+//
+//   new: tmp_MissingCost (no meter/offer ID requirement) gates the cost-repair fallback;
+//        tmp_MissingPrices narrows that further to rows the price-sheet join can resolve
+//        tmp_MissingCost   = (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0)
+//            and x_EffectiveUnitPrice != 0 and not(unused spend commitment)
+//        tmp_MissingPrices = tmp_MissingCost and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+//
+// A row that is tmp_MissingCost but NOT tmp_MissingPrices now enters the repair block (skips the join, all joined
+// columns come back null) and falls through the existing case() branches' null-safe fallback to EffectiveCost-based
+// values -- the same outcome #2214 asked for, reusing logic that already exists for the join's no-match case.
+//
+// How to run: paste into any Kusto database (ADX or Fabric eventhouse; no table access required).
+//   - PASS = the query returns 0 rows.
+//
+// expectedRepaired = whether the row should enter the repair fallback (new_entersRepair) after the fix.
+// expectedChanged  = whether this row's outcome should actually differ from the old gate (old_entersRepair). Rows
+// with a meter/offer ID behave identically under both gates -- only rows missing one or both IDs should change.
+//======================================================================================================================
+
+let cases = datatable(
+    label:string,
+    provider:string,
+    listPrice:real,
+    contractedPrice:real,
+    effectivePrice:real,
+    commitmentCategory:string,
+    commitmentStatus:string,
+    meterId:string,
+    offerId:string,
+    expectedRepaired:bool,
+    expectedChanged:bool
+) [
+    'ISV row, no meter/offer ID -- the #2214 case',
+        'Microsoft', real(0),    real(0),    real(1.5), '',      '',       '',       '',       true,  true,
+    'Microsoft usage, has meter/offer ID -- price-sheet lookup applies as before',
+        'Microsoft', real(0),    real(0),    real(1.5), '',      '',       'meter1', 'offer1', true,  false,
+    'Microsoft usage, only meter ID set (no offer ID) -- still enters repair, still skips the join',
+        'Microsoft', real(0),    real(0),    real(1.5), '',      '',       'meter1', '',       true,  true,
+    'Microsoft usage, only offer ID set (no meter ID) -- still enters repair, still skips the join',
+        'Microsoft', real(0),    real(0),    real(1.5), '',      '',       '',       'offer1', true,  true,
+    'prices already populated -- gate should not fire either way',
+        'Microsoft', real(1.5),  real(1.5),  real(1.5), '',      '',       '',       '',       false, false,
+    'no effective price -- nothing to repair from',
+        'Microsoft', real(0),    real(0),    real(0),   '',      '',       '',       '',       false, false,
+    'unused spend commitment -- excluded on purpose, no consumption to base a fallback on',
+        'Microsoft', real(0),    real(0),    real(1.5), 'Spend', 'Unused', '',       '',       false, false,
+    'non-Microsoft provider -- gate is Microsoft-only',
+        'AWS',       real(0),    real(0),    real(1.5), '',      '',       '',       '',       false, false,
+    'only contracted price missing, meter/offer ID absent -- still enters repair under the new gate',
+        'Microsoft', real(1.5),  real(0),    real(1.5), '',      '',       '',       '',       true,  true,
+]
+| extend old_entersRepair = coalesce(provider == 'Microsoft'
+    and (listPrice == 0 or contractedPrice == 0)
+    and effectivePrice != 0
+    and not(commitmentCategory == 'Spend' and commitmentStatus == 'Unused')
+    and isnotempty(meterId) and isnotempty(offerId), false)
+| extend new_entersRepair = coalesce(provider == 'Microsoft'
+    and (listPrice == 0 or contractedPrice == 0)
+    and effectivePrice != 0
+    and not(commitmentCategory == 'Spend' and commitmentStatus == 'Unused'), false)
+| where new_entersRepair != expectedRepaired or (old_entersRepair != new_entersRepair) != expectedChanged
+| order by label asc

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql

@@ -403,20 +403,28 @@ Costs_transform_v1_0()
         ConsumedQuantity
     )
     //
-    // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID
-    | extend tmp_MissingPrices = ProviderName == 'Microsoft'
+    // Populate missing costs -- the price-sheet lookup below needs a meter ID and offer ID to map to an on-demand
+    // price, but the cost fallback further down doesn't. Rows without either ID (e.g. third-party Marketplace/ISV
+    // purchases, which have no Microsoft retail list price by design) used to be excluded from this whole block and
+    // never reached the cost fallback, leaving ListCost/ContractedCost at 0 despite a real EffectiveCost -- which
+    // breaks downstream savings/ESR math (#2214, #2286). tmp_MissingCost is the broader set that flows through to
+    // the fallback case() blocks below; tmp_MissingPrices narrows that to rows the price-sheet join can actually
+    // resolve. Rows in tmp_MissingCost but not tmp_MissingPrices skip the join (all Prices_final_v1_0 columns come
+    // back null) and fall through the case() blocks' existing null-safe branches straight to the EffectiveCost-based
+    // fallback -- the same outcome #2214's fix asks for, using logic that already exists for the join's no-match case.
+    | extend tmp_MissingCost = ProviderName == 'Microsoft'
         and (ListUnitPrice == 0 or ContractedUnitPrice == 0)
         and x_EffectiveUnitPrice != 0
         and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused')
-        and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+    | extend tmp_MissingPrices = tmp_MissingCost and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
     | as allCosts
-    | where tmp_MissingPrices
+    | where tmp_MissingCost
     | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId))
     | as costsWithMissingPrices
     | join kind=leftouter (
         Prices_final_v1_0
         | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(x_EffectivePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId))
-        | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | summarize by tmp_ReservationPriceLookupKey))
+        | where x_SkuPriceType == 'Consumption' and tmp_ReservationPriceLookupKey in ((costsWithMissingPrices | where tmp_MissingPrices | summarize by tmp_ReservationPriceLookupKey))
         // When duplicate price rows collapse under one lookup key, use the highest on-demand price as the savings baseline; min() can pick an anomalously low row and underreport savings
         | summarize ListUnitPrice = max(ListUnitPrice), ContractedUnitPrice = max(ContractedUnitPrice) by tmp_ReservationPriceLookupKey, x_PricingBlockSize, PricingUnit
     ) on tmp_ReservationPriceLookupKey
@@ -482,12 +490,21 @@ Costs_transform_v1_0()
         ListCost
     )
     // Merge the rest of the unmodified cost records and remove excess columns
-    | union (allCosts | where not(tmp_MissingPrices))
-    | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1
+    | union (allCosts | where not(tmp_MissingCost))
+    | project-away x_PricingBlockSize1, PricingUnit1, ListUnitPrice1, ContractedUnitPrice1, tmp_MissingCost, tmp_MissingPrices, tmp_ReservationPriceLookupKey, tmp_ReservationPriceLookupKey1
     //
     // BUG: Fix ContractedCost that has bad values
     | extend ContractedCost = iff(ProviderName == 'Microsoft' and isnotempty(PricingQuantity) and isnotempty(x_PricingBlockSize) and isnotempty(ContractedUnitPrice) and (isempty(ContractedCost) or abs(ContractedCost - ContractedUnitPrice * PricingQuantity) >= 0.0001), ContractedUnitPrice * PricingQuantity, ContractedCost)
     //
+    // MissingListCost was flagged before the repair above ran, so a repaired row still carries the flag and can't be
+    // told apart from a row the repair couldn't fix (#2214, #2286). Clear it once ListCost is known-good post-repair.
+    // MissingListCost keeps its trailing comma unless it's the last flag in the string (trim_end strips only the
+    // final comma of the whole value), so both forms need stripping; trim_end re-cleans a comma left dangling at
+    // the end if MissingListCost, was that last flag.
+    | extend x_SourceChanges = iff((isnotempty(ListCost) and ListCost != 0),
+        trim_end(',', replace_string(replace_string(x_SourceChanges, 'MissingListCost,', ''), 'MissingListCost', '')),
+        x_SourceChanges)
+    //
     // Handle FOCUS 1.0-preview UsageQuantity/Unit
     | extend ConsumedQuantity = iff(ChargeCategory == 'Usage', coalesce(ConsumedQuantity, UsageQuantity, UsageAmount), todecimal(''))
     | extend ConsumedUnit = iff(ChargeCategory == 'Usage' and isnotempty(ConsumedQuantity), coalesce(ConsumedUnit, UsageUnit, 'Units'), '')