From 1289625ff9a159c0731c1515c95e85cf83e617be Mon Sep 17 00:00:00 2001 From: amanpruthi Date: Mon, 31 Aug 2026 10:54:33 +0530 Subject: [PATCH 1/3] feat(cost-explorer): add aws.cost.undiscounted metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An AWS account under a 100% promotional/committed credit shows Credit == -Usage in Cost Explorer every day, so UnblendedCost/AmortizedCost read ~$0 while real infra consumption continues unseen. Adds a separate GetCostAndUsage call filtered to RECORD_TYPE IN (Usage, SavingsPlanCoveredUsage) — the only record types that carry positive consumption at on-demand/list-price value — exported as aws.cost.undiscounted. A positive filter (include only these types) rather than an exclude-list of every known discount/credit RECORD_TYPE, so a new AWS discount type added later is automatically excluded instead of silently leaking in. Ported from the validated KEPT_RECORD_TYPES set in pde_daily_glassbox_cost_reporter_clone/l9cost/aws_cost.py (measured across a 37-day, six-account audit). Verified live against a credit-covered test account: undiscounted rows export correctly, summing to a real per-day cost vs. the near-zero noise from UnblendedCost/AmortizedCost. Co-Authored-By: Claude Sonnet 5 --- aws-cost-explorer/README.md | 5 ++ aws-cost-explorer/main.py | 154 ++++++++++++++++++++++++++++---- aws-cost-explorer/test_parse.py | 55 +++++++++++- 3 files changed, 195 insertions(+), 19 deletions(-) diff --git a/aws-cost-explorer/README.md b/aws-cost-explorer/README.md index d9bf4e1..7b91d61 100644 --- a/aws-cost-explorer/README.md +++ b/aws-cost-explorer/README.md @@ -68,9 +68,14 @@ docker compose up |---|---|---| | `aws.cost.unblended` | USD | `aws.service`, `aws.account.id`, `aws.region`, `cost.date` | | `aws.cost.amortized` | USD | same — RI and Savings Plan effective rates applied | +| `aws.cost.undiscounted` | USD | same — list-price cost, ignoring credits/discounts/Savings-Plan fees | The CE API caps `GroupBy` at 2 dimensions. Each run calls `GetDimensionValues` to list linked accounts, then runs one `GetCostAndUsage` per account filtered by `LINKED_ACCOUNT` and grouped by `SERVICE + REGION`. This preserves all three dimensions in a single series without exceeding the API limit. +### `aws.cost.undiscounted` — why a third metric + +An account under a promotional or committed AWS credit shows `Credit == -Usage` in Cost Explorer every day, so `UnblendedCost`/`AmortizedCost` read ~$0 while real infra consumption continues unseen. `aws.cost.undiscounted` is a separate `GetCostAndUsage` call filtered to `RECORD_TYPE IN (Usage, SavingsPlanCoveredUsage)` — the only record types that carry positive consumption at on-demand value — so it stays meaningful even when the credit fully offsets the standard metrics. It's a positive filter (include only these types) rather than an exclude-list of every known discount/credit RECORD_TYPE, so a new AWS discount type added later is automatically excluded instead of silently leaking into the total. + `cost.date` (`YYYY-MM-DD`) encodes the billing date as a label so each day forms a distinct series. This matters when `DAYS_BACK > 1`: without it, all fetched days share identical labels and only the last sample survives per series. At the default `DAYS_BACK=1`, `cost.date` adds one unique label value per run with no cardinality overhead. ## Dashboard diff --git a/aws-cost-explorer/main.py b/aws-cost-explorer/main.py index c52187d..4045bce 100644 --- a/aws-cost-explorer/main.py +++ b/aws-cost-explorer/main.py @@ -5,8 +5,14 @@ No CUR setup or S3 bucket required — data is available within minutes. Metrics exported: - aws.cost.unblended (USD) — daily unblended cost per service/account/region - aws.cost.amortized (USD) — daily amortized cost (includes RI/SP effective rates) + aws.cost.unblended (USD) — daily unblended cost per service/account/region + aws.cost.amortized (USD) — daily amortized cost (includes RI/SP effective rates) + aws.cost.undiscounted (USD) — daily list-price cost, ignoring credits/discounts/SP + fees (see KEPT_RECORD_TYPES). For an account under a + promotional/committed AWS credit, UnblendedCost/ + AmortizedCost read near-$0 (Credit == -Usage) even + though real infra consumption continues — this + metric is the only one that still shows it. Deployment modes: Lambda — deploy with deploy.sh; EventBridge triggers daily (recommended) @@ -38,6 +44,15 @@ OTEL_SERVICE_NAME = os.environ.get("OTEL_SERVICE_NAME", "aws-cost-reporter") CUSTOM_LABELS_RAW = os.environ.get("CUSTOM_LABELS", "") +# The only Cost Explorer RECORD_TYPE values that carry positive consumption at +# on-demand (list-price) value. Filtering TO these — rather than excluding a +# growing list of discount/credit RECORD_TYPEs by name — means a new discount +# type Cost Explorer adds later is automatically excluded from "undiscounted" +# instead of silently leaking in. Ported from the validated set in +# pde_daily_glassbox_cost_reporter_clone/l9cost/aws_cost.py (KEPT_RECORD_TYPES), +# measured across a 37-day, six-account audit on 2026-08-06. +UNDISCOUNTED_RECORD_TYPES = ["Usage", "SavingsPlanCoveredUsage"] + # ── Helpers ──────────────────────────────────────────────────────────────────── @@ -130,30 +145,103 @@ def fetch_costs(ce: object) -> list[dict]: return rows +def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], float]: + """ + Fetch daily list-price cost, filtered to RECORD_TYPE IN + UNDISCOUNTED_RECORD_TYPES (Usage, SavingsPlanCoveredUsage only) — the + on-demand value of what was actually consumed, ignoring any Credit, + Discount, or Savings-Plan-fee line item. + + Cost Explorer caps GroupBy at 2 dims and RECORD_TYPE can't be a Filter + dimension alongside a GroupBy on the same call without also grouping by + it, so this is a SEPARATE call from fetch_costs (same SERVICE+REGION + GroupBy, RECORD_TYPE added as a Filter instead of a GroupBy dimension — + that's allowed since Filter and GroupBy dimensions are independent). + + Returns {(date, service, account_id, region): undiscounted_usd} — a dict + keyed to merge directly onto fetch_costs' rows in send_otlp_metrics, + rather than a second parallel row list that would need its own + attribute-building logic. + """ + end = datetime.now(tz=timezone.utc).date() + start = end - timedelta(days=DAYS_BACK) + period = {"Start": str(start), "End": str(end)} + + accounts_resp = ce.get_dimension_values( + TimePeriod=period, Dimension="LINKED_ACCOUNT" + ) + accounts = [v["Value"] for v in accounts_resp.get("DimensionValues", [])] or [""] + + out: dict[tuple[str, str, str, str], float] = {} + for account_id in accounts: + next_token: str | None = None + while True: + filters = [{"Dimensions": {"Key": "RECORD_TYPE", "Values": UNDISCOUNTED_RECORD_TYPES}}] + if account_id: + filters.append({"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}}) + kwargs: dict = { + "TimePeriod": period, + "Granularity": "DAILY", + "Metrics": ["UnblendedCost"], + "GroupBy": [ + {"Type": "DIMENSION", "Key": "SERVICE"}, + {"Type": "DIMENSION", "Key": "REGION"}, + ], + "Filter": {"And": filters} if len(filters) > 1 else filters[0], + } + if next_token: + kwargs["NextPageToken"] = next_token + + resp = ce.get_cost_and_usage(**kwargs) + + for result in resp.get("ResultsByTime", []): + day = result["TimePeriod"]["Start"] + for group in result.get("Groups", []): + service, region = group["Keys"] + amount = float(group["Metrics"]["UnblendedCost"]["Amount"]) + if amount == 0.0: + continue + out[(day, service, account_id, region)] = amount + + next_token = resp.get("NextPageToken") + if not next_token: + break + + log.info("Fetched %d undiscounted cost rows (%s → %s, %d account(s))", + len(out), start, end, len(accounts)) + return out + + # ── OTLP export ──────────────────────────────────────────────────────────────── -def send_otlp_metrics(rows: list[dict]) -> None: - if not rows: +def send_otlp_metrics( + rows: list[dict], + undiscounted: dict[tuple[str, str, str, str], float] | None = None, +) -> None: + if not rows and not undiscounted: log.info("No cost rows to export") return unblended_dps: list[dict] = [] amortized_dps: list[dict] = [] + undiscounted_dps: list[dict] = [] custom_labels = _parse_headers(CUSTOM_LABELS_RAW) - for row in rows: - time_ns = _date_to_ns(row["date"]) - attrs = [{"key": "aws.service", "value": {"stringValue": row["service"]}}] + def _attrs(date: str, service: str, account_id: str, region: str) -> list[dict]: + attrs = [{"key": "aws.service", "value": {"stringValue": service}}] for k, v in custom_labels.items(): attrs.append({"key": k, "value": {"stringValue": v}}) - if row["account_id"]: - attrs.append( - {"key": "aws.account.id", "value": {"stringValue": row["account_id"]}} - ) - if row["region"]: - attrs.append({"key": "aws.region", "value": {"stringValue": row["region"]}}) - attrs.append({"key": "cost.date", "value": {"stringValue": row["date"]}}) + if account_id: + attrs.append({"key": "aws.account.id", "value": {"stringValue": account_id}}) + if region: + attrs.append({"key": "aws.region", "value": {"stringValue": region}}) + attrs.append({"key": "cost.date", "value": {"stringValue": date}}) + return attrs + + for row in rows: + time_ns = _date_to_ns(row["date"]) + attrs = _attrs(row["date"], row["service"], row["account_id"], row["region"]) if row["unblended"] != 0.0: unblended_dps.append( { @@ -171,6 +259,19 @@ def send_otlp_metrics(rows: list[dict]) -> None: } ) + # Independent metric, independent loop — deliberately NOT merged onto + # `rows`: a service can have Usage (and so an undiscounted entry) with + # zero net UnblendedCost after a credit fully offsets it (or vice versa), + # so the two row sets aren't guaranteed to line up 1:1. + for (date, service, account_id, region), amount in (undiscounted or {}).items(): + undiscounted_dps.append( + { + "attributes": _attrs(date, service, account_id, region), + "timeUnixNano": _date_to_ns(date), + "asDouble": amount, + } + ) + metrics = [] if unblended_dps: metrics.append( @@ -190,6 +291,20 @@ def send_otlp_metrics(rows: list[dict]) -> None: "gauge": {"dataPoints": amortized_dps}, } ) + if undiscounted_dps: + metrics.append( + { + "name": "aws.cost.undiscounted", + "unit": "USD", + "description": ( + "Daily list-price AWS cost by service, account, and region — " + "ignores credits/discounts/Savings-Plan fees, so it stays " + "meaningful even when a promotional credit drives " + "UnblendedCost/AmortizedCost to ~$0" + ), + "gauge": {"dataPoints": undiscounted_dps}, + } + ) payload = { "resourceMetrics": [ @@ -229,9 +344,10 @@ def send_otlp_metrics(rows: list[dict]) -> None: f"OTLP export failed: HTTP {resp.status_code} — {resp.text[:200]}" ) log.info( - "Exported %d unblended + %d amortized data points", + "Exported %d unblended + %d amortized + %d undiscounted data points", len(unblended_dps), len(amortized_dps), + len(undiscounted_dps), ) @@ -242,7 +358,8 @@ def send_otlp_metrics(rows: list[dict]) -> None: def poll(ce: object) -> None: rows = fetch_costs(ce) - send_otlp_metrics(rows) + undiscounted = fetch_undiscounted_costs(ce) + send_otlp_metrics(rows, undiscounted) def main() -> None: @@ -275,8 +392,9 @@ def _shutdown(sig, _frame): def lambda_handler(event: dict, context: object) -> dict: ce = boto3.client("ce", region_name="us-east-1") rows = fetch_costs(ce) - send_otlp_metrics(rows) - return {"statusCode": 200, "exported": len(rows)} + undiscounted = fetch_undiscounted_costs(ce) + send_otlp_metrics(rows, undiscounted) + return {"statusCode": 200, "exported": len(rows), "exported_undiscounted": len(undiscounted)} if __name__ == "__main__": diff --git a/aws-cost-explorer/test_parse.py b/aws-cost-explorer/test_parse.py index 5611b45..dbf8be0 100644 --- a/aws-cost-explorer/test_parse.py +++ b/aws-cost-explorer/test_parse.py @@ -9,7 +9,7 @@ os.environ.setdefault("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost") -from main import _date_to_ns, _parse_headers +from main import UNDISCOUNTED_RECORD_TYPES, _date_to_ns, _parse_headers, fetch_undiscounted_costs def test_parse_headers_single() -> None: @@ -47,6 +47,59 @@ def test_date_to_ns_noon_utc() -> None: assert ns == 1768478400 * 1_000_000_000 +class _StubCE: + """Records every get_cost_and_usage call's Filter so the test can assert + RECORD_TYPE is a Filter (not a GroupBy dimension) — a real boto3 call + with RECORD_TYPE as a third GroupBy dimension raises + "Only two values for GroupBy are allowed", which this filter-based + approach must avoid. One fixed ResultsByTime per account, returned in + call order (fetch_undiscounted_costs makes exactly one call per account + with no pagination in these tests).""" + + def __init__(self, account_ids: list[str], results_by_time: list[dict]) -> None: + self.account_ids = account_ids + self.results_by_time = results_by_time + self.calls: list[dict] = [] + + def get_dimension_values(self, **kwargs): + return {"DimensionValues": [{"Value": acct} for acct in self.account_ids]} + + def get_cost_and_usage(self, **kwargs): + self.calls.append(kwargs) + return {"ResultsByTime": self.results_by_time} + + +def test_fetch_undiscounted_costs_filters_by_record_type() -> None: + stub = _StubCE(["111111111111"], [{ + "TimePeriod": {"Start": "2026-08-30"}, + "Groups": [{"Keys": ["Amazon EC2", "us-east-1"], + "Metrics": {"UnblendedCost": {"Amount": "12.5"}}}], + }]) + out = fetch_undiscounted_costs(stub) + + assert out == {("2026-08-30", "Amazon EC2", "111111111111", "us-east-1"): 12.5} + # Every call must filter RECORD_TYPE to exactly UNDISCOUNTED_RECORD_TYPES, + # and RECORD_TYPE must be a Filter dimension, never a third GroupBy dim + # (Cost Explorer caps GroupBy at 2). + assert len(stub.calls) == 1 + call = stub.calls[0] + assert len(call["GroupBy"]) == 2 + assert all(g["Key"] != "RECORD_TYPE" for g in call["GroupBy"]) + record_type_filter = call["Filter"]["And"][0]["Dimensions"] + assert record_type_filter["Key"] == "RECORD_TYPE" + assert record_type_filter["Values"] == UNDISCOUNTED_RECORD_TYPES + + +def test_fetch_undiscounted_costs_skips_zero_amounts() -> None: + stub = _StubCE([""], [{ + "TimePeriod": {"Start": "2026-08-30"}, + "Groups": [{"Keys": ["Amazon S3", "us-east-1"], + "Metrics": {"UnblendedCost": {"Amount": "0"}}}], + }]) + out = fetch_undiscounted_costs(stub) + assert out == {} + + if __name__ == "__main__": tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] passed = failed = 0 From 4fbd9646d94863ae0ac9e69ebc0195a982d7adb9 Mon Sep 17 00:00:00 2001 From: amanpruthi Date: Mon, 31 Aug 2026 10:55:21 +0530 Subject: [PATCH 2/3] fix(cost-explorer): delay capture by CAPTURE_OFFSET_DAYS to avoid estimate drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while independently validating exported numbers against raw AWS CLI: Cost Explorer's daily cost is an ESTIMATE that keeps revising for ~2-3 days after the day passes. Measured live: a Marketplace line item for a given day was captured ~47% low the next day at DAYS_BACK=1, but the SAME day's live Cost Explorer value a few days later had settled to its final, higher value. Older days (5+ days old) matched exactly, confirming the estimate settles within a few days. Since each (tenant, service, region, cost_date) combination is written to Last9's TSDB exactly once, and the TSDB accepts neither backfilled nor overwritten samples for the same labels+timestamp, a day captured too early can never be corrected afterward — every prior day using DAYS_BACK=1/no offset is permanently understated by whatever the drift was at capture time. Adds CAPTURE_OFFSET_DAYS (default 3), which shifts the whole [start, end) fetch window back by this many days — "yesterday" becomes "CAPTURE_OFFSET_DAYS days ago", captured only after Cost Explorer's estimate has settled. Trade-off: the report is CAPTURE_OFFSET_DAYS days behind "today" instead of 1 — documented in README/deploy.sh/.env.example/cloudformation.yaml. New test verifies the fetch window's End date is actually offset from today by CAPTURE_OFFSET_DAYS, not just an unused env var. Co-Authored-By: Claude Sonnet 5 --- aws-cost-explorer/.env.example | 7 +++++++ aws-cost-explorer/README.md | 7 +++++++ aws-cost-explorer/cloudformation.yaml | 6 ++++++ aws-cost-explorer/deploy.sh | 9 ++++++++- aws-cost-explorer/main.py | 23 +++++++++++++++++++-- aws-cost-explorer/test_parse.py | 29 +++++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 3 deletions(-) diff --git a/aws-cost-explorer/.env.example b/aws-cost-explorer/.env.example index f61fcc8..4be95f0 100644 --- a/aws-cost-explorer/.env.example +++ b/aws-cost-explorer/.env.example @@ -6,6 +6,13 @@ AWS_DEFAULT_REGION=us-east-1 # Days of history to fetch per run. Use 1 for steady-state. Set to 30+ for initial backfill. DAYS_BACK=1 +# Shifts the fetch window this many days into the past. Cost Explorer's daily +# estimate keeps revising for ~2-3 days after the day passes, and a day can +# never be re-fetched/corrected once written to Last9 — this captures each +# day only after its estimate has settled. See README "Why capture is +# delayed" before lowering this. +CAPTURE_OFFSET_DAYS=3 + # How often to re-poll (seconds). Cost Explorer data updates ~once/day. POLL_INTERVAL_SECONDS=86400 diff --git a/aws-cost-explorer/README.md b/aws-cost-explorer/README.md index 7b91d61..9b92ffa 100644 --- a/aws-cost-explorer/README.md +++ b/aws-cost-explorer/README.md @@ -57,6 +57,7 @@ docker compose up | `AWS_SECRET_ACCESS_KEY` | No* | — | AWS secret key (Docker only) | | `AWS_DEFAULT_REGION` | No | `us-east-1` | AWS region | | `DAYS_BACK` | No | `1` | Days of history to fetch per run. Set to `30`+ for initial backfill, then drop to `1`. | +| `CAPTURE_OFFSET_DAYS` | No | `3` | Shifts the fetch window this many days into the past — see "Why capture is delayed" below. | | `POLL_INTERVAL_SECONDS` | No | `86400` | Re-poll interval for Docker mode | | `OTEL_SERVICE_NAME` | No | `aws-cost-reporter` | Service name in Last9 | @@ -78,6 +79,12 @@ An account under a promotional or committed AWS credit shows `Credit == -Usage` `cost.date` (`YYYY-MM-DD`) encodes the billing date as a label so each day forms a distinct series. This matters when `DAYS_BACK > 1`: without it, all fetched days share identical labels and only the last sample survives per series. At the default `DAYS_BACK=1`, `cost.date` adds one unique label value per run with no cardinality overhead. +### Why capture is delayed by `CAPTURE_OFFSET_DAYS` + +Cost Explorer's daily cost is an **estimate** that keeps revising for roughly 2-3 days after the day passes — measured live: a Marketplace line item read 47% low the day after capture (`DAYS_BACK=1`/no offset), then settled to its final value by day 3. Each `(tenant, service, region, cost_date)` combination is written to Last9 exactly once, and Last9's TSDB accepts neither backfilled nor overwritten samples for the same labels+timestamp — so a day captured too early can never be corrected afterward. + +`CAPTURE_OFFSET_DAYS` (default `3`) shifts the whole fetch window back by this many days, so "yesterday" (with `DAYS_BACK=1`) becomes "3 days ago" — captured only after Cost Explorer's estimate has settled. The tradeoff: the report is `CAPTURE_OFFSET_DAYS` days behind "today" instead of 1. Do not lower this to chase freshness without accepting that recent days will read low until they settle. + ## Dashboard Import `last9-dashboard.json` into Last9 for a pre-built 14-panel cost dashboard covering spend totals, daily trends, top services, and per-account breakdown. diff --git a/aws-cost-explorer/cloudformation.yaml b/aws-cost-explorer/cloudformation.yaml index e7253f4..e16c917 100644 --- a/aws-cost-explorer/cloudformation.yaml +++ b/aws-cost-explorer/cloudformation.yaml @@ -28,6 +28,11 @@ Parameters: Default: 1 Description: "Days of cost history to fetch per run. Use 1 for steady-state daily runs. Set to 30+ for initial backfill, then reduce to 1." + CaptureOffsetDays: + Type: Number + Default: 3 + Description: "Shifts the fetch window this many days into the past. Cost Explorer's daily estimate keeps revising for ~2-3 days after the day passes, and a day can never be re-fetched/corrected once written to Last9 — this captures each day only after its estimate has settled." + Schedule: Type: String Default: rate(1 day) @@ -84,6 +89,7 @@ Resources: OTEL_EXPORTER_OTLP_HEADERS: !Ref OtlpHeaders OTEL_SERVICE_NAME: !Ref ServiceName DAYS_BACK: !Ref DaysBack + CAPTURE_OFFSET_DAYS: !Ref CaptureOffsetDays CUSTOM_LABELS: !Ref CustomLabels Code: S3Bucket: !Ref LambdaS3Bucket diff --git a/aws-cost-explorer/deploy.sh b/aws-cost-explorer/deploy.sh index 1f73dd5..88d49bc 100755 --- a/aws-cost-explorer/deploy.sh +++ b/aws-cost-explorer/deploy.sh @@ -18,6 +18,11 @@ # FUNCTION_NAME=aws-cost-reporter (default) # AWS_REGION=us-east-1 (default) # DAYS_BACK=1 (default; use 30+ for initial backfill) +# CAPTURE_OFFSET_DAYS=3 (default; Cost Explorer's daily estimate keeps +# revising for ~2-3 days after the day passes, and +# a day can never be re-fetched/corrected once +# written — this shifts the fetch window back so +# each day is captured only after it has settled) # SCHEDULE=rate(1 day) (default) # OTEL_SERVICE_NAME=aws-cost-reporter # CUSTOM_LABELS=env=prod,team=platform (optional; extra labels on every exported metric) @@ -29,6 +34,7 @@ set -euo pipefail FUNCTION_NAME="${FUNCTION_NAME:-aws-cost-reporter}" AWS_REGION="${AWS_REGION:-us-east-1}" DAYS_BACK="${DAYS_BACK:-1}" +CAPTURE_OFFSET_DAYS="${CAPTURE_OFFSET_DAYS:-3}" SCHEDULE="${SCHEDULE:-rate(1 day)}" OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-aws-cost-reporter}" CUSTOM_LABELS="${CUSTOM_LABELS:-}" @@ -142,7 +148,7 @@ else # JWTs, comma-bearing service names). JSON escapes everything correctly. ENV_FILE=$(mktemp /tmp/aws-cost-reporter-env-XXXXXX.json) TRAP_ENV_FILE="${ENV_FILE}" - export OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_HEADERS OTEL_SERVICE_NAME DAYS_BACK CUSTOM_LABELS + export OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_HEADERS OTEL_SERVICE_NAME DAYS_BACK CAPTURE_OFFSET_DAYS CUSTOM_LABELS python3 -c ' import json, os print(json.dumps({"Variables": { @@ -150,6 +156,7 @@ print(json.dumps({"Variables": { "OTEL_EXPORTER_OTLP_HEADERS": os.environ["OTEL_EXPORTER_OTLP_HEADERS"], "OTEL_SERVICE_NAME": os.environ["OTEL_SERVICE_NAME"], "DAYS_BACK": os.environ["DAYS_BACK"], + "CAPTURE_OFFSET_DAYS": os.environ["CAPTURE_OFFSET_DAYS"], "CUSTOM_LABELS": os.environ["CUSTOM_LABELS"], }}))' > "${ENV_FILE}" ENV_VARS="file://${ENV_FILE}" diff --git a/aws-cost-explorer/main.py b/aws-cost-explorer/main.py index 4045bce..8f77124 100644 --- a/aws-cost-explorer/main.py +++ b/aws-cost-explorer/main.py @@ -14,6 +14,15 @@ though real infra consumption continues — this metric is the only one that still shows it. +Cost Explorer's daily cost is an ESTIMATE that keeps revising for ~2-3 days +after the day passes (measured: a Marketplace line item read 47% low one day +after capture, settled to its final value by day 3). Every day is captured +exactly once and Last9's TSDB accepts neither backfilled nor overwritten +samples for the same (labels, timestamp) — so CAPTURE_OFFSET_DAYS (default 3) +shifts what "today" means for this job's fetch window, capturing each day +late enough that its estimate has already settled instead of catching it +mid-revision. + Deployment modes: Lambda — deploy with deploy.sh; EventBridge triggers daily (recommended) Docker — docker compose up (for local testing or non-AWS environments) @@ -37,6 +46,16 @@ AWS_REGION = os.environ.get("AWS_DEFAULT_REGION", "us-east-1") DAYS_BACK = int(os.environ.get("DAYS_BACK", "1")) +# Cost Explorer's daily estimate keeps revising for ~2-3 days after the day +# passes (measured live 2026-08-31: a day captured at DAYS_BACK=1/OFFSET=0 +# read 47% low one day later — e.g. tazapay's Altinity line went from +# $13.995 to its settled $20.522). Since Last9's TSDB accepts neither +# backfilled nor overwritten samples for an existing (labels, timestamp) +# pair, there is no way to correct a day after it's written — so capture +# each day late enough that it's already settled instead. OFFSET shifts the +# whole [start, end) window back by this many days; default 3 waits past +# the observed drift window. +CAPTURE_OFFSET_DAYS = int(os.environ.get("CAPTURE_OFFSET_DAYS", "3")) POLL_INTERVAL_SECONDS = int(os.environ.get("POLL_INTERVAL_SECONDS", "86400")) OTLP_ENDPOINT = os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] @@ -81,7 +100,7 @@ def fetch_costs(ce: object) -> list[dict]: outer Filter to preserve the service × account × region combination. Returns flat list of {date, service, account_id, region, unblended, amortized}. """ - end = datetime.now(tz=timezone.utc).date() + end = datetime.now(tz=timezone.utc).date() - timedelta(days=CAPTURE_OFFSET_DAYS) start = end - timedelta(days=DAYS_BACK) period = {"Start": str(start), "End": str(end)} @@ -163,7 +182,7 @@ def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], floa rather than a second parallel row list that would need its own attribute-building logic. """ - end = datetime.now(tz=timezone.utc).date() + end = datetime.now(tz=timezone.utc).date() - timedelta(days=CAPTURE_OFFSET_DAYS) start = end - timedelta(days=DAYS_BACK) period = {"Start": str(start), "End": str(end)} diff --git a/aws-cost-explorer/test_parse.py b/aws-cost-explorer/test_parse.py index dbf8be0..ace8ad8 100644 --- a/aws-cost-explorer/test_parse.py +++ b/aws-cost-explorer/test_parse.py @@ -90,6 +90,35 @@ def test_fetch_undiscounted_costs_filters_by_record_type() -> None: assert record_type_filter["Values"] == UNDISCOUNTED_RECORD_TYPES +def test_fetch_costs_respects_capture_offset_days() -> None: + """The fetch window must end CAPTURE_OFFSET_DAYS ago, not "today" — + Cost Explorer's daily estimate keeps revising for ~2-3 days after the day + passes, and a day can never be corrected once written to Last9, so the + default (3) must actually shift the window, not just exist as an unused + env var.""" + import importlib + import os + from datetime import datetime, timedelta, timezone + + os.environ["CAPTURE_OFFSET_DAYS"] = "5" + try: + import main + importlib.reload(main) + + stub = _StubCE([""], []) + main.fetch_costs(stub) + + assert len(stub.calls) == 1 + period = stub.calls[0]["TimePeriod"] + expected_end = datetime.now(tz=timezone.utc).date() - timedelta(days=5) + assert period["End"] == str(expected_end), ( + f"expected End={expected_end} (today - CAPTURE_OFFSET_DAYS=5), got {period['End']}" + ) + finally: + del os.environ["CAPTURE_OFFSET_DAYS"] + importlib.reload(main) + + def test_fetch_undiscounted_costs_skips_zero_amounts() -> None: stub = _StubCE([""], [{ "TimePeriod": {"Start": "2026-08-30"}, From c45fc7475ecc463daeb15b7c77d3d510ba9d9b49 Mon Sep 17 00:00:00 2001 From: amanpruthi Date: Mon, 28 Sep 2026 12:59:25 +0530 Subject: [PATCH 3/3] fix(cost-explorer): pin one billing window per run, pass CaptureOffsetDays to CF - poll/lambda_handler compute the [start, end) period once and pass it to both fetch_costs and fetch_undiscounted_costs, so a run crossing UTC midnight can't query different billing days for different metric families. - deploy.sh: forward CAPTURE_OFFSET_DAYS as CaptureOffsetDays in the CloudFormation --parameter-overrides (direct-Lambda path already did). - Regression tests for both (midnight rollover; deploy.sh with stubbed aws in both CF and direct-Lambda modes). - ruff format + lint fixes; aws-cur lint (unused noqa, date.today) was also failing on main under current ruff. Co-Authored-By: Claude Opus 5.5 --- aws-cost-explorer/deploy.sh | 1 + aws-cost-explorer/main.py | 67 ++++++++---- aws-cost-explorer/test_parse.py | 174 ++++++++++++++++++++++++++++---- aws-cur/test_local.py | 6 +- 4 files changed, 207 insertions(+), 41 deletions(-) diff --git a/aws-cost-explorer/deploy.sh b/aws-cost-explorer/deploy.sh index 88d49bc..71ec01b 100755 --- a/aws-cost-explorer/deploy.sh +++ b/aws-cost-explorer/deploy.sh @@ -86,6 +86,7 @@ if [[ "${USE_CF}" == "1" ]]; then LambdaS3Bucket="${CF_S3_BUCKET}" \ LambdaS3Key="${CF_S3_KEY}" \ DaysBack="${DAYS_BACK}" \ + CaptureOffsetDays="${CAPTURE_OFFSET_DAYS}" \ Schedule="${SCHEDULE}" \ ServiceName="${OTEL_SERVICE_NAME}" \ CustomLabels="${CUSTOM_LABELS}" diff --git a/aws-cost-explorer/main.py b/aws-cost-explorer/main.py index 8f77124..8787c12 100644 --- a/aws-cost-explorer/main.py +++ b/aws-cost-explorer/main.py @@ -92,7 +92,16 @@ def _date_to_ns(date_str: str) -> str: # ── Cost Explorer fetch ──────────────────────────────────────────────────────── -def fetch_costs(ce: object) -> list[dict]: +def _billing_period() -> dict[str, str]: + """[start, end) window shared by every fetch in one run. Computed once per + run so a collection that crosses UTC midnight can't query different + billing days for different metric families.""" + end = datetime.now(tz=timezone.utc).date() - timedelta(days=CAPTURE_OFFSET_DAYS) + start = end - timedelta(days=DAYS_BACK) + return {"Start": str(start), "End": str(end)} + + +def fetch_costs(ce: object, period: dict[str, str]) -> list[dict]: """ Fetch daily costs grouped by SERVICE, REGION, looped per LINKED_ACCOUNT. @@ -100,9 +109,6 @@ def fetch_costs(ce: object) -> list[dict]: outer Filter to preserve the service × account × region combination. Returns flat list of {date, service, account_id, region, unblended, amortized}. """ - end = datetime.now(tz=timezone.utc).date() - timedelta(days=CAPTURE_OFFSET_DAYS) - start = end - timedelta(days=DAYS_BACK) - period = {"Start": str(start), "End": str(end)} accounts_resp = ce.get_dimension_values( TimePeriod=period, Dimension="LINKED_ACCOUNT" @@ -157,14 +163,16 @@ def fetch_costs(ce: object) -> list[dict]: log.info( "Fetched %d cost rows (%s → %s, %d account(s))", len(rows), - start, - end, + period["Start"], + period["End"], len(accounts), ) return rows -def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], float]: +def fetch_undiscounted_costs( + ce: object, period: dict[str, str] +) -> dict[tuple[str, str, str, str], float]: """ Fetch daily list-price cost, filtered to RECORD_TYPE IN UNDISCOUNTED_RECORD_TYPES (Usage, SavingsPlanCoveredUsage only) — the @@ -182,9 +190,6 @@ def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], floa rather than a second parallel row list that would need its own attribute-building logic. """ - end = datetime.now(tz=timezone.utc).date() - timedelta(days=CAPTURE_OFFSET_DAYS) - start = end - timedelta(days=DAYS_BACK) - period = {"Start": str(start), "End": str(end)} accounts_resp = ce.get_dimension_values( TimePeriod=period, Dimension="LINKED_ACCOUNT" @@ -195,9 +200,18 @@ def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], floa for account_id in accounts: next_token: str | None = None while True: - filters = [{"Dimensions": {"Key": "RECORD_TYPE", "Values": UNDISCOUNTED_RECORD_TYPES}}] + filters = [ + { + "Dimensions": { + "Key": "RECORD_TYPE", + "Values": UNDISCOUNTED_RECORD_TYPES, + } + } + ] if account_id: - filters.append({"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}}) + filters.append( + {"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}} + ) kwargs: dict = { "TimePeriod": period, "Granularity": "DAILY", @@ -226,8 +240,13 @@ def fetch_undiscounted_costs(ce: object) -> dict[tuple[str, str, str, str], floa if not next_token: break - log.info("Fetched %d undiscounted cost rows (%s → %s, %d account(s))", - len(out), start, end, len(accounts)) + log.info( + "Fetched %d undiscounted cost rows (%s → %s, %d account(s))", + len(out), + period["Start"], + period["End"], + len(accounts), + ) return out @@ -252,7 +271,9 @@ def _attrs(date: str, service: str, account_id: str, region: str) -> list[dict]: for k, v in custom_labels.items(): attrs.append({"key": k, "value": {"stringValue": v}}) if account_id: - attrs.append({"key": "aws.account.id", "value": {"stringValue": account_id}}) + attrs.append( + {"key": "aws.account.id", "value": {"stringValue": account_id}} + ) if region: attrs.append({"key": "aws.region", "value": {"stringValue": region}}) attrs.append({"key": "cost.date", "value": {"stringValue": date}}) @@ -376,8 +397,9 @@ def _attrs(date: str, service: str, account_id: str, region: str) -> list[dict]: def poll(ce: object) -> None: - rows = fetch_costs(ce) - undiscounted = fetch_undiscounted_costs(ce) + period = _billing_period() + rows = fetch_costs(ce, period) + undiscounted = fetch_undiscounted_costs(ce, period) send_otlp_metrics(rows, undiscounted) @@ -410,10 +432,15 @@ def _shutdown(sig, _frame): def lambda_handler(event: dict, context: object) -> dict: ce = boto3.client("ce", region_name="us-east-1") - rows = fetch_costs(ce) - undiscounted = fetch_undiscounted_costs(ce) + period = _billing_period() + rows = fetch_costs(ce, period) + undiscounted = fetch_undiscounted_costs(ce, period) send_otlp_metrics(rows, undiscounted) - return {"statusCode": 200, "exported": len(rows), "exported_undiscounted": len(undiscounted)} + return { + "statusCode": 200, + "exported": len(rows), + "exported_undiscounted": len(undiscounted), + } if __name__ == "__main__": diff --git a/aws-cost-explorer/test_parse.py b/aws-cost-explorer/test_parse.py index ace8ad8..996dedf 100644 --- a/aws-cost-explorer/test_parse.py +++ b/aws-cost-explorer/test_parse.py @@ -9,7 +9,14 @@ os.environ.setdefault("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost") -from main import UNDISCOUNTED_RECORD_TYPES, _date_to_ns, _parse_headers, fetch_undiscounted_costs +from main import ( + UNDISCOUNTED_RECORD_TYPES, + _date_to_ns, + _parse_headers, + fetch_undiscounted_costs, +) + +PERIOD = {"Start": "2026-08-30", "End": "2026-08-31"} def test_parse_headers_single() -> None: @@ -70,12 +77,21 @@ def get_cost_and_usage(self, **kwargs): def test_fetch_undiscounted_costs_filters_by_record_type() -> None: - stub = _StubCE(["111111111111"], [{ - "TimePeriod": {"Start": "2026-08-30"}, - "Groups": [{"Keys": ["Amazon EC2", "us-east-1"], - "Metrics": {"UnblendedCost": {"Amount": "12.5"}}}], - }]) - out = fetch_undiscounted_costs(stub) + stub = _StubCE( + ["111111111111"], + [ + { + "TimePeriod": {"Start": "2026-08-30"}, + "Groups": [ + { + "Keys": ["Amazon EC2", "us-east-1"], + "Metrics": {"UnblendedCost": {"Amount": "12.5"}}, + } + ], + } + ], + ) + out = fetch_undiscounted_costs(stub, PERIOD) assert out == {("2026-08-30", "Amazon EC2", "111111111111", "us-east-1"): 12.5} # Every call must filter RECORD_TYPE to exactly UNDISCOUNTED_RECORD_TYPES, @@ -103,13 +119,10 @@ def test_fetch_costs_respects_capture_offset_days() -> None: os.environ["CAPTURE_OFFSET_DAYS"] = "5" try: import main - importlib.reload(main) - stub = _StubCE([""], []) - main.fetch_costs(stub) + importlib.reload(main) - assert len(stub.calls) == 1 - period = stub.calls[0]["TimePeriod"] + period = main._billing_period() expected_end = datetime.now(tz=timezone.utc).date() - timedelta(days=5) assert period["End"] == str(expected_end), ( f"expected End={expected_end} (today - CAPTURE_OFFSET_DAYS=5), got {period['End']}" @@ -120,15 +133,140 @@ def test_fetch_costs_respects_capture_offset_days() -> None: def test_fetch_undiscounted_costs_skips_zero_amounts() -> None: - stub = _StubCE([""], [{ - "TimePeriod": {"Start": "2026-08-30"}, - "Groups": [{"Keys": ["Amazon S3", "us-east-1"], - "Metrics": {"UnblendedCost": {"Amount": "0"}}}], - }]) - out = fetch_undiscounted_costs(stub) + stub = _StubCE( + [""], + [ + { + "TimePeriod": {"Start": "2026-08-30"}, + "Groups": [ + { + "Keys": ["Amazon S3", "us-east-1"], + "Metrics": {"UnblendedCost": {"Amount": "0"}}, + } + ], + } + ], + ) + out = fetch_undiscounted_costs(stub, PERIOD) assert out == {} +def test_poll_uses_one_billing_window_across_utc_midnight() -> None: + """poll must compute the window once: if the clock rolls past UTC + midnight between fetch_costs and fetch_undiscounted_costs, both metric + families must still query the same [start, end) — otherwise undiscounted + describes a different day than unblended/amortized and, at DAYS_BACK=1, + silently skips a day.""" + from datetime import datetime, timezone + from unittest.mock import patch + + import main + + ticks = iter( + [ + datetime(2026, 9, 15, 23, 59, 59, tzinfo=timezone.utc), + datetime(2026, 9, 16, 0, 0, 1, tzinfo=timezone.utc), + ] + ) + + class _Clock(datetime): + @classmethod + def now(cls, tz=None): + return next(ticks, datetime(2026, 9, 16, 0, 0, 2, tzinfo=timezone.utc)) + + periods: list[dict] = [] + + class _CE: + def get_dimension_values(self, **kwargs): + periods.append(kwargs["TimePeriod"]) + return {"DimensionValues": [{"Value": "111111111111"}]} + + def get_cost_and_usage(self, **kwargs): + periods.append(kwargs["TimePeriod"]) + return {"ResultsByTime": []} + + with ( + patch.object(main, "datetime", _Clock), + patch.object(main, "send_otlp_metrics"), + ): + main.poll(_CE()) + + windows = {(p["Start"], p["End"]) for p in periods} + assert len(windows) == 1, f"one poll mixed billing windows: {windows}" + + +def _run_deploy(use_cf: str) -> list[dict]: + """Runs deploy.sh with aws/pip3/zip stubbed on PATH; returns every aws + CLI invocation (plus the parsed --environment file, if any).""" + import json + import subprocess + import sys + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + log = root / "aws.jsonl" + aws = root / "aws" + aws.write_text( + "#!" + sys.executable + "\n" + "import json, os, sys\n" + "from pathlib import Path\n" + "args = sys.argv[1:]\n" + "entry = {'args': args}\n" + "if '--environment' in args:\n" + " v = args[args.index('--environment') + 1]\n" + " entry['environment'] = json.loads(" + "Path(v.removeprefix('file://')).read_text())\n" + "open(os.environ['STUB_AWS_LOG'], 'a').write(json.dumps(entry) + '\\n')\n" + "if args[:2] == ['sts', 'get-caller-identity']:\n" + " print('111111111111')\n" + "else:\n" + " print('arn:aws:lambda:us-east-1:111111111111:function:stub')\n" + ) + aws.chmod(0o755) + for cmd in ("pip3", "zip"): + (root / cmd).write_text("#!/bin/sh\nexit 0\n") + (root / cmd).chmod(0o755) + env = dict( + os.environ, + PATH=str(root) + os.pathsep + os.environ["PATH"], + USE_CF=use_cf, + CF_S3_BUCKET="stub-bucket", + OTEL_EXPORTER_OTLP_ENDPOINT="https://collector.example.test", + OTEL_EXPORTER_OTLP_HEADERS="x-test=stub", + CAPTURE_OFFSET_DAYS="7", + DAYS_BACK="2", + SMOKE_TEST="0", + STUB_AWS_LOG=str(log), + ) + subprocess.run( + ["bash", "deploy.sh"], + env=env, + check=True, + capture_output=True, + timeout=30, + cwd=Path(__file__).parent, + ) + return [json.loads(line) for line in log.read_text().splitlines()] + + +def test_deploy_cloudformation_passes_capture_offset() -> None: + calls = _run_deploy("1") + args = next( + c["args"] for c in calls if c["args"][:2] == ["cloudformation", "deploy"] + ) + assert "DaysBack=2" in args # existing override: positive control + assert "CaptureOffsetDays=7" in args, "CloudFormation drops CAPTURE_OFFSET_DAYS" + + +def test_deploy_direct_lambda_passes_capture_offset() -> None: + calls = _run_deploy("0") + env = next(c["environment"]["Variables"] for c in calls if "environment" in c) + assert env["CAPTURE_OFFSET_DAYS"] == "7" + assert env["DAYS_BACK"] == "2" + + if __name__ == "__main__": tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] passed = failed = 0 diff --git a/aws-cur/test_local.py b/aws-cur/test_local.py index 7981675..5893e5f 100644 --- a/aws-cur/test_local.py +++ b/aws-cur/test_local.py @@ -16,7 +16,7 @@ from __future__ import annotations import os -from datetime import date, timedelta +from datetime import datetime, timedelta, timezone import pandas as pd @@ -24,13 +24,13 @@ os.environ.setdefault("CUR_S3_BUCKET", "local") os.environ.setdefault("CUR_REPORT_NAME", "local") -from main import aggregate_costs, send_otlp_metrics # noqa: E402 +from main import aggregate_costs, send_otlp_metrics def _make_cur_df(days: int = 7) -> pd.DataFrame: """Build a minimal CUR-shaped DataFrame covering key line item types.""" rows = [] - today = date.today() + today = datetime.now(tz=timezone.utc).date() services = [ ("AmazonEC2", "us-east-1", "BoxUsage:t3.medium"),